diff --git "a/logs/none_1_grtqk5/attempt_0/1/stderr.log" "b/logs/none_1_grtqk5/attempt_0/1/stderr.log" new file mode 100644--- /dev/null +++ "b/logs/none_1_grtqk5/attempt_0/1/stderr.log" @@ -0,0 +1,42688 @@ +OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k +OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k +OpenBLAS WARNING - could not determine the L2 cache size on this system, assuming 256k +wandb: Appending key for api.wandb.ai to your netrc file: /home/cvm/.netrc +wandb: Currently logged in as: zaydzuhri to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured. +[titan] 2025-07-09 13:32:39,773 - root - INFO - Starting job: default job +[titan] 2025-07-09 13:32:39,773 - root - INFO - { + "activation_checkpoint": { + "mode": "none", + "selective_ac_option": "2" + }, + "activation_offload": { + "mode": "none" + }, + "checkpoint": { + "async_mode": "disabled", + "convert_to_hf_on_save": false, + "create_seed_checkpoint": false, + "enable_checkpoint": true, + "exclude_from_loading": [], + "export_dtype": "float32", + "folder": "checkpoint", + "hf_repo_base_name": "zaydzuhri/mtp-340M-4096-batch16-steps100000", + "hf_upload_enabled": true, + "hf_upload_format": "dcp", + "interval": 10000, + "interval_type": "steps", + "keep_latest_k": 0, + "load_step": -1, + "model_weights_only": false + }, + "comm": { + "init_timeout_seconds": 1800, + "trace_buf_size": 20000, + "train_timeout_seconds": 900 + }, + "experimental": { + "context_parallel_degree": 1, + "context_parallel_rotate_method": "allgather", + "custom_model_path": "", + "enable_async_tensor_parallel": false, + "enable_compiled_autograd": false, + "pipeline_parallel_degree": 1, + "pipeline_parallel_microbatches": null, + "pipeline_parallel_schedule": "1F1B", + "pipeline_parallel_schedule_csv": "", + "pipeline_parallel_split_points": [] + }, + "fault_tolerance": { + "enable": false, + "group_size": 0, + "min_replica_size": 1, + "replica_id": 0 + }, + "float8": { + "enable_fsdp_float8_all_gather": false, + "force_recompute_fp8_weight_in_bwd": false, + "precompute_float8_dynamic_scale_for_fsdp": false, + "recipe_name": null + }, + "job": { + "config_file": "flame/models/fla.toml", + "description": "default job", + "dump_folder": "exp/mtp.340M.batch16.seqlen4096.context4096.warmup1000.update1.steps100000.lr3e-4.cosine", + "print_args": true, + "use_for_integration_test": false + }, + "lr_scheduler": { + "decay_ratio": null, + "decay_type": "cosine", + "lr_min": 0.1, + "warmup_steps": 1000 + }, + "memory_estimation": { + "disable_fake_mode": false, + "enabled": false + }, + "metrics": { + "disable_color_printing": false, + "enable_tensorboard": false, + "enable_wandb": true, + "log_freq": 5, + "save_for_all_ranks": false, + "save_tb_folder": "tb" + }, + "model": { + "config": "configs/mtp_transformer_340M.json", + "converters": [], + "name": "fla", + "print_after_conversion": false, + "tokenizer_path": "fla-hub/transformer-1.3B-100B" + }, + "optimizer": { + "early_step_in_backward": false, + "eps": 1e-15, + "implementation": "fused", + "lr": 0.0003, + "name": "AdamW" + }, + "profiling": { + "enable_memory_snapshot": false, + "enable_profiling": true, + "profile_freq": 512, + "save_memory_snapshot_folder": "memory_snapshot", + "save_traces_folder": "profile_trace" + }, + "training": { + "batch_size": 16, + "compile": true, + "context_len": 4096, + "data_dir": null, + "data_files": null, + "data_parallel_replicate_degree": 1, + "data_parallel_shard_degree": -1, + "data_probs": null, + "dataset": "/home/cvm/.cache/HuggingFaceFW___fineweb-edu/sample-100BT", + "dataset_name": "default", + "dataset_split": "train", + "deterministic": false, + "disable_loss_parallel": false, + "enable_cpu_offload": false, + "fsdp_reshard_after_forward": "default", + "gc_freq": 50, + "gradient_accumulation_steps": 1, + "max_norm": 1.0, + "mixed_precision_param": "bfloat16", + "mixed_precision_reduce": "float32", + "num_workers": 32, + "persistent_workers": false, + "pin_memory": false, + "prefetch_factor": 2, + "seed": 79, + "seq_len": 4096, + "skip_nan_inf": true, + "steps": 100000, + "streaming": false, + "tensor_parallel_degree": 1, + "varlen": false + } +} +[titan] 2025-07-09 13:32:39,774 - root - INFO - [GC] Initial GC collection. 0.00 seconds. +[titan] 2025-07-09 13:33:06,685 - root - INFO - Target Hugging Face repository for this run: zaydzuhri/mtp-340M-4096-batch16-steps100000-20250709-133306 +[titan] 2025-07-09 13:33:06,685 - root - WARNING - ENV[TORCH_NCCL_ASYNC_ERROR_HANDLING] = 1 will be overridden to 3 based on job config +[titan] 2025-07-09 13:33:06,687 - root - INFO - CUDA capacity: NVIDIA H200 with 139.36GiB memory +[titan] 2025-07-09 13:33:06,740 - root - INFO - Peak FLOPS used for computing MFU: 9.890e+14 +[titan] 2025-07-09 13:33:06,740 - root - INFO - Building 1-D device mesh with ['dp_shard'], [8] +[titan] 2025-07-09 13:33:08,611 - root - INFO - Loading tokenizer... +[titan] 2025-07-09 13:33:08,769 - root - INFO - LlamaTokenizerFast(name_or_path='fla-hub/transformer-1.3B-100B', vocab_size=32000, model_max_length=10000000000, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'bos_token': '', 'eos_token': '', 'unk_token': ''}, clean_up_tokenization_spaces=False, added_tokens_decoder={ + 0: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 1: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), + 2: AddedToken("", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True), +} +) +[titan] 2025-07-09 13:33:08,769 - root - INFO - Loading dataset /home/cvm/.cache/HuggingFaceFW___fineweb-edu/sample-100BT:default +[titan] 2025-07-09 13:33:14,313 - root - INFO - Dataset({ + features: ['text', 'id', 'dump', 'url', 'file_path', 'language', 'language_score', 'token_count', 'score', 'int_score'], + num_rows: 97270686 +}) +[titan] 2025-07-09 13:33:14,313 - root - INFO - Shuffling the dataset with seed 79 +[titan] 2025-07-09 13:34:54,507 - root - INFO - Loading model config from configs/mtp_transformer_340M.json +[titan] 2025-07-09 13:34:54,521 - root - INFO - Building dataloader... +[titan] 2025-07-09 13:34:54,533 - root - INFO - Building model from the config +MTPTransformerConfig { + "attention_bias": false, + "bos_token_id": 1, + "elementwise_affine": true, + "eos_token_id": 2, + "fuse_cross_entropy": true, + "fuse_norm": true, + "fuse_swiglu": true, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 1024, + "initializer_range": 0.006, + "intermediate_size": null, + "max_position_embeddings": 8192, + "model_type": "mtp_transformer", + "n_future_tokens": 4, + "norm_eps": 1e-06, + "num_heads": 16, + "num_hidden_layers": 24, + "num_kv_heads": null, + "qk_norm": false, + "qkv_bias": false, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "transformers_version": "4.51.3", + "use_cache": true, + "use_custom_backward": false, + "vocab_size": 32000, + "window_size": null +} + +[titan] 2025-07-09 13:34:54,700 - root - INFO -  +MTPTransformerForCausalLM( + (model): MTPTransformerModel( + (embeddings): Embedding(32000, 1024) + (layers): ModuleList( + (0-19): 20 x MTPTransformerBlock( + (attn_norm): RMSNorm(1024, eps=1e-06) + (attn): Attention( + (q_proj): Linear(in_features=1024, out_features=1024, bias=False) + (k_proj): Linear(in_features=1024, out_features=1024, bias=False) + (v_proj): Linear(in_features=1024, out_features=1024, bias=False) + (o_proj): Linear(in_features=1024, out_features=1024, bias=False) + (rotary): RotaryEmbedding(dim=64, base=10000.0, interleaved=False, pos_idx_in_fp32=True) + ) + (mlp_norm): RMSNorm(1024, eps=1e-06) + (mlp): GatedMLP( + (gate_proj): Linear(in_features=1024, out_features=2816, bias=False) + (up_proj): Linear(in_features=1024, out_features=2816, bias=False) + (down_proj): Linear(in_features=2816, out_features=1024, bias=False) + (swiglu_linear): SwiGLULinear() + ) + ) + ) + (extra_heads): ModuleList( + (0-3): 4 x MTPTransformerBlock( + (attn_norm): RMSNorm(1024, eps=1e-06) + (attn): Attention( + (q_proj): Linear(in_features=1024, out_features=1024, bias=False) + (k_proj): Linear(in_features=1024, out_features=1024, bias=False) + (v_proj): Linear(in_features=1024, out_features=1024, bias=False) + (o_proj): Linear(in_features=1024, out_features=1024, bias=False) + (rotary): RotaryEmbedding(dim=64, base=10000.0, interleaved=False, pos_idx_in_fp32=True) + ) + (mlp_norm): RMSNorm(1024, eps=1e-06) + (mlp): GatedMLP( + (gate_proj): Linear(in_features=1024, out_features=2816, bias=False) + (up_proj): Linear(in_features=1024, out_features=2816, bias=False) + (down_proj): Linear(in_features=2816, out_features=1024, bias=False) + (swiglu_linear): SwiGLULinear() + ) + ) + ) + (norm): RMSNorm(1024, eps=1e-06) + ) + (lm_head): Linear(in_features=1024, out_features=32000, bias=False) + (criterion): FusedLinearCrossEntropyLoss() +) + +[titan] 2025-07-09 13:34:54,747 - root - INFO - Compiling each block with torch.compile +[titan] 2025-07-09 13:34:54,748 - root - INFO - Compiling the embedding, norm, and lm_head layers with torch.compile +[titan] 2025-07-09 13:34:54,749 - root - INFO - Compiling the entire model with torch.compile +[titan] 2025-07-09 13:34:54,844 - root - INFO - Applied FSDP to the model +[titan] 2025-07-09 13:34:55,263 - root - INFO - CUDA memory usage for model: 0.19GiB(0.13%) +[titan] 2025-07-09 13:34:55,279 - root - INFO - Checkpointing active. Checkpoints will be loaded from and saved to exp/mtp.340M.batch16.seqlen4096.context4096.warmup1000.update1.steps100000.lr3e-4.cosine/checkpoint +[titan] 2025-07-09 13:34:55,279 - root - INFO - CUDA capacity: NVIDIA H200 with 139.36GiB memory +[titan] 2025-07-09 13:35:08,956 - root - INFO - ***** Running training ***** +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Training starts at step 1 +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Number of tokens per sequence = 4,096 +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Gradient Accumulation steps = 1 +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Instantaneous batch size (per device) = 16 +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Global batch size (w. parallel, distributed & accumulation) = 128 (524,288 tokens) +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Total optimization steps = 100,000 (52,428,800,000 tokens) +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Warmup steps = 1,000 (524,288,000 tokens) +[titan] 2025-07-09 13:35:08,959 - root - INFO -  Number of parameters = 373,867,520  +[titan] 2025-07-09 13:35:08,959 - root - INFO - Profiling active. Traces will be saved at exp/mtp.340M.batch16.seqlen4096.context4096.warmup1000.update1.steps100000.lr3e-4.cosine/profile_trace +[titan] 2025-07-09 13:35:14,638 - fla.models.transformer_mtp.modeling_transformer - WARNING - `return_all_heads=False` is incompatible with training. Setting `return_all_heads=True`... +[titan] 2025-07-09 13:35:35,255 - root - INFO - step: 1 loss: 41.6257 memory: 44.27GiB(31.77%) tps: 1,640 tflops: 5.66 mfu: 0.57% global_avg_ntp_loss: 10.4071 global_avg_mtp_loss: 31.2185 +[titan] 2025-07-09 13:35:35,255 - root - INFO - lr: 5.9940e-07 gnorm: 18.29 [ 0:00:39<46 days, 5:45:28] +[titan] 2025-07-09 13:35:35,255 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-09 13:35:42,772 - root - INFO - [GC] GC collection invoked by checkpointer. 4.72 seconds. +[titan] 2025-07-09 13:35:42,773 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 7.52 seconds. +[titan] 2025-07-09 13:35:42,773 - root - INFO - Synchronizing and adjusting timeout for all ProcessGroups to 0:15:00 +[titan] 2025-07-09 13:36:46,980 - root - INFO - step: 5 loss: 41.4029 memory: 44.58GiB(31.99%) tps: 3,655 tflops: 12.61 mfu: 1.28% global_avg_ntp_loss: 10.3506 global_avg_mtp_loss: 31.0523 +[titan] 2025-07-09 13:36:46,980 - root - INFO - lr: 1.7982e-06 gnorm: 17.78 [ 0:01:51<25 days, 20:23:20] +[titan] 2025-07-09 13:36:51,020 - root - INFO - step: 10 loss: 40.7648 memory: 44.58GiB(31.99%) tps: 81,101 tflops: 279.89 mfu: 28.30% global_avg_ntp_loss: 10.1900 global_avg_mtp_loss: 30.5749 +[titan] 2025-07-09 13:36:51,021 - root - INFO - lr: 3.2967e-06 gnorm: 14.63 [ 0:01:55<13 days, 9:24:03] +[titan] 2025-07-09 13:36:54,934 - root - INFO - step: 15 loss: 40.0943 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 10.0227 global_avg_mtp_loss: 30.0716 +[titan] 2025-07-09 13:36:54,934 - root - INFO - lr: 4.7952e-06 gnorm: 11.26 [ 0:01:59<9 days, 5:30:08] +[titan] 2025-07-09 13:36:58,910 - root - INFO - step: 20 loss: 39.6481 memory: 44.58GiB(31.99%) tps: 82,416 tflops: 284.43 mfu: 28.76% global_avg_ntp_loss: 9.9114 global_avg_mtp_loss: 29.7366 +[titan] 2025-07-09 13:36:58,911 - root - INFO - lr: 6.2937e-06 gnorm: 10.39 [ 0:02:03<7 days, 3:38:21] +[titan] 2025-07-09 13:37:02,894 - root - INFO - step: 25 loss: 39.3717 memory: 44.58GiB(31.99%) tps: 82,262 tflops: 283.90 mfu: 28.71% global_avg_ntp_loss: 9.8427 global_avg_mtp_loss: 29.5290 +[titan] 2025-07-09 13:37:02,894 - root - INFO - lr: 7.7922e-06 gnorm: 10.35 [ 0:02:07<5 days, 21:43:45] +[titan] 2025-07-09 13:37:06,841 - root - INFO - step: 30 loss: 39.1982 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 9.7999 global_avg_mtp_loss: 29.3984 +[titan] 2025-07-09 13:37:06,841 - root - INFO - lr: 9.2907e-06 gnorm: 10.26 [ 0:02:11<5 days, 1:45:15] +[titan] 2025-07-09 13:37:10,809 - root - INFO - step: 35 loss: 38.9595 memory: 44.58GiB(31.99%) tps: 82,576 tflops: 284.98 mfu: 28.82% global_avg_ntp_loss: 9.7400 global_avg_mtp_loss: 29.2194 +[titan] 2025-07-09 13:37:10,809 - root - INFO - lr: 1.0789e-05 gnorm: 10.27 [ 0:02:15<4 days, 11:30:14] +[titan] 2025-07-09 13:37:14,697 - root - INFO - step: 40 loss: 38.6958 memory: 44.58GiB(31.99%) tps: 84,289 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 9.6738 global_avg_mtp_loss: 29.0220 +[titan] 2025-07-09 13:37:14,697 - root - INFO - lr: 1.2288e-05 gnorm: 10.22 [ 0:02:19<4 days, 0:45:35] +[titan] 2025-07-09 13:37:18,601 - root - INFO - step: 45 loss: 38.3543 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 9.5884 global_avg_mtp_loss: 28.7658 +[titan] 2025-07-09 13:37:18,602 - root - INFO - lr: 1.3786e-05 gnorm: 10.40 [ 0:02:23<3 days, 16:24:47] +[titan] 2025-07-09 13:37:21,748 - root - INFO - [GC] Peforming periodical GC collection. 0.04 seconds. +[titan] 2025-07-09 13:37:22,551 - root - INFO - step: 50 loss: 38.0887 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 9.5221 global_avg_mtp_loss: 28.5667 +[titan] 2025-07-09 13:37:22,551 - root - INFO - lr: 1.5285e-05 gnorm: 10.03 [ 0:02:27<3 days, 9:45:38] +[titan] 2025-07-09 13:37:26,451 - root - INFO - step: 55 loss: 37.6547 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 9.4135 global_avg_mtp_loss: 28.2412 +[titan] 2025-07-09 13:37:26,452 - root - INFO - lr: 1.6783e-05 gnorm: 10.37 [ 0:02:31<3 days, 4:17:33] +[titan] 2025-07-09 13:37:30,384 - root - INFO - step: 60 loss: 37.2911 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 9.3226 global_avg_mtp_loss: 27.9686 +[titan] 2025-07-09 13:37:30,384 - root - INFO - lr: 1.8282e-05 gnorm: 10.12 [ 0:02:35<2 days, 23:45:02] +[titan] 2025-07-09 13:37:34,284 - root - INFO - step: 65 loss: 36.8765 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 9.2189 global_avg_mtp_loss: 27.6577 +[titan] 2025-07-09 13:37:34,284 - root - INFO - lr: 1.9780e-05 gnorm: 10.09 [ 0:02:38<2 days, 19:53:36] +[titan] 2025-07-09 13:37:38,196 - root - INFO - step: 70 loss: 36.4230 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 9.1055 global_avg_mtp_loss: 27.3174 +[titan] 2025-07-09 13:37:38,196 - root - INFO - lr: 2.1279e-05 gnorm: 10.13 [ 0:02:42<2 days, 16:35:30] +[titan] 2025-07-09 13:37:42,072 - root - INFO - step: 75 loss: 35.9475 memory: 44.58GiB(31.99%) tps: 84,552 tflops: 291.80 mfu: 29.50% global_avg_ntp_loss: 8.9866 global_avg_mtp_loss: 26.9610 +[titan] 2025-07-09 13:37:42,072 - root - INFO - lr: 2.2777e-05 gnorm: 10.08 [ 0:02:46<2 days, 13:43:01] +[titan] 2025-07-09 13:37:45,965 - root - INFO - step: 80 loss: 35.5001 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 8.8746 global_avg_mtp_loss: 26.6255 +[titan] 2025-07-09 13:37:45,966 - root - INFO - lr: 2.4276e-05 gnorm: 9.94 [ 0:02:50<2 days, 11:12:27] +[titan] 2025-07-09 13:37:49,852 - root - INFO - step: 85 loss: 34.9535 memory: 44.58GiB(31.99%) tps: 84,313 tflops: 290.98 mfu: 29.42% global_avg_ntp_loss: 8.7379 global_avg_mtp_loss: 26.2156 +[titan] 2025-07-09 13:37:49,852 - root - INFO - lr: 2.5774e-05 gnorm: 9.94 [ 0:02:54<2 days, 8:59:27] +[titan] 2025-07-09 13:37:53,753 - root - INFO - step: 90 loss: 34.5071 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 8.6262 global_avg_mtp_loss: 25.8809 +[titan] 2025-07-09 13:37:53,753 - root - INFO - lr: 2.7273e-05 gnorm: 9.77 [ 0:02:58<2 days, 7:01:29] +[titan] 2025-07-09 13:37:57,673 - root - INFO - step: 95 loss: 34.0127 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 8.5024 global_avg_mtp_loss: 25.5103 +[titan] 2025-07-09 13:37:57,674 - root - INFO - lr: 2.8771e-05 gnorm: 9.80 [ 0:03:02<2 days, 5:16:16] +[titan] 2025-07-09 13:38:00,769 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:38:01,551 - root - INFO - step: 100 loss: 33.5612 memory: 44.58GiB(31.99%) tps: 84,512 tflops: 291.66 mfu: 29.49% global_avg_ntp_loss: 8.3892 global_avg_mtp_loss: 25.1720 +[titan] 2025-07-09 13:38:01,551 - root - INFO - lr: 3.0270e-05 gnorm: 9.50 [ 0:03:06<2 days, 3:40:51] +[titan] 2025-07-09 13:38:05,461 - root - INFO - step: 105 loss: 33.0853 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 8.2699 global_avg_mtp_loss: 24.8154 +[titan] 2025-07-09 13:38:05,461 - root - INFO - lr: 3.1768e-05 gnorm: 9.39 [ 0:03:10<2 days, 2:15:02] +[titan] 2025-07-09 13:38:09,337 - root - INFO - step: 110 loss: 32.6796 memory: 44.58GiB(31.99%) tps: 84,543 tflops: 291.77 mfu: 29.50% global_avg_ntp_loss: 8.1679 global_avg_mtp_loss: 24.5116 +[titan] 2025-07-09 13:38:09,337 - root - INFO - lr: 3.3267e-05 gnorm: 9.08 [ 0:03:14<2 days, 0:56:30] +[titan] 2025-07-09 13:38:13,254 - root - INFO - step: 115 loss: 32.1844 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 8.0427 global_avg_mtp_loss: 24.1416 +[titan] 2025-07-09 13:38:13,254 - root - INFO - lr: 3.4765e-05 gnorm: 8.84 [ 0:03:17<1 day, 23:45:23] +[titan] 2025-07-09 13:38:17,150 - root - INFO - step: 120 loss: 31.7588 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 7.9330 global_avg_mtp_loss: 23.8258 +[titan] 2025-07-09 13:38:17,150 - root - INFO - lr: 3.6264e-05 gnorm: 8.47 [ 0:03:21<1 day, 22:39:54] +[titan] 2025-07-09 13:38:21,036 - root - INFO - step: 125 loss: 31.4794 memory: 44.58GiB(31.99%) tps: 84,339 tflops: 291.07 mfu: 29.43% global_avg_ntp_loss: 7.8575 global_avg_mtp_loss: 23.6219 +[titan] 2025-07-09 13:38:21,036 - root - INFO - lr: 3.7762e-05 gnorm: 8.35 [ 0:03:25<1 day, 21:39:30] +[titan] 2025-07-09 13:38:24,954 - root - INFO - step: 130 loss: 31.0249 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 7.7399 global_avg_mtp_loss: 23.2850 +[titan] 2025-07-09 13:38:24,954 - root - INFO - lr: 3.9261e-05 gnorm: 7.20 [ 0:03:29<1 day, 20:44:10] +[titan] 2025-07-09 13:38:28,869 - root - INFO - step: 135 loss: 30.6492 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 7.6426 global_avg_mtp_loss: 23.0066 +[titan] 2025-07-09 13:38:28,869 - root - INFO - lr: 4.0759e-05 gnorm: 6.64 [ 0:03:33<1 day, 19:52:53] +[titan] 2025-07-09 13:38:32,808 - root - INFO - step: 140 loss: 30.4715 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 7.5908 global_avg_mtp_loss: 22.8807 +[titan] 2025-07-09 13:38:32,808 - root - INFO - lr: 4.2258e-05 gnorm: 5.41 [ 0:03:37<1 day, 19:05:33] +[titan] 2025-07-09 13:38:36,722 - root - INFO - step: 145 loss: 30.0772 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 7.4831 global_avg_mtp_loss: 22.5941 +[titan] 2025-07-09 13:38:36,723 - root - INFO - lr: 4.3756e-05 gnorm: 4.61 [ 0:03:41<1 day, 18:21:11] +[titan] 2025-07-09 13:38:39,830 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:38:40,610 - root - INFO - step: 150 loss: 29.7804 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 7.4018 global_avg_mtp_loss: 22.3786 +[titan] 2025-07-09 13:38:40,610 - root - INFO - lr: 4.5255e-05 gnorm: 3.55 [ 0:03:45<1 day, 17:39:29] +[titan] 2025-07-09 13:38:44,494 - root - INFO - step: 155 loss: 29.6802 memory: 44.58GiB(31.99%) tps: 84,364 tflops: 291.15 mfu: 29.44% global_avg_ntp_loss: 7.3602 global_avg_mtp_loss: 22.3200 +[titan] 2025-07-09 13:38:44,494 - root - INFO - lr: 4.6753e-05 gnorm: 2.51 [ 0:03:49<1 day, 17:00:26] +[titan] 2025-07-09 13:38:48,391 - root - INFO - step: 160 loss: 29.6068 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 7.3231 global_avg_mtp_loss: 22.2838 +[titan] 2025-07-09 13:38:48,391 - root - INFO - lr: 4.8252e-05 gnorm: 3.03 [ 0:03:53<1 day, 16:23:56] +[titan] 2025-07-09 13:38:52,278 - root - INFO - step: 165 loss: 29.4900 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 7.2688 global_avg_mtp_loss: 22.2212 +[titan] 2025-07-09 13:38:52,278 - root - INFO - lr: 4.9750e-05 gnorm: 2.20 [ 0:03:56<1 day, 15:49:34] +[titan] 2025-07-09 13:38:56,171 - root - INFO - step: 170 loss: 29.4372 memory: 44.58GiB(31.99%) tps: 84,187 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 7.2308 global_avg_mtp_loss: 22.2065 +[titan] 2025-07-09 13:38:56,171 - root - INFO - lr: 5.1249e-05 gnorm: 3.79 [ 0:04:00<1 day, 15:17:16] +[titan] 2025-07-09 13:39:00,067 - root - INFO - step: 175 loss: 29.2562 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 7.1600 global_avg_mtp_loss: 22.0962 +[titan] 2025-07-09 13:39:00,068 - root - INFO - lr: 5.2747e-05 gnorm: 2.11 [ 0:04:04<1 day, 14:46:50] +[titan] 2025-07-09 13:39:03,958 - root - INFO - step: 180 loss: 29.1585 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 7.1098 global_avg_mtp_loss: 22.0486 +[titan] 2025-07-09 13:39:03,958 - root - INFO - lr: 5.4246e-05 gnorm: 2.60 [ 0:04:08<1 day, 14:18:03] +[titan] 2025-07-09 13:39:07,865 - root - INFO - step: 185 loss: 29.2457 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 7.1096 global_avg_mtp_loss: 22.1360 +[titan] 2025-07-09 13:39:07,865 - root - INFO - lr: 5.5744e-05 gnorm: 2.14 [ 0:04:12<1 day, 13:50:57] +[titan] 2025-07-09 13:39:11,754 - root - INFO - step: 190 loss: 28.8235 memory: 44.58GiB(31.99%) tps: 84,275 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 6.9718 global_avg_mtp_loss: 21.8517 +[titan] 2025-07-09 13:39:11,754 - root - INFO - lr: 5.7243e-05 gnorm: 2.61 [ 0:04:16<1 day, 13:25:07] +[titan] 2025-07-09 13:39:15,650 - root - INFO - step: 195 loss: 28.8686 memory: 44.58GiB(31.99%) tps: 84,109 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 6.9596 global_avg_mtp_loss: 21.9090 +[titan] 2025-07-09 13:39:15,650 - root - INFO - lr: 5.8741e-05 gnorm: 3.69 [ 0:04:20<1 day, 13:00:41] +[titan] 2025-07-09 13:39:18,770 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:39:19,563 - root - INFO - step: 200 loss: 28.7916 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 6.9194 global_avg_mtp_loss: 21.8722 +[titan] 2025-07-09 13:39:19,563 - root - INFO - lr: 6.0240e-05 gnorm: 2.36 [ 0:04:24<1 day, 12:37:35] +[titan] 2025-07-09 13:39:23,462 - root - INFO - step: 205 loss: 28.6078 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 6.8435 global_avg_mtp_loss: 21.7642 +[titan] 2025-07-09 13:39:23,463 - root - INFO - lr: 6.1738e-05 gnorm: 4.68 [ 0:04:28<1 day, 12:15:31] +[titan] 2025-07-09 13:39:27,380 - root - INFO - step: 210 loss: 28.4140 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 6.7789 global_avg_mtp_loss: 21.6351 +[titan] 2025-07-09 13:39:27,380 - root - INFO - lr: 6.3237e-05 gnorm: 3.62 [ 0:04:32<1 day, 11:54:38] +[titan] 2025-07-09 13:39:31,299 - root - INFO - step: 215 loss: 28.1716 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 6.6929 global_avg_mtp_loss: 21.4787 +[titan] 2025-07-09 13:39:31,299 - root - INFO - lr: 6.4735e-05 gnorm: 1.79 [ 0:04:35<1 day, 11:34:44] +[titan] 2025-07-09 13:39:35,224 - root - INFO - step: 220 loss: 28.4224 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 6.7343 global_avg_mtp_loss: 21.6881 +[titan] 2025-07-09 13:39:35,224 - root - INFO - lr: 6.6234e-05 gnorm: 4.01 [ 0:04:39<1 day, 11:15:47] +[titan] 2025-07-09 13:39:39,122 - root - INFO - step: 225 loss: 28.3019 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 6.6800 global_avg_mtp_loss: 21.6219 +[titan] 2025-07-09 13:39:39,122 - root - INFO - lr: 6.7732e-05 gnorm: 2.84 [ 0:04:43<1 day, 10:57:28] +[titan] 2025-07-09 13:39:43,041 - root - INFO - step: 230 loss: 28.1603 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 6.6223 global_avg_mtp_loss: 21.5380 +[titan] 2025-07-09 13:39:43,041 - root - INFO - lr: 6.9231e-05 gnorm: 1.95 [ 0:04:47<1 day, 10:40:06] +[titan] 2025-07-09 13:39:46,953 - root - INFO - step: 235 loss: 28.1402 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 6.5930 global_avg_mtp_loss: 21.5472 +[titan] 2025-07-09 13:39:46,954 - root - INFO - lr: 7.0729e-05 gnorm: 3.07 [ 0:04:51<1 day, 10:23:25] +[titan] 2025-07-09 13:39:50,919 - root - INFO - step: 240 loss: 28.1595 memory: 44.58GiB(31.99%) tps: 82,643 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 6.5835 global_avg_mtp_loss: 21.5760 +[titan] 2025-07-09 13:39:50,919 - root - INFO - lr: 7.2228e-05 gnorm: 4.69 [ 0:04:55<1 day, 10:07:48] +[titan] 2025-07-09 13:39:54,831 - root - INFO - step: 245 loss: 27.9672 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 6.5252 global_avg_mtp_loss: 21.4420 +[titan] 2025-07-09 13:39:54,831 - root - INFO - lr: 7.3726e-05 gnorm: 3.24 [ 0:04:59<1 day, 9:52:27] +[titan] 2025-07-09 13:39:57,998 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:39:58,784 - root - INFO - step: 250 loss: 27.9513 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 6.5033 global_avg_mtp_loss: 21.4480 +[titan] 2025-07-09 13:39:58,785 - root - INFO - lr: 7.5225e-05 gnorm: 2.69 [ 0:05:03<1 day, 9:37:59] +[titan] 2025-07-09 13:40:02,732 - root - INFO - step: 255 loss: 27.9403 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 6.4724 global_avg_mtp_loss: 21.4679 +[titan] 2025-07-09 13:40:02,733 - root - INFO - lr: 7.6723e-05 gnorm: 3.30 [ 0:05:07<1 day, 9:24:03] +[titan] 2025-07-09 13:40:06,684 - root - INFO - step: 260 loss: 27.8960 memory: 44.58GiB(31.99%) tps: 82,934 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 6.4544 global_avg_mtp_loss: 21.4416 +[titan] 2025-07-09 13:40:06,684 - root - INFO - lr: 7.8222e-05 gnorm: 4.67 [ 0:05:11<1 day, 9:10:40] +[titan] 2025-07-09 13:40:10,602 - root - INFO - step: 265 loss: 27.6850 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 6.3680 global_avg_mtp_loss: 21.3170 +[titan] 2025-07-09 13:40:10,602 - root - INFO - lr: 7.9720e-05 gnorm: 4.05 [ 0:05:15<1 day, 8:57:35] +[titan] 2025-07-09 13:40:14,513 - root - INFO - step: 270 loss: 27.5679 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 6.3291 global_avg_mtp_loss: 21.2388 +[titan] 2025-07-09 13:40:14,514 - root - INFO - lr: 8.1219e-05 gnorm: 4.98 [ 0:05:19<1 day, 8:44:57] +[titan] 2025-07-09 13:40:18,477 - root - INFO - step: 275 loss: 27.5661 memory: 44.58GiB(31.99%) tps: 82,683 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 6.3085 global_avg_mtp_loss: 21.2576 +[titan] 2025-07-09 13:40:18,477 - root - INFO - lr: 8.2717e-05 gnorm: 4.50 [ 0:05:23<1 day, 8:33:05] +[titan] 2025-07-09 13:40:22,411 - root - INFO - step: 280 loss: 27.4175 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 6.2563 global_avg_mtp_loss: 21.1612 +[titan] 2025-07-09 13:40:22,411 - root - INFO - lr: 8.4216e-05 gnorm: 3.35 [ 0:05:27<1 day, 8:21:27] +[titan] 2025-07-09 13:40:26,380 - root - INFO - step: 285 loss: 27.2571 memory: 44.58GiB(31.99%) tps: 82,559 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 6.1994 global_avg_mtp_loss: 21.0577 +[titan] 2025-07-09 13:40:26,380 - root - INFO - lr: 8.5714e-05 gnorm: 6.08 [ 0:05:31<1 day, 8:10:26] +[titan] 2025-07-09 13:40:30,313 - root - INFO - step: 290 loss: 27.0573 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 6.1301 global_avg_mtp_loss: 20.9272 +[titan] 2025-07-09 13:40:30,314 - root - INFO - lr: 8.7213e-05 gnorm: 3.61 [ 0:05:34<1 day, 7:59:36] +[titan] 2025-07-09 13:40:34,227 - root - INFO - step: 295 loss: 27.2709 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 6.1696 global_avg_mtp_loss: 21.1014 +[titan] 2025-07-09 13:40:34,227 - root - INFO - lr: 8.8711e-05 gnorm: 2.99 [ 0:05:38<1 day, 7:49:00] +[titan] 2025-07-09 13:40:37,343 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:40:38,128 - root - INFO - step: 300 loss: 27.2200 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 6.1404 global_avg_mtp_loss: 21.0795 +[titan] 2025-07-09 13:40:38,128 - root - INFO - lr: 9.0210e-05 gnorm: 4.08 [ 0:05:42<1 day, 7:38:42] +[titan] 2025-07-09 13:40:42,051 - root - INFO - step: 305 loss: 27.1079 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 6.1268 global_avg_mtp_loss: 20.9811 +[titan] 2025-07-09 13:40:42,051 - root - INFO - lr: 9.1708e-05 gnorm: 3.82 [ 0:05:46<1 day, 7:28:51] +[titan] 2025-07-09 13:40:46,001 - root - INFO - step: 310 loss: 27.0896 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 6.0891 global_avg_mtp_loss: 21.0005 +[titan] 2025-07-09 13:40:46,001 - root - INFO - lr: 9.3207e-05 gnorm: 6.12 [ 0:05:50<1 day, 7:19:28] +[titan] 2025-07-09 13:40:49,884 - root - INFO - step: 315 loss: 27.0481 memory: 44.58GiB(31.99%) tps: 84,396 tflops: 291.26 mfu: 29.45% global_avg_ntp_loss: 6.0817 global_avg_mtp_loss: 20.9664 +[titan] 2025-07-09 13:40:49,885 - root - INFO - lr: 9.4705e-05 gnorm: 2.74 [ 0:05:54<1 day, 7:10:01] +[titan] 2025-07-09 13:40:53,824 - root - INFO - step: 320 loss: 26.9812 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 6.0405 global_avg_mtp_loss: 20.9407 +[titan] 2025-07-09 13:40:53,825 - root - INFO - lr: 9.6204e-05 gnorm: 3.38 [ 0:05:58<1 day, 7:01:09] +[titan] 2025-07-09 13:40:57,757 - root - INFO - step: 325 loss: 26.8209 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 5.9968 global_avg_mtp_loss: 20.8240 +[titan] 2025-07-09 13:40:57,757 - root - INFO - lr: 9.7702e-05 gnorm: 4.87 [ 0:06:02<1 day, 6:52:32] +[titan] 2025-07-09 13:41:01,696 - root - INFO - step: 330 loss: 26.7854 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 5.9705 global_avg_mtp_loss: 20.8149 +[titan] 2025-07-09 13:41:01,696 - root - INFO - lr: 9.9201e-05 gnorm: 3.75 [ 0:06:06<1 day, 6:44:12] +[titan] 2025-07-09 13:41:05,605 - root - INFO - step: 335 loss: 26.8198 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 5.9721 global_avg_mtp_loss: 20.8476 +[titan] 2025-07-09 13:41:05,605 - root - INFO - lr: 1.0070e-04 gnorm: 6.51 [ 0:06:10<1 day, 6:35:58] +[titan] 2025-07-09 13:41:09,504 - root - INFO - step: 340 loss: 26.7791 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 5.9499 global_avg_mtp_loss: 20.8292 +[titan] 2025-07-09 13:41:09,504 - root - INFO - lr: 1.0220e-04 gnorm: 5.76 [ 0:06:14<1 day, 6:27:55] +[titan] 2025-07-09 13:41:13,437 - root - INFO - step: 345 loss: 26.4270 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 5.8542 global_avg_mtp_loss: 20.5727 +[titan] 2025-07-09 13:41:13,437 - root - INFO - lr: 1.0370e-04 gnorm: 6.58 [ 0:06:18<1 day, 6:20:16] +[titan] 2025-07-09 13:41:16,562 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:41:17,359 - root - INFO - step: 350 loss: 26.5046 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 5.8683 global_avg_mtp_loss: 20.6362 +[titan] 2025-07-09 13:41:17,360 - root - INFO - lr: 1.0519e-04 gnorm: 5.38 [ 0:06:22<1 day, 6:12:47] +[titan] 2025-07-09 13:41:21,279 - root - INFO - step: 355 loss: 26.4512 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 5.8292 global_avg_mtp_loss: 20.6220 +[titan] 2025-07-09 13:41:21,279 - root - INFO - lr: 1.0669e-04 gnorm: 3.85 [ 0:06:25<1 day, 6:05:30] +[titan] 2025-07-09 13:41:25,175 - root - INFO - step: 360 loss: 26.4261 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 5.8113 global_avg_mtp_loss: 20.6148 +[titan] 2025-07-09 13:41:25,175 - root - INFO - lr: 1.0819e-04 gnorm: 5.56 [ 0:06:29<1 day, 5:58:18] +[titan] 2025-07-09 13:41:29,147 - root - INFO - step: 365 loss: 26.4498 memory: 44.58GiB(31.99%) tps: 82,495 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 5.8205 global_avg_mtp_loss: 20.6294 +[titan] 2025-07-09 13:41:29,147 - root - INFO - lr: 1.0969e-04 gnorm: 6.96 [ 0:06:33<1 day, 5:51:39] +[titan] 2025-07-09 13:41:33,091 - root - INFO - step: 370 loss: 26.2527 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 5.7772 global_avg_mtp_loss: 20.4755 +[titan] 2025-07-09 13:41:33,091 - root - INFO - lr: 1.1119e-04 gnorm: 3.51 [ 0:06:37<1 day, 5:45:02] +[titan] 2025-07-09 13:41:37,008 - root - INFO - step: 375 loss: 26.2782 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 5.7610 global_avg_mtp_loss: 20.5171 +[titan] 2025-07-09 13:41:37,008 - root - INFO - lr: 1.1269e-04 gnorm: 6.32 [ 0:06:41<1 day, 5:38:30] +[titan] 2025-07-09 13:41:40,923 - root - INFO - step: 380 loss: 26.3040 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 5.7675 global_avg_mtp_loss: 20.5365 +[titan] 2025-07-09 13:41:40,923 - root - INFO - lr: 1.1419e-04 gnorm: 6.75 [ 0:06:45<1 day, 5:32:06] +[titan] 2025-07-09 13:41:44,825 - root - INFO - step: 385 loss: 26.2119 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 5.7218 global_avg_mtp_loss: 20.4901 +[titan] 2025-07-09 13:41:44,825 - root - INFO - lr: 1.1568e-04 gnorm: 5.88 [ 0:06:49<1 day, 5:25:50] +[titan] 2025-07-09 13:41:48,728 - root - INFO - step: 390 loss: 26.2309 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 5.7220 global_avg_mtp_loss: 20.5090 +[titan] 2025-07-09 13:41:48,728 - root - INFO - lr: 1.1718e-04 gnorm: 7.81 [ 0:06:53<1 day, 5:19:43] +[titan] 2025-07-09 13:41:52,633 - root - INFO - step: 395 loss: 26.0285 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 5.6642 global_avg_mtp_loss: 20.3643 +[titan] 2025-07-09 13:41:52,634 - root - INFO - lr: 1.1868e-04 gnorm: 5.82 [ 0:06:57<1 day, 5:13:46] +[titan] 2025-07-09 13:41:55,753 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:41:56,541 - root - INFO - step: 400 loss: 25.8222 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 5.6104 global_avg_mtp_loss: 20.2118 +[titan] 2025-07-09 13:41:56,542 - root - INFO - lr: 1.2018e-04 gnorm: 4.37 [ 0:07:01<1 day, 5:07:58] +[titan] 2025-07-09 13:42:00,451 - root - INFO - step: 405 loss: 26.0377 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 5.6615 global_avg_mtp_loss: 20.3762 +[titan] 2025-07-09 13:42:00,451 - root - INFO - lr: 1.2168e-04 gnorm: 4.82 [ 0:07:05<1 day, 5:02:20] +[titan] 2025-07-09 13:42:04,341 - root - INFO - step: 410 loss: 26.0250 memory: 44.58GiB(31.99%) tps: 84,242 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 5.6520 global_avg_mtp_loss: 20.3730 +[titan] 2025-07-09 13:42:04,341 - root - INFO - lr: 1.2318e-04 gnorm: 4.97 [ 0:07:08<1 day, 4:56:44] +[titan] 2025-07-09 13:42:08,259 - root - INFO - step: 415 loss: 25.9229 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 5.6125 global_avg_mtp_loss: 20.3104 +[titan] 2025-07-09 13:42:08,259 - root - INFO - lr: 1.2468e-04 gnorm: 6.42 [ 0:07:12<1 day, 4:51:24] +[titan] 2025-07-09 13:42:12,152 - root - INFO - step: 420 loss: 25.7331 memory: 44.58GiB(31.99%) tps: 84,169 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 5.5650 global_avg_mtp_loss: 20.1681 +[titan] 2025-07-09 13:42:12,152 - root - INFO - lr: 1.2617e-04 gnorm: 4.02 [ 0:07:16<1 day, 4:46:05] +[titan] 2025-07-09 13:42:16,060 - root - INFO - step: 425 loss: 25.7041 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 5.5349 global_avg_mtp_loss: 20.1692 +[titan] 2025-07-09 13:42:16,061 - root - INFO - lr: 1.2767e-04 gnorm: 3.98 [ 0:07:20<1 day, 4:40:57] +[titan] 2025-07-09 13:42:19,964 - root - INFO - step: 430 loss: 25.5732 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 5.4951 global_avg_mtp_loss: 20.0781 +[titan] 2025-07-09 13:42:19,964 - root - INFO - lr: 1.2917e-04 gnorm: 3.87 [ 0:07:24<1 day, 4:35:55] +[titan] 2025-07-09 13:42:23,881 - root - INFO - step: 435 loss: 25.6057 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 5.5073 global_avg_mtp_loss: 20.0984 +[titan] 2025-07-09 13:42:23,882 - root - INFO - lr: 1.3067e-04 gnorm: 5.95 [ 0:07:28<1 day, 4:31:03] +[titan] 2025-07-09 13:42:27,807 - root - INFO - step: 440 loss: 25.5757 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 5.4836 global_avg_mtp_loss: 20.0921 +[titan] 2025-07-09 13:42:27,808 - root - INFO - lr: 1.3217e-04 gnorm: 3.72 [ 0:07:32<1 day, 4:26:19] +[titan] 2025-07-09 13:42:31,739 - root - INFO - step: 445 loss: 25.4654 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 5.4620 global_avg_mtp_loss: 20.0034 +[titan] 2025-07-09 13:42:31,739 - root - INFO - lr: 1.3367e-04 gnorm: 4.97 [ 0:07:36<1 day, 4:21:43] +[titan] 2025-07-09 13:42:34,903 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:42:35,689 - root - INFO - step: 450 loss: 25.7442 memory: 44.58GiB(31.99%) tps: 82,952 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 5.5127 global_avg_mtp_loss: 20.2315 +[titan] 2025-07-09 13:42:35,690 - root - INFO - lr: 1.3516e-04 gnorm: 5.86 [ 0:07:40<1 day, 4:17:18] +[titan] 2025-07-09 13:42:39,607 - root - INFO - step: 455 loss: 25.3915 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 5.4195 global_avg_mtp_loss: 19.9720 +[titan] 2025-07-09 13:42:39,608 - root - INFO - lr: 1.3666e-04 gnorm: 4.57 [ 0:07:44<1 day, 4:12:51] +[titan] 2025-07-09 13:42:43,525 - root - INFO - step: 460 loss: 25.5109 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 5.4468 global_avg_mtp_loss: 20.0640 +[titan] 2025-07-09 13:42:43,525 - root - INFO - lr: 1.3816e-04 gnorm: 7.43 [ 0:07:48<1 day, 4:08:29] +[titan] 2025-07-09 13:42:47,442 - root - INFO - step: 465 loss: 25.3400 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 5.4047 global_avg_mtp_loss: 19.9353 +[titan] 2025-07-09 13:42:47,442 - root - INFO - lr: 1.3966e-04 gnorm: 3.65 [ 0:07:52<1 day, 4:04:13] +[titan] 2025-07-09 13:42:51,339 - root - INFO - step: 470 loss: 25.2725 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 5.3718 global_avg_mtp_loss: 19.9007 +[titan] 2025-07-09 13:42:51,339 - root - INFO - lr: 1.4116e-04 gnorm: 4.93 [ 0:07:55<1 day, 3:59:58] +[titan] 2025-07-09 13:42:55,236 - root - INFO - step: 475 loss: 25.2677 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 5.3865 global_avg_mtp_loss: 19.8812 +[titan] 2025-07-09 13:42:55,237 - root - INFO - lr: 1.4266e-04 gnorm: 3.61 [ 0:07:59<1 day, 3:55:49] +[titan] 2025-07-09 13:42:59,143 - root - INFO - step: 480 loss: 25.1716 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 5.3338 global_avg_mtp_loss: 19.8378 +[titan] 2025-07-09 13:42:59,143 - root - INFO - lr: 1.4416e-04 gnorm: 4.78 [ 0:08:03<1 day, 3:51:46] +[titan] 2025-07-09 13:43:03,036 - root - INFO - step: 485 loss: 25.2356 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 5.3361 global_avg_mtp_loss: 19.8995 +[titan] 2025-07-09 13:43:03,037 - root - INFO - lr: 1.4565e-04 gnorm: 9.34 [ 0:08:07<1 day, 3:47:46] +[titan] 2025-07-09 13:43:06,959 - root - INFO - step: 490 loss: 25.1966 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 5.3325 global_avg_mtp_loss: 19.8641 +[titan] 2025-07-09 13:43:06,960 - root - INFO - lr: 1.4715e-04 gnorm: 5.36 [ 0:08:11<1 day, 3:43:56] +[titan] 2025-07-09 13:43:10,872 - root - INFO - step: 495 loss: 25.0073 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 5.2812 global_avg_mtp_loss: 19.7261 +[titan] 2025-07-09 13:43:10,872 - root - INFO - lr: 1.4865e-04 gnorm: 5.88 [ 0:08:15<1 day, 3:40:09] +[titan] 2025-07-09 13:43:14,014 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:43:14,806 - root - INFO - step: 500 loss: 25.0322 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 5.2820 global_avg_mtp_loss: 19.7501 +[titan] 2025-07-09 13:43:14,806 - root - INFO - lr: 1.5015e-04 gnorm: 3.76 [ 0:08:19<1 day, 3:36:31] +[titan] 2025-07-09 13:43:18,699 - root - INFO - step: 505 loss: 24.8798 memory: 44.58GiB(31.99%) tps: 84,181 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 5.2237 global_avg_mtp_loss: 19.6562 +[titan] 2025-07-09 13:43:18,699 - root - INFO - lr: 1.5165e-04 gnorm: 3.00 [ 0:08:23<1 day, 3:32:49] +[titan] 2025-07-09 13:43:23,135 - root - INFO - step: 510 loss: 25.0385 memory: 44.58GiB(31.99%) tps: 73,863 tflops: 254.91 mfu: 25.77% global_avg_ntp_loss: 5.2612 global_avg_mtp_loss: 19.7773 +[titan] 2025-07-09 13:43:23,136 - root - INFO - lr: 1.5315e-04 gnorm: 3.52 [ 0:08:27<1 day, 3:30:57] +[titan] 2025-07-09 13:43:24,995 - root - INFO - Dumping profiler traces at step 512 +[titan] 2025-07-09 13:43:25,026 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 13:43:27,366 - root - INFO - step: 515 loss: 24.8767 memory: 44.58GiB(31.99%) tps: 77,459 tflops: 267.32 mfu: 27.03% global_avg_ntp_loss: 5.2227 global_avg_mtp_loss: 19.6540 +[titan] 2025-07-09 13:43:27,366 - root - INFO - lr: 1.5465e-04 gnorm: 4.58 [ 0:08:32<1 day, 3:28:28] +[titan] 2025-07-09 13:43:31,264 - root - INFO - step: 520 loss: 24.7976 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 5.2065 global_avg_mtp_loss: 19.5910 +[titan] 2025-07-09 13:43:31,264 - root - INFO - lr: 1.5614e-04 gnorm: 4.39 [ 0:08:35<1 day, 3:24:57] +[titan] 2025-07-09 13:43:35,144 - root - INFO - step: 525 loss: 24.9164 memory: 44.58GiB(31.99%) tps: 84,459 tflops: 291.48 mfu: 29.47% global_avg_ntp_loss: 5.2234 global_avg_mtp_loss: 19.6930 +[titan] 2025-07-09 13:43:35,145 - root - INFO - lr: 1.5764e-04 gnorm: 4.51 [ 0:08:39<1 day, 3:21:28] +[titan] 2025-07-09 13:43:39,039 - root - INFO - step: 530 loss: 24.8236 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 5.1912 global_avg_mtp_loss: 19.6324 +[titan] 2025-07-09 13:43:39,039 - root - INFO - lr: 1.5914e-04 gnorm: 4.30 [ 0:08:43<1 day, 3:18:04] +[titan] 2025-07-09 13:43:42,936 - root - INFO - step: 535 loss: 24.7984 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 5.1711 global_avg_mtp_loss: 19.6273 +[titan] 2025-07-09 13:43:42,937 - root - INFO - lr: 1.6064e-04 gnorm: 4.43 [ 0:08:47<1 day, 3:14:45] +[titan] 2025-07-09 13:43:46,840 - root - INFO - step: 540 loss: 24.7139 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 5.1518 global_avg_mtp_loss: 19.5622 +[titan] 2025-07-09 13:43:46,840 - root - INFO - lr: 1.6214e-04 gnorm: 4.88 [ 0:08:51<1 day, 3:11:31] +[titan] 2025-07-09 13:43:50,735 - root - INFO - step: 545 loss: 24.7401 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 5.1516 global_avg_mtp_loss: 19.5885 +[titan] 2025-07-09 13:43:50,735 - root - INFO - lr: 1.6364e-04 gnorm: 5.72 [ 0:08:55<1 day, 3:08:19] +[titan] 2025-07-09 13:43:53,842 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:43:54,647 - root - INFO - step: 550 loss: 24.8899 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 5.1809 global_avg_mtp_loss: 19.7090 +[titan] 2025-07-09 13:43:54,647 - root - INFO - lr: 1.6513e-04 gnorm: 4.81 [ 0:08:59<1 day, 3:05:13] +[titan] 2025-07-09 13:43:58,577 - root - INFO - step: 555 loss: 24.7548 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 5.1580 global_avg_mtp_loss: 19.5967 +[titan] 2025-07-09 13:43:58,578 - root - INFO - lr: 1.6663e-04 gnorm: 4.51 [ 0:09:03<1 day, 3:02:14] +[titan] 2025-07-09 13:44:02,481 - root - INFO - step: 560 loss: 24.7238 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 5.1408 global_avg_mtp_loss: 19.5830 +[titan] 2025-07-09 13:44:02,481 - root - INFO - lr: 1.6813e-04 gnorm: 5.63 [ 0:09:07<1 day, 2:59:13] +[titan] 2025-07-09 13:44:06,421 - root - INFO - step: 565 loss: 24.6039 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 5.0864 global_avg_mtp_loss: 19.5175 +[titan] 2025-07-09 13:44:06,421 - root - INFO - lr: 1.6963e-04 gnorm: 4.73 [ 0:09:11<1 day, 2:56:22] +[titan] 2025-07-09 13:44:10,349 - root - INFO - step: 570 loss: 24.3436 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 5.0219 global_avg_mtp_loss: 19.3217 +[titan] 2025-07-09 13:44:10,349 - root - INFO - lr: 1.7113e-04 gnorm: 4.11 [ 0:09:14<1 day, 2:53:31] +[titan] 2025-07-09 13:44:14,286 - root - INFO - step: 575 loss: 24.6384 memory: 44.58GiB(31.99%) tps: 83,229 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 5.0961 global_avg_mtp_loss: 19.5423 +[titan] 2025-07-09 13:44:14,287 - root - INFO - lr: 1.7263e-04 gnorm: 4.40 [ 0:09:18<1 day, 2:50:46] +[titan] 2025-07-09 13:44:18,217 - root - INFO - step: 580 loss: 24.3620 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 5.0226 global_avg_mtp_loss: 19.3395 +[titan] 2025-07-09 13:44:18,217 - root - INFO - lr: 1.7413e-04 gnorm: 2.74 [ 0:09:22<1 day, 2:48:01] +[titan] 2025-07-09 13:44:22,117 - root - INFO - step: 585 loss: 24.4019 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 5.0247 global_avg_mtp_loss: 19.3772 +[titan] 2025-07-09 13:44:22,118 - root - INFO - lr: 1.7562e-04 gnorm: 3.26 [ 0:09:26<1 day, 2:45:15] +[titan] 2025-07-09 13:44:26,014 - root - INFO - step: 590 loss: 24.5148 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 5.0402 global_avg_mtp_loss: 19.4746 +[titan] 2025-07-09 13:44:26,014 - root - INFO - lr: 1.7712e-04 gnorm: 4.23 [ 0:09:30<1 day, 2:42:30] +[titan] 2025-07-09 13:44:29,932 - root - INFO - step: 595 loss: 24.4339 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 5.0260 global_avg_mtp_loss: 19.4079 +[titan] 2025-07-09 13:44:29,932 - root - INFO - lr: 1.7862e-04 gnorm: 3.75 [ 0:09:34<1 day, 2:39:52] +[titan] 2025-07-09 13:44:33,037 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:44:33,828 - root - INFO - step: 600 loss: 24.5771 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 5.0576 global_avg_mtp_loss: 19.5195 +[titan] 2025-07-09 13:44:33,828 - root - INFO - lr: 1.8012e-04 gnorm: 4.62 [ 0:09:38<1 day, 2:37:12] +[titan] 2025-07-09 13:44:37,742 - root - INFO - step: 605 loss: 24.1236 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 4.9136 global_avg_mtp_loss: 19.2100 +[titan] 2025-07-09 13:44:37,742 - root - INFO - lr: 1.8162e-04 gnorm: 4.39 [ 0:09:42<1 day, 2:34:38] +[titan] 2025-07-09 13:44:41,663 - root - INFO - step: 610 loss: 24.3657 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 4.9934 global_avg_mtp_loss: 19.3723 +[titan] 2025-07-09 13:44:41,663 - root - INFO - lr: 1.8312e-04 gnorm: 4.58 [ 0:09:46<1 day, 2:32:08] +[titan] 2025-07-09 13:44:45,581 - root - INFO - step: 615 loss: 24.3215 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 4.9731 global_avg_mtp_loss: 19.3484 +[titan] 2025-07-09 13:44:45,581 - root - INFO - lr: 1.8462e-04 gnorm: 3.39 [ 0:09:50<1 day, 2:29:40] +[titan] 2025-07-09 13:44:49,491 - root - INFO - step: 620 loss: 24.1270 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 4.9373 global_avg_mtp_loss: 19.1897 +[titan] 2025-07-09 13:44:49,491 - root - INFO - lr: 1.8611e-04 gnorm: 3.73 [ 0:09:54<1 day, 2:27:13] +[titan] 2025-07-09 13:44:53,398 - root - INFO - step: 625 loss: 24.1604 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 4.9085 global_avg_mtp_loss: 19.2518 +[titan] 2025-07-09 13:44:53,398 - root - INFO - lr: 1.8761e-04 gnorm: 3.39 [ 0:09:58<1 day, 2:24:47] +[titan] 2025-07-09 13:44:57,322 - root - INFO - step: 630 loss: 24.1965 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 4.9195 global_avg_mtp_loss: 19.2770 +[titan] 2025-07-09 13:44:57,322 - root - INFO - lr: 1.8911e-04 gnorm: 3.17 [ 0:10:01<1 day, 2:22:27] +[titan] 2025-07-09 13:45:01,219 - root - INFO - step: 635 loss: 24.1608 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 4.9245 global_avg_mtp_loss: 19.2363 +[titan] 2025-07-09 13:45:01,219 - root - INFO - lr: 1.9061e-04 gnorm: 3.16 [ 0:10:05<1 day, 2:20:04] +[titan] 2025-07-09 13:45:05,138 - root - INFO - step: 640 loss: 23.6857 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 4.8038 global_avg_mtp_loss: 18.8819 +[titan] 2025-07-09 13:45:05,138 - root - INFO - lr: 1.9211e-04 gnorm: 4.46 [ 0:10:09<1 day, 2:17:47] +[titan] 2025-07-09 13:45:09,034 - root - INFO - step: 645 loss: 24.1565 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 4.9191 global_avg_mtp_loss: 19.2373 +[titan] 2025-07-09 13:45:09,034 - root - INFO - lr: 1.9361e-04 gnorm: 3.67 [ 0:10:13<1 day, 2:15:28] +[titan] 2025-07-09 13:45:12,168 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:45:12,958 - root - INFO - step: 650 loss: 23.9986 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 4.8703 global_avg_mtp_loss: 19.1283 +[titan] 2025-07-09 13:45:12,959 - root - INFO - lr: 1.9510e-04 gnorm: 3.94 [ 0:10:17<1 day, 2:13:16] +[titan] 2025-07-09 13:45:16,864 - root - INFO - step: 655 loss: 24.0494 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 4.8659 global_avg_mtp_loss: 19.1835 +[titan] 2025-07-09 13:45:16,864 - root - INFO - lr: 1.9660e-04 gnorm: 3.32 [ 0:10:21<1 day, 2:11:03] +[titan] 2025-07-09 13:45:20,757 - root - INFO - step: 660 loss: 24.0738 memory: 44.58GiB(31.99%) tps: 84,176 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 4.8722 global_avg_mtp_loss: 19.2016 +[titan] 2025-07-09 13:45:20,757 - root - INFO - lr: 1.9810e-04 gnorm: 3.01 [ 0:10:25<1 day, 2:08:50] +[titan] 2025-07-09 13:45:24,652 - root - INFO - step: 665 loss: 23.9771 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 4.8423 global_avg_mtp_loss: 19.1348 +[titan] 2025-07-09 13:45:24,652 - root - INFO - lr: 1.9960e-04 gnorm: 4.21 [ 0:10:29<1 day, 2:06:40] +[titan] 2025-07-09 13:45:28,566 - root - INFO - step: 670 loss: 23.7431 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 4.7882 global_avg_mtp_loss: 18.9549 +[titan] 2025-07-09 13:45:28,566 - root - INFO - lr: 2.0110e-04 gnorm: 3.41 [ 0:10:33<1 day, 2:04:34] +[titan] 2025-07-09 13:45:32,457 - root - INFO - step: 675 loss: 23.7926 memory: 44.58GiB(31.99%) tps: 84,233 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 4.7716 global_avg_mtp_loss: 19.0210 +[titan] 2025-07-09 13:45:32,457 - root - INFO - lr: 2.0260e-04 gnorm: 3.80 [ 0:10:37<1 day, 2:02:26] +[titan] 2025-07-09 13:45:36,373 - root - INFO - step: 680 loss: 23.9003 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 4.8275 global_avg_mtp_loss: 19.0728 +[titan] 2025-07-09 13:45:36,374 - root - INFO - lr: 2.0410e-04 gnorm: 4.14 [ 0:10:41<1 day, 2:00:24] +[titan] 2025-07-09 13:45:40,277 - root - INFO - step: 685 loss: 23.6650 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 4.7392 global_avg_mtp_loss: 18.9258 +[titan] 2025-07-09 13:45:40,277 - root - INFO - lr: 2.0559e-04 gnorm: 3.71 [ 0:10:44<1 day, 1:58:22] +[titan] 2025-07-09 13:45:44,171 - root - INFO - step: 690 loss: 23.8472 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 4.7659 global_avg_mtp_loss: 19.0813 +[titan] 2025-07-09 13:45:44,171 - root - INFO - lr: 2.0709e-04 gnorm: 2.94 [ 0:10:48<1 day, 1:56:20] +[titan] 2025-07-09 13:45:48,064 - root - INFO - step: 695 loss: 23.6660 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 4.7387 global_avg_mtp_loss: 18.9273 +[titan] 2025-07-09 13:45:48,065 - root - INFO - lr: 2.0859e-04 gnorm: 3.48 [ 0:10:52<1 day, 1:54:20] +[titan] 2025-07-09 13:45:51,184 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:45:51,986 - root - INFO - step: 700 loss: 23.5264 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 4.7102 global_avg_mtp_loss: 18.8163 +[titan] 2025-07-09 13:45:51,986 - root - INFO - lr: 2.1009e-04 gnorm: 3.16 [ 0:10:56<1 day, 1:52:25] +[titan] 2025-07-09 13:45:55,951 - root - INFO - step: 705 loss: 23.5381 memory: 44.58GiB(31.99%) tps: 82,660 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 4.6966 global_avg_mtp_loss: 18.8414 +[titan] 2025-07-09 13:45:55,951 - root - INFO - lr: 2.1159e-04 gnorm: 3.40 [ 0:11:00<1 day, 1:50:38] +[titan] 2025-07-09 13:45:59,873 - root - INFO - step: 710 loss: 23.5407 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 4.6795 global_avg_mtp_loss: 18.8612 +[titan] 2025-07-09 13:45:59,873 - root - INFO - lr: 2.1309e-04 gnorm: 3.66 [ 0:11:04<1 day, 1:48:47] +[titan] 2025-07-09 13:46:03,775 - root - INFO - step: 715 loss: 23.4161 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 4.6644 global_avg_mtp_loss: 18.7517 +[titan] 2025-07-09 13:46:03,775 - root - INFO - lr: 2.1459e-04 gnorm: 3.06 [ 0:11:08<1 day, 1:46:54] +[titan] 2025-07-09 13:46:07,676 - root - INFO - step: 720 loss: 23.6424 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 4.7138 global_avg_mtp_loss: 18.9287 +[titan] 2025-07-09 13:46:07,676 - root - INFO - lr: 2.1608e-04 gnorm: 3.53 [ 0:11:12<1 day, 1:45:03] +[titan] 2025-07-09 13:46:11,593 - root - INFO - step: 725 loss: 23.6231 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 4.6916 global_avg_mtp_loss: 18.9314 +[titan] 2025-07-09 13:46:11,593 - root - INFO - lr: 2.1758e-04 gnorm: 2.77 [ 0:11:16<1 day, 1:43:15] +[titan] 2025-07-09 13:46:15,503 - root - INFO - step: 730 loss: 23.5489 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 4.6602 global_avg_mtp_loss: 18.8887 +[titan] 2025-07-09 13:46:15,503 - root - INFO - lr: 2.1908e-04 gnorm: 3.17 [ 0:11:20<1 day, 1:41:28] +[titan] 2025-07-09 13:46:19,405 - root - INFO - step: 735 loss: 23.6404 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 4.6939 global_avg_mtp_loss: 18.9465 +[titan] 2025-07-09 13:46:19,405 - root - INFO - lr: 2.2058e-04 gnorm: 3.80 [ 0:11:24<1 day, 1:39:41] +[titan] 2025-07-09 13:46:23,300 - root - INFO - step: 740 loss: 23.3380 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 4.6265 global_avg_mtp_loss: 18.7114 +[titan] 2025-07-09 13:46:23,301 - root - INFO - lr: 2.2208e-04 gnorm: 2.98 [ 0:11:27<1 day, 1:37:55] +[titan] 2025-07-09 13:46:27,221 - root - INFO - step: 745 loss: 23.4361 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 4.6313 global_avg_mtp_loss: 18.8048 +[titan] 2025-07-09 13:46:27,221 - root - INFO - lr: 2.2358e-04 gnorm: 2.54 [ 0:11:31<1 day, 1:36:13] +[titan] 2025-07-09 13:46:30,352 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:46:31,143 - root - INFO - step: 750 loss: 23.3949 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 4.6220 global_avg_mtp_loss: 18.7728 +[titan] 2025-07-09 13:46:31,144 - root - INFO - lr: 2.2507e-04 gnorm: 2.71 [ 0:11:35<1 day, 1:34:33] +[titan] 2025-07-09 13:46:35,066 - root - INFO - step: 755 loss: 23.4534 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 4.6212 global_avg_mtp_loss: 18.8322 +[titan] 2025-07-09 13:46:35,066 - root - INFO - lr: 2.2657e-04 gnorm: 3.31 [ 0:11:39<1 day, 1:32:54] +[titan] 2025-07-09 13:46:39,006 - root - INFO - step: 760 loss: 23.2897 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 4.5756 global_avg_mtp_loss: 18.7141 +[titan] 2025-07-09 13:46:39,006 - root - INFO - lr: 2.2807e-04 gnorm: 4.70 [ 0:11:43<1 day, 1:31:19] +[titan] 2025-07-09 13:46:42,926 - root - INFO - step: 765 loss: 23.1819 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 4.5502 global_avg_mtp_loss: 18.6317 +[titan] 2025-07-09 13:46:42,926 - root - INFO - lr: 2.2957e-04 gnorm: 3.63 [ 0:11:47<1 day, 1:29:42] +[titan] 2025-07-09 13:46:46,833 - root - INFO - step: 770 loss: 23.3216 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 4.5786 global_avg_mtp_loss: 18.7430 +[titan] 2025-07-09 13:46:46,833 - root - INFO - lr: 2.3107e-04 gnorm: 2.92 [ 0:11:51<1 day, 1:28:05] +[titan] 2025-07-09 13:46:50,744 - root - INFO - step: 775 loss: 23.3122 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 4.5785 global_avg_mtp_loss: 18.7337 +[titan] 2025-07-09 13:46:50,744 - root - INFO - lr: 2.3257e-04 gnorm: 2.63 [ 0:11:55<1 day, 1:26:30] +[titan] 2025-07-09 13:46:54,641 - root - INFO - step: 780 loss: 23.2893 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 4.5623 global_avg_mtp_loss: 18.7270 +[titan] 2025-07-09 13:46:54,641 - root - INFO - lr: 2.3407e-04 gnorm: 3.90 [ 0:11:59<1 day, 1:24:54] +[titan] 2025-07-09 13:46:58,555 - root - INFO - step: 785 loss: 23.2593 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 4.5583 global_avg_mtp_loss: 18.7009 +[titan] 2025-07-09 13:46:58,555 - root - INFO - lr: 2.3556e-04 gnorm: 3.50 [ 0:12:03<1 day, 1:23:21] +[titan] 2025-07-09 13:47:02,522 - root - INFO - step: 790 loss: 23.2018 memory: 44.58GiB(31.99%) tps: 82,608 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 4.5405 global_avg_mtp_loss: 18.6613 +[titan] 2025-07-09 13:47:02,522 - root - INFO - lr: 2.3706e-04 gnorm: 3.23 [ 0:12:07<1 day, 1:21:56] +[titan] 2025-07-09 13:47:06,418 - root - INFO - step: 795 loss: 23.2533 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 4.5318 global_avg_mtp_loss: 18.7215 +[titan] 2025-07-09 13:47:06,418 - root - INFO - lr: 2.3856e-04 gnorm: 3.14 [ 0:12:11<1 day, 1:20:23] +[titan] 2025-07-09 13:47:09,522 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:47:10,329 - root - INFO - step: 800 loss: 23.0409 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 4.5011 global_avg_mtp_loss: 18.5398 +[titan] 2025-07-09 13:47:10,329 - root - INFO - lr: 2.4006e-04 gnorm: 3.08 [ 0:12:14<1 day, 1:18:53] +[titan] 2025-07-09 13:47:14,285 - root - INFO - step: 805 loss: 23.1213 memory: 44.58GiB(31.99%) tps: 82,834 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 4.4951 global_avg_mtp_loss: 18.6262 +[titan] 2025-07-09 13:47:14,285 - root - INFO - lr: 2.4156e-04 gnorm: 2.60 [ 0:12:18<1 day, 1:17:30] +[titan] 2025-07-09 13:47:18,176 - root - INFO - step: 810 loss: 23.0358 memory: 44.58GiB(31.99%) tps: 84,230 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 4.4873 global_avg_mtp_loss: 18.5485 +[titan] 2025-07-09 13:47:18,176 - root - INFO - lr: 2.4306e-04 gnorm: 2.73 [ 0:12:22<1 day, 1:16:00] +[titan] 2025-07-09 13:47:22,089 - root - INFO - step: 815 loss: 22.9980 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 4.4620 global_avg_mtp_loss: 18.5360 +[titan] 2025-07-09 13:47:22,089 - root - INFO - lr: 2.4456e-04 gnorm: 2.81 [ 0:12:26<1 day, 1:14:33] +[titan] 2025-07-09 13:47:26,004 - root - INFO - step: 820 loss: 22.9001 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 4.4459 global_avg_mtp_loss: 18.4542 +[titan] 2025-07-09 13:47:26,004 - root - INFO - lr: 2.4605e-04 gnorm: 2.92 [ 0:12:30<1 day, 1:13:08] +[titan] 2025-07-09 13:47:29,919 - root - INFO - step: 825 loss: 22.9151 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 4.4477 global_avg_mtp_loss: 18.4675 +[titan] 2025-07-09 13:47:29,919 - root - INFO - lr: 2.4755e-04 gnorm: 2.93 [ 0:12:34<1 day, 1:11:44] +[titan] 2025-07-09 13:47:33,822 - root - INFO - step: 830 loss: 22.6662 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 4.3835 global_avg_mtp_loss: 18.2827 +[titan] 2025-07-09 13:47:33,822 - root - INFO - lr: 2.4905e-04 gnorm: 2.89 [ 0:12:38<1 day, 1:10:19] +[titan] 2025-07-09 13:47:37,725 - root - INFO - step: 835 loss: 23.0099 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 4.4684 global_avg_mtp_loss: 18.5415 +[titan] 2025-07-09 13:47:37,725 - root - INFO - lr: 2.5055e-04 gnorm: 2.25 [ 0:12:42<1 day, 1:08:56] +[titan] 2025-07-09 13:47:41,653 - root - INFO - step: 840 loss: 22.9140 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 4.4135 global_avg_mtp_loss: 18.5005 +[titan] 2025-07-09 13:47:41,653 - root - INFO - lr: 2.5205e-04 gnorm: 2.83 [ 0:12:46<1 day, 1:07:36] +[titan] 2025-07-09 13:47:45,571 - root - INFO - step: 845 loss: 22.8056 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 4.3890 global_avg_mtp_loss: 18.4167 +[titan] 2025-07-09 13:47:45,571 - root - INFO - lr: 2.5355e-04 gnorm: 3.28 [ 0:12:50<1 day, 1:06:16] +[titan] 2025-07-09 13:47:48,665 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:47:49,522 - root - INFO - step: 850 loss: 22.7173 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 4.3622 global_avg_mtp_loss: 18.3551 +[titan] 2025-07-09 13:47:49,522 - root - INFO - lr: 2.5504e-04 gnorm: 2.73 [ 0:12:54<1 day, 1:05:00] +[titan] 2025-07-09 13:47:53,429 - root - INFO - step: 855 loss: 22.7150 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 4.3582 global_avg_mtp_loss: 18.3568 +[titan] 2025-07-09 13:47:53,429 - root - INFO - lr: 2.5654e-04 gnorm: 2.49 [ 0:12:58<1 day, 1:03:41] +[titan] 2025-07-09 13:47:57,360 - root - INFO - step: 860 loss: 22.8633 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 4.3921 global_avg_mtp_loss: 18.4711 +[titan] 2025-07-09 13:47:57,360 - root - INFO - lr: 2.5804e-04 gnorm: 3.42 [ 0:13:01<1 day, 1:02:25] +[titan] 2025-07-09 13:48:01,252 - root - INFO - step: 865 loss: 22.6532 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 4.3433 global_avg_mtp_loss: 18.3099 +[titan] 2025-07-09 13:48:01,252 - root - INFO - lr: 2.5954e-04 gnorm: 2.42 [ 0:13:05<1 day, 1:01:05] +[titan] 2025-07-09 13:48:05,133 - root - INFO - step: 870 loss: 22.7563 memory: 44.58GiB(31.99%) tps: 84,443 tflops: 291.43 mfu: 29.47% global_avg_ntp_loss: 4.3794 global_avg_mtp_loss: 18.3770 +[titan] 2025-07-09 13:48:05,133 - root - INFO - lr: 2.6104e-04 gnorm: 3.08 [ 0:13:09<1 day, 0:59:45] +[titan] 2025-07-09 13:48:09,037 - root - INFO - step: 875 loss: 22.8732 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 4.3851 global_avg_mtp_loss: 18.4881 +[titan] 2025-07-09 13:48:09,037 - root - INFO - lr: 2.6254e-04 gnorm: 2.88 [ 0:13:13<1 day, 0:58:29] +[titan] 2025-07-09 13:48:13,005 - root - INFO - step: 880 loss: 22.5287 memory: 44.58GiB(31.99%) tps: 82,590 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 4.3088 global_avg_mtp_loss: 18.2199 +[titan] 2025-07-09 13:48:13,005 - root - INFO - lr: 2.6404e-04 gnorm: 2.45 [ 0:13:17<1 day, 0:57:20] +[titan] 2025-07-09 13:48:16,892 - root - INFO - step: 885 loss: 22.7184 memory: 44.58GiB(31.99%) tps: 84,325 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 4.3461 global_avg_mtp_loss: 18.3723 +[titan] 2025-07-09 13:48:16,892 - root - INFO - lr: 2.6553e-04 gnorm: 2.99 [ 0:13:21<1 day, 0:56:03] +[titan] 2025-07-09 13:48:20,806 - root - INFO - step: 890 loss: 22.8129 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 4.3829 global_avg_mtp_loss: 18.4300 +[titan] 2025-07-09 13:48:20,807 - root - INFO - lr: 2.6703e-04 gnorm: 7.44 [ 0:13:25<1 day, 0:54:51] +[titan] 2025-07-09 13:48:24,740 - root - INFO - step: 895 loss: 22.8623 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 4.3894 global_avg_mtp_loss: 18.4729 +[titan] 2025-07-09 13:48:24,741 - root - INFO - lr: 2.6853e-04 gnorm: 2.67 [ 0:13:29<1 day, 0:53:41] +[titan] 2025-07-09 13:48:27,852 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:48:28,644 - root - INFO - step: 900 loss: 22.7996 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 4.3559 global_avg_mtp_loss: 18.4437 +[titan] 2025-07-09 13:48:28,644 - root - INFO - lr: 2.7003e-04 gnorm: 2.59 [ 0:13:33<1 day, 0:52:28] +[titan] 2025-07-09 13:48:32,589 - root - INFO - step: 905 loss: 22.6282 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 4.2883 global_avg_mtp_loss: 18.3400 +[titan] 2025-07-09 13:48:32,589 - root - INFO - lr: 2.7153e-04 gnorm: 2.63 [ 0:13:37<1 day, 0:51:21] +[titan] 2025-07-09 13:48:36,508 - root - INFO - step: 910 loss: 22.3200 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 4.2457 global_avg_mtp_loss: 18.0743 +[titan] 2025-07-09 13:48:36,508 - root - INFO - lr: 2.7303e-04 gnorm: 2.80 [ 0:13:41<1 day, 0:50:11] +[titan] 2025-07-09 13:48:40,424 - root - INFO - step: 915 loss: 22.5480 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 4.2798 global_avg_mtp_loss: 18.2682 +[titan] 2025-07-09 13:48:40,424 - root - INFO - lr: 2.7453e-04 gnorm: 2.32 [ 0:13:45<1 day, 0:49:02] +[titan] 2025-07-09 13:48:44,349 - root - INFO - step: 920 loss: 22.4656 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 4.2435 global_avg_mtp_loss: 18.2221 +[titan] 2025-07-09 13:48:44,349 - root - INFO - lr: 2.7602e-04 gnorm: 2.43 [ 0:13:48<1 day, 0:47:55] +[titan] 2025-07-09 13:48:48,249 - root - INFO - step: 925 loss: 22.4517 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 4.2816 global_avg_mtp_loss: 18.1701 +[titan] 2025-07-09 13:48:48,249 - root - INFO - lr: 2.7752e-04 gnorm: 2.65 [ 0:13:52<1 day, 0:46:45] +[titan] 2025-07-09 13:48:52,156 - root - INFO - step: 930 loss: 22.4799 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 4.2364 global_avg_mtp_loss: 18.2435 +[titan] 2025-07-09 13:48:52,156 - root - INFO - lr: 2.7902e-04 gnorm: 2.43 [ 0:13:56<1 day, 0:45:37] +[titan] 2025-07-09 13:48:56,065 - root - INFO - step: 935 loss: 22.3826 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 4.2209 global_avg_mtp_loss: 18.1617 +[titan] 2025-07-09 13:48:56,065 - root - INFO - lr: 2.8052e-04 gnorm: 2.23 [ 0:14:00<1 day, 0:44:30] +[titan] 2025-07-09 13:48:59,957 - root - INFO - step: 940 loss: 22.4852 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 4.2624 global_avg_mtp_loss: 18.2229 +[titan] 2025-07-09 13:48:59,958 - root - INFO - lr: 2.8202e-04 gnorm: 2.20 [ 0:14:04<1 day, 0:43:22] +[titan] 2025-07-09 13:49:03,846 - root - INFO - step: 945 loss: 22.5863 memory: 44.58GiB(31.99%) tps: 84,269 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 4.2794 global_avg_mtp_loss: 18.3069 +[titan] 2025-07-09 13:49:03,847 - root - INFO - lr: 2.8352e-04 gnorm: 2.45 [ 0:14:08<1 day, 0:42:14] +[titan] 2025-07-09 13:49:06,965 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:49:07,750 - root - INFO - step: 950 loss: 22.3379 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 4.2130 global_avg_mtp_loss: 18.1250 +[titan] 2025-07-09 13:49:07,750 - root - INFO - lr: 2.8501e-04 gnorm: 2.34 [ 0:14:12<1 day, 0:41:09] +[titan] 2025-07-09 13:49:11,650 - root - INFO - step: 955 loss: 22.4331 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 4.2375 global_avg_mtp_loss: 18.1957 +[titan] 2025-07-09 13:49:11,650 - root - INFO - lr: 2.8651e-04 gnorm: 2.03 [ 0:14:16<1 day, 0:40:04] +[titan] 2025-07-09 13:49:15,567 - root - INFO - step: 960 loss: 22.2770 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 4.1874 global_avg_mtp_loss: 18.0896 +[titan] 2025-07-09 13:49:15,567 - root - INFO - lr: 2.8801e-04 gnorm: 2.20 [ 0:14:20<1 day, 0:39:01] +[titan] 2025-07-09 13:49:19,472 - root - INFO - step: 965 loss: 22.4498 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 4.2375 global_avg_mtp_loss: 18.2123 +[titan] 2025-07-09 13:49:19,472 - root - INFO - lr: 2.8951e-04 gnorm: 2.39 [ 0:14:24<1 day, 0:37:57] +[titan] 2025-07-09 13:49:23,390 - root - INFO - step: 970 loss: 22.1882 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 4.1669 global_avg_mtp_loss: 18.0213 +[titan] 2025-07-09 13:49:23,390 - root - INFO - lr: 2.9101e-04 gnorm: 2.47 [ 0:14:27<1 day, 0:36:55] +[titan] 2025-07-09 13:49:27,279 - root - INFO - step: 975 loss: 22.3187 memory: 44.58GiB(31.99%) tps: 84,281 tflops: 290.87 mfu: 29.41% global_avg_ntp_loss: 4.1842 global_avg_mtp_loss: 18.1345 +[titan] 2025-07-09 13:49:27,279 - root - INFO - lr: 2.9251e-04 gnorm: 2.70 [ 0:14:31<1 day, 0:35:51] +[titan] 2025-07-09 13:49:31,194 - root - INFO - step: 980 loss: 22.2601 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 4.1679 global_avg_mtp_loss: 18.0922 +[titan] 2025-07-09 13:49:31,194 - root - INFO - lr: 2.9401e-04 gnorm: 2.28 [ 0:14:35<1 day, 0:34:51] +[titan] 2025-07-09 13:49:35,114 - root - INFO - step: 985 loss: 22.1465 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 4.1409 global_avg_mtp_loss: 18.0056 +[titan] 2025-07-09 13:49:35,114 - root - INFO - lr: 2.9550e-04 gnorm: 2.87 [ 0:14:39<1 day, 0:33:51] +[titan] 2025-07-09 13:49:39,026 - root - INFO - step: 990 loss: 22.0955 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 4.1403 global_avg_mtp_loss: 17.9552 +[titan] 2025-07-09 13:49:39,026 - root - INFO - lr: 2.9700e-04 gnorm: 2.65 [ 0:14:43<1 day, 0:32:51] +[titan] 2025-07-09 13:49:42,933 - root - INFO - step: 995 loss: 22.1558 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 4.1218 global_avg_mtp_loss: 18.0340 +[titan] 2025-07-09 13:49:42,933 - root - INFO - lr: 2.9850e-04 gnorm: 2.18 [ 0:14:47<1 day, 0:31:51] +[titan] 2025-07-09 13:49:46,067 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:49:46,852 - root - INFO - step: 1000 loss: 22.0445 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 4.1055 global_avg_mtp_loss: 17.9391 +[titan] 2025-07-09 13:49:46,852 - root - INFO - lr: 3.0000e-04 gnorm: 1.68 [ 0:14:51<1 day, 0:30:53] +[titan] 2025-07-09 13:49:50,745 - root - INFO - step: 1005 loss: 22.1762 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 4.1349 global_avg_mtp_loss: 18.0412 +[titan] 2025-07-09 13:49:50,745 - root - INFO - lr: 3.0000e-04 gnorm: 1.96 [ 0:14:55<1 day, 0:29:53] +[titan] 2025-07-09 13:49:54,654 - root - INFO - step: 1010 loss: 22.1284 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 4.1246 global_avg_mtp_loss: 18.0038 +[titan] 2025-07-09 13:49:54,654 - root - INFO - lr: 3.0000e-04 gnorm: 2.62 [ 0:14:59<1 day, 0:28:55] +[titan] 2025-07-09 13:49:58,582 - root - INFO - step: 1015 loss: 22.0153 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 4.0940 global_avg_mtp_loss: 17.9212 +[titan] 2025-07-09 13:49:58,583 - root - INFO - lr: 3.0000e-04 gnorm: 1.97 [ 0:15:03<1 day, 0:28:00] +[titan] 2025-07-09 13:50:02,515 - root - INFO - step: 1020 loss: 21.9242 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 4.0732 global_avg_mtp_loss: 17.8510 +[titan] 2025-07-09 13:50:02,516 - root - INFO - lr: 3.0000e-04 gnorm: 2.24 [ 0:15:07<1 day, 0:27:05] +[titan] 2025-07-09 13:50:05,842 - root - INFO - Dumping profiler traces at step 1024 +[titan] 2025-07-09 13:50:05,873 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 13:50:06,661 - root - INFO - step: 1025 loss: 22.0676 memory: 44.58GiB(31.99%) tps: 79,051 tflops: 272.82 mfu: 27.59% global_avg_ntp_loss: 4.0818 global_avg_mtp_loss: 17.9858 +[titan] 2025-07-09 13:50:06,661 - root - INFO - lr: 3.0000e-04 gnorm: 1.68 [ 0:15:11<1 day, 0:26:32] +[titan] 2025-07-09 13:50:10,583 - root - INFO - step: 1030 loss: 22.0345 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 4.0870 global_avg_mtp_loss: 17.9475 +[titan] 2025-07-09 13:50:10,583 - root - INFO - lr: 3.0000e-04 gnorm: 2.31 [ 0:15:15<1 day, 0:25:37] +[titan] 2025-07-09 13:50:14,479 - root - INFO - step: 1035 loss: 21.9197 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 4.0679 global_avg_mtp_loss: 17.8518 +[titan] 2025-07-09 13:50:14,479 - root - INFO - lr: 3.0000e-04 gnorm: 2.26 [ 0:15:19<1 day, 0:24:40] +[titan] 2025-07-09 13:50:18,396 - root - INFO - step: 1040 loss: 21.5995 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 3.9990 global_avg_mtp_loss: 17.6005 +[titan] 2025-07-09 13:50:18,397 - root - INFO - lr: 3.0000e-04 gnorm: 2.02 [ 0:15:22<1 day, 0:23:46] +[titan] 2025-07-09 13:50:22,301 - root - INFO - step: 1045 loss: 21.8319 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 4.0240 global_avg_mtp_loss: 17.8079 +[titan] 2025-07-09 13:50:22,302 - root - INFO - lr: 3.0000e-04 gnorm: 2.22 [ 0:15:26<1 day, 0:22:51] +[titan] 2025-07-09 13:50:25,399 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:50:26,193 - root - INFO - step: 1050 loss: 21.8010 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 4.0283 global_avg_mtp_loss: 17.7727 +[titan] 2025-07-09 13:50:26,194 - root - INFO - lr: 3.0000e-04 gnorm: 1.85 [ 0:15:30<1 day, 0:21:55] +[titan] 2025-07-09 13:50:30,083 - root - INFO - step: 1055 loss: 21.8416 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 4.0330 global_avg_mtp_loss: 17.8086 +[titan] 2025-07-09 13:50:30,084 - root - INFO - lr: 3.0000e-04 gnorm: 1.96 [ 0:15:34<1 day, 0:21:00] +[titan] 2025-07-09 13:50:33,994 - root - INFO - step: 1060 loss: 21.6889 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 3.9862 global_avg_mtp_loss: 17.7027 +[titan] 2025-07-09 13:50:33,994 - root - INFO - lr: 3.0000e-04 gnorm: 2.09 [ 0:15:38<1 day, 0:20:07] +[titan] 2025-07-09 13:50:37,915 - root - INFO - step: 1065 loss: 21.8328 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 4.0243 global_avg_mtp_loss: 17.8085 +[titan] 2025-07-09 13:50:37,916 - root - INFO - lr: 3.0000e-04 gnorm: 1.80 [ 0:15:42<1 day, 0:19:16] +[titan] 2025-07-09 13:50:41,821 - root - INFO - step: 1070 loss: 21.6780 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 3.9850 global_avg_mtp_loss: 17.6930 +[titan] 2025-07-09 13:50:41,821 - root - INFO - lr: 3.0000e-04 gnorm: 2.22 [ 0:15:46<1 day, 0:18:23] +[titan] 2025-07-09 13:50:45,752 - root - INFO - step: 1075 loss: 21.7192 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 3.9808 global_avg_mtp_loss: 17.7383 +[titan] 2025-07-09 13:50:45,752 - root - INFO - lr: 3.0000e-04 gnorm: 1.70 [ 0:15:50<1 day, 0:17:33] +[titan] 2025-07-09 13:50:49,660 - root - INFO - step: 1080 loss: 21.8013 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 3.9789 global_avg_mtp_loss: 17.8224 +[titan] 2025-07-09 13:50:49,660 - root - INFO - lr: 3.0000e-04 gnorm: 1.99 [ 0:15:54<1 day, 0:16:42] +[titan] 2025-07-09 13:50:53,570 - root - INFO - step: 1085 loss: 21.8929 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 4.0262 global_avg_mtp_loss: 17.8667 +[titan] 2025-07-09 13:50:53,570 - root - INFO - lr: 3.0000e-04 gnorm: 1.79 [ 0:15:58<1 day, 0:15:51] +[titan] 2025-07-09 13:50:57,465 - root - INFO - step: 1090 loss: 21.6130 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 3.9382 global_avg_mtp_loss: 17.6748 +[titan] 2025-07-09 13:50:57,466 - root - INFO - lr: 3.0000e-04 gnorm: 1.79 [ 0:16:02<1 day, 0:15:00] +[titan] 2025-07-09 13:51:01,351 - root - INFO - step: 1095 loss: 21.7184 memory: 44.58GiB(31.99%) tps: 84,336 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 3.9611 global_avg_mtp_loss: 17.7573 +[titan] 2025-07-09 13:51:01,351 - root - INFO - lr: 3.0000e-04 gnorm: 1.66 [ 0:16:05<1 day, 0:14:08] +[titan] 2025-07-09 13:51:04,475 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:51:05,270 - root - INFO - step: 1100 loss: 21.6362 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 3.9369 global_avg_mtp_loss: 17.6993 +[titan] 2025-07-09 13:51:05,270 - root - INFO - lr: 3.0000e-04 gnorm: 1.65 [ 0:16:09<1 day, 0:13:19] +[titan] 2025-07-09 13:51:09,154 - root - INFO - step: 1105 loss: 21.4048 memory: 44.58GiB(31.99%) tps: 84,377 tflops: 291.20 mfu: 29.44% global_avg_ntp_loss: 3.9115 global_avg_mtp_loss: 17.4933 +[titan] 2025-07-09 13:51:09,154 - root - INFO - lr: 3.0000e-04 gnorm: 1.86 [ 0:16:13<1 day, 0:12:28] +[titan] 2025-07-09 13:51:13,063 - root - INFO - step: 1110 loss: 21.6929 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 3.9679 global_avg_mtp_loss: 17.7250 +[titan] 2025-07-09 13:51:13,063 - root - INFO - lr: 3.0000e-04 gnorm: 1.82 [ 0:16:17<1 day, 0:11:39] +[titan] 2025-07-09 13:51:16,972 - root - INFO - step: 1115 loss: 21.3457 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 3.8677 global_avg_mtp_loss: 17.4781 +[titan] 2025-07-09 13:51:16,973 - root - INFO - lr: 3.0000e-04 gnorm: 1.69 [ 0:16:21<1 day, 0:10:51] +[titan] 2025-07-09 13:51:20,909 - root - INFO - step: 1120 loss: 21.7261 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 3.9415 global_avg_mtp_loss: 17.7845 +[titan] 2025-07-09 13:51:20,909 - root - INFO - lr: 3.0000e-04 gnorm: 2.16 [ 0:16:25<1 day, 0:10:05] +[titan] 2025-07-09 13:51:24,833 - root - INFO - step: 1125 loss: 21.2836 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 3.8517 global_avg_mtp_loss: 17.4319 +[titan] 2025-07-09 13:51:24,834 - root - INFO - lr: 3.0000e-04 gnorm: 1.70 [ 0:16:29<1 day, 0:09:19] +[titan] 2025-07-09 13:51:28,763 - root - INFO - step: 1130 loss: 21.3907 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 3.9088 global_avg_mtp_loss: 17.4819 +[titan] 2025-07-09 13:51:28,763 - root - INFO - lr: 3.0000e-04 gnorm: 7.50 [ 0:16:33<1 day, 0:08:33] +[titan] 2025-07-09 13:51:32,686 - root - INFO - step: 1135 loss: 21.5662 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 3.9091 global_avg_mtp_loss: 17.6571 +[titan] 2025-07-09 13:51:32,686 - root - INFO - lr: 3.0000e-04 gnorm: 1.56 [ 0:16:37<1 day, 0:07:48] +[titan] 2025-07-09 13:51:36,642 - root - INFO - step: 1140 loss: 21.3432 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 3.8526 global_avg_mtp_loss: 17.4906 +[titan] 2025-07-09 13:51:36,642 - root - INFO - lr: 3.0000e-04 gnorm: 2.05 [ 0:16:41<1 day, 0:07:05] +[titan] 2025-07-09 13:51:40,563 - root - INFO - step: 1145 loss: 21.5527 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 3.8895 global_avg_mtp_loss: 17.6632 +[titan] 2025-07-09 13:51:40,563 - root - INFO - lr: 3.0000e-04 gnorm: 1.70 [ 0:16:45<1 day, 0:06:20] +[titan] 2025-07-09 13:51:43,686 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:51:44,473 - root - INFO - step: 1150 loss: 21.4322 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 3.8693 global_avg_mtp_loss: 17.5628 +[titan] 2025-07-09 13:51:44,473 - root - INFO - lr: 3.0000e-04 gnorm: 1.62 [ 0:16:49<1 day, 0:05:35] +[titan] 2025-07-09 13:51:48,376 - root - INFO - step: 1155 loss: 21.3312 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 3.8349 global_avg_mtp_loss: 17.4963 +[titan] 2025-07-09 13:51:48,377 - root - INFO - lr: 3.0000e-04 gnorm: 1.63 [ 0:16:52<1 day, 0:04:49] +[titan] 2025-07-09 13:51:52,296 - root - INFO - step: 1160 loss: 21.3501 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 3.8316 global_avg_mtp_loss: 17.5184 +[titan] 2025-07-09 13:51:52,296 - root - INFO - lr: 3.0000e-04 gnorm: 1.61 [ 0:16:56<1 day, 0:04:05] +[titan] 2025-07-09 13:51:56,198 - root - INFO - step: 1165 loss: 21.2182 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 3.7931 global_avg_mtp_loss: 17.4251 +[titan] 2025-07-09 13:51:56,198 - root - INFO - lr: 3.0000e-04 gnorm: 1.51 [ 0:17:00<1 day, 0:03:20] +[titan] 2025-07-09 13:52:00,096 - root - INFO - step: 1170 loss: 21.3028 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 3.8264 global_avg_mtp_loss: 17.4764 +[titan] 2025-07-09 13:52:00,096 - root - INFO - lr: 3.0000e-04 gnorm: 1.70 [ 0:17:04<1 day, 0:02:34] +[titan] 2025-07-09 13:52:04,000 - root - INFO - step: 1175 loss: 21.3747 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 3.8333 global_avg_mtp_loss: 17.5414 +[titan] 2025-07-09 13:52:04,000 - root - INFO - lr: 3.0000e-04 gnorm: 1.52 [ 0:17:08<1 day, 0:01:50] +[titan] 2025-07-09 13:52:07,908 - root - INFO - step: 1180 loss: 21.2570 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 3.7952 global_avg_mtp_loss: 17.4618 +[titan] 2025-07-09 13:52:07,908 - root - INFO - lr: 3.0000e-04 gnorm: 1.74 [ 0:17:12<1 day, 0:01:06] +[titan] 2025-07-09 13:52:11,822 - root - INFO - step: 1185 loss: 20.9544 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 3.7260 global_avg_mtp_loss: 17.2283 +[titan] 2025-07-09 13:52:11,822 - root - INFO - lr: 3.0000e-04 gnorm: 1.55 [ 0:17:16<1 day, 0:00:23] +[titan] 2025-07-09 13:52:15,727 - root - INFO - step: 1190 loss: 21.0634 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 3.7411 global_avg_mtp_loss: 17.3223 +[titan] 2025-07-09 13:52:15,727 - root - INFO - lr: 3.0000e-04 gnorm: 1.69 [ 0:17:20<23:59:40] +[titan] 2025-07-09 13:52:19,621 - root - INFO - step: 1195 loss: 21.1666 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 3.7723 global_avg_mtp_loss: 17.3944 +[titan] 2025-07-09 13:52:19,621 - root - INFO - lr: 3.0000e-04 gnorm: 1.63 [ 0:17:24<23:58:56] +[titan] 2025-07-09 13:52:22,748 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:52:23,538 - root - INFO - step: 1200 loss: 21.3407 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 3.8290 global_avg_mtp_loss: 17.5117 +[titan] 2025-07-09 13:52:23,539 - root - INFO - lr: 3.0000e-04 gnorm: 1.73 [ 0:17:28<23:58:15] +[titan] 2025-07-09 13:52:27,464 - root - INFO - step: 1205 loss: 21.0454 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 3.7506 global_avg_mtp_loss: 17.2948 +[titan] 2025-07-09 13:52:27,464 - root - INFO - lr: 3.0000e-04 gnorm: 1.75 [ 0:17:32<23:57:34] +[titan] 2025-07-09 13:52:31,405 - root - INFO - step: 1210 loss: 21.2327 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 3.7721 global_avg_mtp_loss: 17.4606 +[titan] 2025-07-09 13:52:31,405 - root - INFO - lr: 3.0000e-04 gnorm: 1.66 [ 0:17:35<23:56:55] +[titan] 2025-07-09 13:52:35,304 - root - INFO - step: 1215 loss: 21.3702 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 3.8264 global_avg_mtp_loss: 17.5438 +[titan] 2025-07-09 13:52:35,304 - root - INFO - lr: 3.0000e-04 gnorm: 1.63 [ 0:17:39<23:56:13] +[titan] 2025-07-09 13:52:39,237 - root - INFO - step: 1220 loss: 21.1405 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 3.7477 global_avg_mtp_loss: 17.3928 +[titan] 2025-07-09 13:52:39,237 - root - INFO - lr: 3.0000e-04 gnorm: 1.84 [ 0:17:43<23:55:34] +[titan] 2025-07-09 13:52:43,140 - root - INFO - step: 1225 loss: 20.9837 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 3.7222 global_avg_mtp_loss: 17.2616 +[titan] 2025-07-09 13:52:43,140 - root - INFO - lr: 3.0000e-04 gnorm: 1.67 [ 0:17:47<23:54:53] +[titan] 2025-07-09 13:52:47,050 - root - INFO - step: 1230 loss: 21.1082 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 3.7508 global_avg_mtp_loss: 17.3574 +[titan] 2025-07-09 13:52:47,050 - root - INFO - lr: 3.0000e-04 gnorm: 1.65 [ 0:17:51<23:54:12] +[titan] 2025-07-09 13:52:50,947 - root - INFO - step: 1235 loss: 21.0691 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 3.7180 global_avg_mtp_loss: 17.3511 +[titan] 2025-07-09 13:52:50,948 - root - INFO - lr: 3.0000e-04 gnorm: 1.45 [ 0:17:55<23:53:31] +[titan] 2025-07-09 13:52:54,876 - root - INFO - step: 1240 loss: 20.9054 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 3.6811 global_avg_mtp_loss: 17.2243 +[titan] 2025-07-09 13:52:54,877 - root - INFO - lr: 3.0000e-04 gnorm: 1.34 [ 0:17:59<23:52:53] +[titan] 2025-07-09 13:52:58,786 - root - INFO - step: 1245 loss: 20.9044 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 3.6970 global_avg_mtp_loss: 17.2074 +[titan] 2025-07-09 13:52:58,787 - root - INFO - lr: 3.0000e-04 gnorm: 1.89 [ 0:18:03<23:52:13] +[titan] 2025-07-09 13:53:01,920 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:53:02,711 - root - INFO - step: 1250 loss: 20.8829 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 3.6849 global_avg_mtp_loss: 17.1980 +[titan] 2025-07-09 13:53:02,712 - root - INFO - lr: 3.0000e-04 gnorm: 1.68 [ 0:18:07<23:51:35] +[titan] 2025-07-09 13:53:06,626 - root - INFO - step: 1255 loss: 21.0053 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 3.6973 global_avg_mtp_loss: 17.3080 +[titan] 2025-07-09 13:53:06,626 - root - INFO - lr: 3.0000e-04 gnorm: 1.51 [ 0:18:11<23:50:57] +[titan] 2025-07-09 13:53:10,529 - root - INFO - step: 1260 loss: 20.7961 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 3.6474 global_avg_mtp_loss: 17.1487 +[titan] 2025-07-09 13:53:10,530 - root - INFO - lr: 3.0000e-04 gnorm: 1.60 [ 0:18:15<23:50:18] +[titan] 2025-07-09 13:53:14,476 - root - INFO - step: 1265 loss: 20.7249 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 3.6152 global_avg_mtp_loss: 17.1097 +[titan] 2025-07-09 13:53:14,476 - root - INFO - lr: 3.0000e-04 gnorm: 1.27 [ 0:18:19<23:49:42] +[titan] 2025-07-09 13:53:18,366 - root - INFO - step: 1270 loss: 20.9774 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 3.7092 global_avg_mtp_loss: 17.2682 +[titan] 2025-07-09 13:53:18,366 - root - INFO - lr: 3.0000e-04 gnorm: 1.51 [ 0:18:22<23:49:02] +[titan] 2025-07-09 13:53:22,288 - root - INFO - step: 1275 loss: 20.8809 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 3.6871 global_avg_mtp_loss: 17.1938 +[titan] 2025-07-09 13:53:22,289 - root - INFO - lr: 2.9999e-04 gnorm: 1.52 [ 0:18:26<23:48:25] +[titan] 2025-07-09 13:53:26,249 - root - INFO - step: 1280 loss: 20.6487 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 3.6087 global_avg_mtp_loss: 17.0401 +[titan] 2025-07-09 13:53:26,249 - root - INFO - lr: 2.9999e-04 gnorm: 1.30 [ 0:18:30<23:47:52] +[titan] 2025-07-09 13:53:30,143 - root - INFO - step: 1285 loss: 20.9716 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 3.6809 global_avg_mtp_loss: 17.2907 +[titan] 2025-07-09 13:53:30,144 - root - INFO - lr: 2.9999e-04 gnorm: 1.25 [ 0:18:34<23:47:13] +[titan] 2025-07-09 13:53:34,059 - root - INFO - step: 1290 loss: 20.6370 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 3.6058 global_avg_mtp_loss: 17.0311 +[titan] 2025-07-09 13:53:34,059 - root - INFO - lr: 2.9999e-04 gnorm: 1.48 [ 0:18:38<23:46:37] +[titan] 2025-07-09 13:53:37,996 - root - INFO - step: 1295 loss: 21.0033 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 3.6998 global_avg_mtp_loss: 17.3035 +[titan] 2025-07-09 13:53:37,996 - root - INFO - lr: 2.9999e-04 gnorm: 1.38 [ 0:18:42<23:46:02] +[titan] 2025-07-09 13:53:41,141 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:53:41,946 - root - INFO - step: 1300 loss: 20.6832 memory: 44.58GiB(31.99%) tps: 82,969 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 3.6212 global_avg_mtp_loss: 17.0620 +[titan] 2025-07-09 13:53:41,946 - root - INFO - lr: 2.9999e-04 gnorm: 1.44 [ 0:18:46<23:45:28] +[titan] 2025-07-09 13:53:45,868 - root - INFO - step: 1305 loss: 20.5199 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 3.5754 global_avg_mtp_loss: 16.9445 +[titan] 2025-07-09 13:53:45,868 - root - INFO - lr: 2.9999e-04 gnorm: 1.19 [ 0:18:50<23:44:53] +[titan] 2025-07-09 13:53:49,772 - root - INFO - step: 1310 loss: 20.8114 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 3.6413 global_avg_mtp_loss: 17.1701 +[titan] 2025-07-09 13:53:49,772 - root - INFO - lr: 2.9999e-04 gnorm: 1.27 [ 0:18:54<23:44:16] +[titan] 2025-07-09 13:53:53,706 - root - INFO - step: 1315 loss: 20.6029 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 3.5872 global_avg_mtp_loss: 17.0157 +[titan] 2025-07-09 13:53:53,706 - root - INFO - lr: 2.9999e-04 gnorm: 1.56 [ 0:18:58<23:43:42] +[titan] 2025-07-09 13:53:57,624 - root - INFO - step: 1320 loss: 20.6885 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 3.5976 global_avg_mtp_loss: 17.0909 +[titan] 2025-07-09 13:53:57,624 - root - INFO - lr: 2.9999e-04 gnorm: 1.31 [ 0:19:02<23:43:07] +[titan] 2025-07-09 13:54:01,521 - root - INFO - step: 1325 loss: 20.3463 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 3.5347 global_avg_mtp_loss: 16.8117 +[titan] 2025-07-09 13:54:01,522 - root - INFO - lr: 2.9999e-04 gnorm: 1.47 [ 0:19:06<23:42:31] +[titan] 2025-07-09 13:54:05,439 - root - INFO - step: 1330 loss: 20.5898 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 3.5870 global_avg_mtp_loss: 17.0028 +[titan] 2025-07-09 13:54:05,439 - root - INFO - lr: 2.9999e-04 gnorm: 1.36 [ 0:19:10<23:41:56] +[titan] 2025-07-09 13:54:09,342 - root - INFO - step: 1335 loss: 20.8697 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 3.6388 global_avg_mtp_loss: 17.2309 +[titan] 2025-07-09 13:54:09,342 - root - INFO - lr: 2.9999e-04 gnorm: 1.57 [ 0:19:13<23:41:21] +[titan] 2025-07-09 13:54:13,284 - root - INFO - step: 1340 loss: 20.7587 memory: 44.58GiB(31.99%) tps: 83,137 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 3.6090 global_avg_mtp_loss: 17.1496 +[titan] 2025-07-09 13:54:13,284 - root - INFO - lr: 2.9999e-04 gnorm: 1.38 [ 0:19:17<23:40:49] +[titan] 2025-07-09 13:54:17,183 - root - INFO - step: 1345 loss: 20.8172 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 3.6234 global_avg_mtp_loss: 17.1938 +[titan] 2025-07-09 13:54:17,184 - root - INFO - lr: 2.9999e-04 gnorm: 1.32 [ 0:19:21<23:40:13] +[titan] 2025-07-09 13:54:20,312 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:54:21,101 - root - INFO - step: 1350 loss: 20.5095 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 3.5507 global_avg_mtp_loss: 16.9589 +[titan] 2025-07-09 13:54:21,102 - root - INFO - lr: 2.9999e-04 gnorm: 1.24 [ 0:19:25<23:39:40] +[titan] 2025-07-09 13:54:25,011 - root - INFO - step: 1355 loss: 20.6167 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 3.6021 global_avg_mtp_loss: 17.0146 +[titan] 2025-07-09 13:54:25,012 - root - INFO - lr: 2.9999e-04 gnorm: 1.31 [ 0:19:29<23:39:06] +[titan] 2025-07-09 13:54:28,932 - root - INFO - step: 1360 loss: 20.5498 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 3.5697 global_avg_mtp_loss: 16.9801 +[titan] 2025-07-09 13:54:28,932 - root - INFO - lr: 2.9999e-04 gnorm: 1.43 [ 0:19:33<23:38:33] +[titan] 2025-07-09 13:54:32,859 - root - INFO - step: 1365 loss: 20.4102 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 3.5478 global_avg_mtp_loss: 16.8625 +[titan] 2025-07-09 13:54:32,860 - root - INFO - lr: 2.9999e-04 gnorm: 1.39 [ 0:19:37<23:38:00] +[titan] 2025-07-09 13:54:36,752 - root - INFO - step: 1370 loss: 20.5289 memory: 44.58GiB(31.99%) tps: 84,187 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 3.5613 global_avg_mtp_loss: 16.9676 +[titan] 2025-07-09 13:54:36,752 - root - INFO - lr: 2.9999e-04 gnorm: 1.38 [ 0:19:41<23:37:26] +[titan] 2025-07-09 13:54:40,657 - root - INFO - step: 1375 loss: 20.3826 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 3.5267 global_avg_mtp_loss: 16.8559 +[titan] 2025-07-09 13:54:40,657 - root - INFO - lr: 2.9999e-04 gnorm: 1.30 [ 0:19:45<23:36:52] +[titan] 2025-07-09 13:54:44,562 - root - INFO - step: 1380 loss: 20.5336 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 3.5476 global_avg_mtp_loss: 16.9860 +[titan] 2025-07-09 13:54:44,562 - root - INFO - lr: 2.9999e-04 gnorm: 1.42 [ 0:19:49<23:36:19] +[titan] 2025-07-09 13:54:48,473 - root - INFO - step: 1385 loss: 20.5238 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 3.5468 global_avg_mtp_loss: 16.9770 +[titan] 2025-07-09 13:54:48,474 - root - INFO - lr: 2.9999e-04 gnorm: 1.40 [ 0:19:53<23:35:46] +[titan] 2025-07-09 13:54:52,431 - root - INFO - step: 1390 loss: 20.2688 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 3.5189 global_avg_mtp_loss: 16.7500 +[titan] 2025-07-09 13:54:52,432 - root - INFO - lr: 2.9999e-04 gnorm: 1.23 [ 0:19:56<23:35:17] +[titan] 2025-07-09 13:54:56,316 - root - INFO - step: 1395 loss: 20.5244 memory: 44.58GiB(31.99%) tps: 84,369 tflops: 291.17 mfu: 29.44% global_avg_ntp_loss: 3.5585 global_avg_mtp_loss: 16.9659 +[titan] 2025-07-09 13:54:56,316 - root - INFO - lr: 2.9999e-04 gnorm: 1.38 [ 0:20:00<23:34:43] +[titan] 2025-07-09 13:54:59,460 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:55:00,248 - root - INFO - step: 1400 loss: 20.1911 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 3.4710 global_avg_mtp_loss: 16.7201 +[titan] 2025-07-09 13:55:00,248 - root - INFO - lr: 2.9999e-04 gnorm: 1.37 [ 0:20:04<23:34:13] +[titan] 2025-07-09 13:55:04,142 - root - INFO - step: 1405 loss: 20.2161 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 3.4747 global_avg_mtp_loss: 16.7414 +[titan] 2025-07-09 13:55:04,142 - root - INFO - lr: 2.9999e-04 gnorm: 1.17 [ 0:20:08<23:33:40] +[titan] 2025-07-09 13:55:08,053 - root - INFO - step: 1410 loss: 20.4312 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 3.5146 global_avg_mtp_loss: 16.9166 +[titan] 2025-07-09 13:55:08,054 - root - INFO - lr: 2.9999e-04 gnorm: 1.28 [ 0:20:12<23:33:08] +[titan] 2025-07-09 13:55:11,954 - root - INFO - step: 1415 loss: 20.3747 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 3.5181 global_avg_mtp_loss: 16.8566 +[titan] 2025-07-09 13:55:11,954 - root - INFO - lr: 2.9999e-04 gnorm: 1.17 [ 0:20:16<23:32:36] +[titan] 2025-07-09 13:55:15,854 - root - INFO - step: 1420 loss: 20.2021 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 3.4869 global_avg_mtp_loss: 16.7152 +[titan] 2025-07-09 13:55:15,854 - root - INFO - lr: 2.9999e-04 gnorm: 1.39 [ 0:20:20<23:32:04] +[titan] 2025-07-09 13:55:19,764 - root - INFO - step: 1425 loss: 20.1733 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 3.4781 global_avg_mtp_loss: 16.6951 +[titan] 2025-07-09 13:55:19,764 - root - INFO - lr: 2.9999e-04 gnorm: 1.25 [ 0:20:24<23:31:33] +[titan] 2025-07-09 13:55:23,708 - root - INFO - step: 1430 loss: 20.2346 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 3.5036 global_avg_mtp_loss: 16.7309 +[titan] 2025-07-09 13:55:23,708 - root - INFO - lr: 2.9999e-04 gnorm: 1.20 [ 0:20:28<23:31:04] +[titan] 2025-07-09 13:55:27,619 - root - INFO - step: 1435 loss: 20.3942 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 3.5347 global_avg_mtp_loss: 16.8595 +[titan] 2025-07-09 13:55:27,619 - root - INFO - lr: 2.9999e-04 gnorm: 1.16 [ 0:20:32<23:30:33] +[titan] 2025-07-09 13:55:31,541 - root - INFO - step: 1440 loss: 20.3806 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 3.5193 global_avg_mtp_loss: 16.8613 +[titan] 2025-07-09 13:55:31,541 - root - INFO - lr: 2.9999e-04 gnorm: 1.30 [ 0:20:36<23:30:04] +[titan] 2025-07-09 13:55:35,431 - root - INFO - step: 1445 loss: 20.2266 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 3.4687 global_avg_mtp_loss: 16.7579 +[titan] 2025-07-09 13:55:35,432 - root - INFO - lr: 2.9999e-04 gnorm: 1.25 [ 0:20:39<23:29:32] +[titan] 2025-07-09 13:55:38,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:55:39,349 - root - INFO - step: 1450 loss: 20.2744 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 3.4847 global_avg_mtp_loss: 16.7896 +[titan] 2025-07-09 13:55:39,350 - root - INFO - lr: 2.9999e-04 gnorm: 1.29 [ 0:20:43<23:29:02] +[titan] 2025-07-09 13:55:43,297 - root - INFO - step: 1455 loss: 20.2835 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 3.4896 global_avg_mtp_loss: 16.7939 +[titan] 2025-07-09 13:55:43,297 - root - INFO - lr: 2.9999e-04 gnorm: 1.48 [ 0:20:47<23:28:35] +[titan] 2025-07-09 13:55:47,273 - root - INFO - step: 1460 loss: 20.0337 memory: 44.58GiB(31.99%) tps: 82,426 tflops: 284.46 mfu: 28.76% global_avg_ntp_loss: 3.4339 global_avg_mtp_loss: 16.5998 +[titan] 2025-07-09 13:55:47,273 - root - INFO - lr: 2.9999e-04 gnorm: 1.17 [ 0:20:51<23:28:10] +[titan] 2025-07-09 13:55:51,169 - root - INFO - step: 1465 loss: 20.2660 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 3.4763 global_avg_mtp_loss: 16.7896 +[titan] 2025-07-09 13:55:51,169 - root - INFO - lr: 2.9999e-04 gnorm: 1.26 [ 0:20:55<23:27:39] +[titan] 2025-07-09 13:55:55,081 - root - INFO - step: 1470 loss: 20.3474 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 3.5019 global_avg_mtp_loss: 16.8455 +[titan] 2025-07-09 13:55:55,082 - root - INFO - lr: 2.9998e-04 gnorm: 1.22 [ 0:20:59<23:27:10] +[titan] 2025-07-09 13:55:59,003 - root - INFO - step: 1475 loss: 20.1974 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 3.4736 global_avg_mtp_loss: 16.7238 +[titan] 2025-07-09 13:55:59,003 - root - INFO - lr: 2.9998e-04 gnorm: 1.20 [ 0:21:03<23:26:41] +[titan] 2025-07-09 13:56:02,893 - root - INFO - step: 1480 loss: 20.2161 memory: 44.58GiB(31.99%) tps: 84,236 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 3.4709 global_avg_mtp_loss: 16.7452 +[titan] 2025-07-09 13:56:02,893 - root - INFO - lr: 2.9998e-04 gnorm: 1.20 [ 0:21:07<23:26:11] +[titan] 2025-07-09 13:56:06,826 - root - INFO - step: 1485 loss: 20.2850 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 3.4762 global_avg_mtp_loss: 16.8087 +[titan] 2025-07-09 13:56:06,826 - root - INFO - lr: 2.9998e-04 gnorm: 1.22 [ 0:21:11<23:25:43] +[titan] 2025-07-09 13:56:10,727 - root - INFO - step: 1490 loss: 20.2381 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 3.4743 global_avg_mtp_loss: 16.7638 +[titan] 2025-07-09 13:56:10,727 - root - INFO - lr: 2.9998e-04 gnorm: 1.21 [ 0:21:15<23:25:14] +[titan] 2025-07-09 13:56:14,666 - root - INFO - step: 1495 loss: 19.9442 memory: 44.58GiB(31.99%) tps: 83,199 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 3.3998 global_avg_mtp_loss: 16.5444 +[titan] 2025-07-09 13:56:14,666 - root - INFO - lr: 2.9998e-04 gnorm: 1.25 [ 0:21:19<23:24:47] +[titan] 2025-07-09 13:56:17,852 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:56:18,651 - root - INFO - step: 1500 loss: 20.1351 memory: 44.58GiB(31.99%) tps: 82,224 tflops: 283.77 mfu: 28.69% global_avg_ntp_loss: 3.4497 global_avg_mtp_loss: 16.6854 +[titan] 2025-07-09 13:56:18,651 - root - INFO - lr: 2.9998e-04 gnorm: 1.14 [ 0:21:23<23:24:23] +[titan] 2025-07-09 13:56:22,580 - root - INFO - step: 1505 loss: 20.1780 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 3.4632 global_avg_mtp_loss: 16.7148 +[titan] 2025-07-09 13:56:22,580 - root - INFO - lr: 2.9998e-04 gnorm: 1.27 [ 0:21:27<23:23:56] +[titan] 2025-07-09 13:56:26,554 - root - INFO - step: 1510 loss: 20.0416 memory: 44.58GiB(31.99%) tps: 82,472 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 3.4267 global_avg_mtp_loss: 16.6149 +[titan] 2025-07-09 13:56:26,554 - root - INFO - lr: 2.9998e-04 gnorm: 1.08 [ 0:21:31<23:23:32] +[titan] 2025-07-09 13:56:30,496 - root - INFO - step: 1515 loss: 20.0848 memory: 44.58GiB(31.99%) tps: 83,122 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 3.4301 global_avg_mtp_loss: 16.6547 +[titan] 2025-07-09 13:56:30,496 - root - INFO - lr: 2.9998e-04 gnorm: 1.08 [ 0:21:35<23:23:06] +[titan] 2025-07-09 13:56:34,426 - root - INFO - step: 1520 loss: 20.1078 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 3.4367 global_avg_mtp_loss: 16.6711 +[titan] 2025-07-09 13:56:34,426 - root - INFO - lr: 2.9998e-04 gnorm: 1.20 [ 0:21:38<23:22:40] +[titan] 2025-07-09 13:56:38,355 - root - INFO - step: 1525 loss: 19.9861 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 3.4045 global_avg_mtp_loss: 16.5816 +[titan] 2025-07-09 13:56:38,355 - root - INFO - lr: 2.9998e-04 gnorm: 1.19 [ 0:21:42<23:22:13] +[titan] 2025-07-09 13:56:42,291 - root - INFO - step: 1530 loss: 20.0307 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 3.4183 global_avg_mtp_loss: 16.6124 +[titan] 2025-07-09 13:56:42,291 - root - INFO - lr: 2.9998e-04 gnorm: 1.25 [ 0:21:46<23:21:47] +[titan] 2025-07-09 13:56:46,342 - root - INFO - step: 1535 loss: 19.9553 memory: 44.58GiB(31.99%) tps: 80,894 tflops: 279.18 mfu: 28.23% global_avg_ntp_loss: 3.4055 global_avg_mtp_loss: 16.5498 +[titan] 2025-07-09 13:56:46,342 - root - INFO - lr: 2.9998e-04 gnorm: 1.23 [ 0:21:50<23:21:29] +[titan] 2025-07-09 13:56:47,250 - root - INFO - Dumping profiler traces at step 1536 +[titan] 2025-07-09 13:56:47,281 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 13:56:50,406 - root - INFO - step: 1540 loss: 19.9601 memory: 44.58GiB(31.99%) tps: 80,625 tflops: 278.25 mfu: 28.13% global_avg_ntp_loss: 3.3998 global_avg_mtp_loss: 16.5603 +[titan] 2025-07-09 13:56:50,407 - root - INFO - lr: 2.9998e-04 gnorm: 1.12 [ 0:21:54<23:21:11] +[titan] 2025-07-09 13:56:54,354 - root - INFO - step: 1545 loss: 19.7696 memory: 44.58GiB(31.99%) tps: 83,010 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 3.3775 global_avg_mtp_loss: 16.3921 +[titan] 2025-07-09 13:56:54,354 - root - INFO - lr: 2.9998e-04 gnorm: 1.28 [ 0:21:58<23:20:47] +[titan] 2025-07-09 13:56:57,488 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:56:58,276 - root - INFO - step: 1550 loss: 19.9357 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 3.3926 global_avg_mtp_loss: 16.5431 +[titan] 2025-07-09 13:56:58,276 - root - INFO - lr: 2.9998e-04 gnorm: 1.26 [ 0:22:02<23:20:20] +[titan] 2025-07-09 13:57:02,171 - root - INFO - step: 1555 loss: 19.8914 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 3.4085 global_avg_mtp_loss: 16.4830 +[titan] 2025-07-09 13:57:02,171 - root - INFO - lr: 2.9998e-04 gnorm: 1.29 [ 0:22:06<23:19:52] +[titan] 2025-07-09 13:57:06,079 - root - INFO - step: 1560 loss: 20.0631 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 3.4190 global_avg_mtp_loss: 16.6441 +[titan] 2025-07-09 13:57:06,079 - root - INFO - lr: 2.9998e-04 gnorm: 1.11 [ 0:22:10<23:19:26] +[titan] 2025-07-09 13:57:09,973 - root - INFO - step: 1565 loss: 19.9638 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 3.3969 global_avg_mtp_loss: 16.5668 +[titan] 2025-07-09 13:57:09,974 - root - INFO - lr: 2.9998e-04 gnorm: 1.29 [ 0:22:14<23:18:58] +[titan] 2025-07-09 13:57:13,914 - root - INFO - step: 1570 loss: 20.1797 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 3.4418 global_avg_mtp_loss: 16.7379 +[titan] 2025-07-09 13:57:13,914 - root - INFO - lr: 2.9998e-04 gnorm: 1.32 [ 0:22:18<23:18:33] +[titan] 2025-07-09 13:57:17,805 - root - INFO - step: 1575 loss: 19.9689 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 3.3930 global_avg_mtp_loss: 16.5759 +[titan] 2025-07-09 13:57:17,805 - root - INFO - lr: 2.9998e-04 gnorm: 1.30 [ 0:22:22<23:18:06] +[titan] 2025-07-09 13:57:21,697 - root - INFO - step: 1580 loss: 19.9850 memory: 44.58GiB(31.99%) tps: 84,201 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 3.4048 global_avg_mtp_loss: 16.5802 +[titan] 2025-07-09 13:57:21,697 - root - INFO - lr: 2.9998e-04 gnorm: 1.22 [ 0:22:26<23:17:39] +[titan] 2025-07-09 13:57:25,592 - root - INFO - step: 1585 loss: 19.9564 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 3.3983 global_avg_mtp_loss: 16.5582 +[titan] 2025-07-09 13:57:25,592 - root - INFO - lr: 2.9998e-04 gnorm: 1.11 [ 0:22:30<23:17:12] +[titan] 2025-07-09 13:57:29,486 - root - INFO - step: 1590 loss: 19.8842 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 3.3868 global_avg_mtp_loss: 16.4974 +[titan] 2025-07-09 13:57:29,486 - root - INFO - lr: 2.9998e-04 gnorm: 1.17 [ 0:22:34<23:16:45] +[titan] 2025-07-09 13:57:33,424 - root - INFO - step: 1595 loss: 20.0375 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 3.4097 global_avg_mtp_loss: 16.6277 +[titan] 2025-07-09 13:57:33,424 - root - INFO - lr: 2.9998e-04 gnorm: 1.12 [ 0:22:37<23:16:21] +[titan] 2025-07-09 13:57:36,529 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:57:37,338 - root - INFO - step: 1600 loss: 19.8995 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 3.3878 global_avg_mtp_loss: 16.5117 +[titan] 2025-07-09 13:57:37,338 - root - INFO - lr: 2.9998e-04 gnorm: 1.13 [ 0:22:41<23:15:55] +[titan] 2025-07-09 13:57:41,264 - root - INFO - step: 1605 loss: 20.0143 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 3.4188 global_avg_mtp_loss: 16.5956 +[titan] 2025-07-09 13:57:41,264 - root - INFO - lr: 2.9998e-04 gnorm: 1.45 [ 0:22:45<23:15:31] +[titan] 2025-07-09 13:57:45,203 - root - INFO - step: 1610 loss: 19.9085 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 3.3893 global_avg_mtp_loss: 16.5192 +[titan] 2025-07-09 13:57:45,203 - root - INFO - lr: 2.9997e-04 gnorm: 1.13 [ 0:22:49<23:15:07] +[titan] 2025-07-09 13:57:49,110 - root - INFO - step: 1615 loss: 19.9515 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 3.3929 global_avg_mtp_loss: 16.5586 +[titan] 2025-07-09 13:57:49,111 - root - INFO - lr: 2.9997e-04 gnorm: 1.10 [ 0:22:53<23:14:42] +[titan] 2025-07-09 13:57:53,051 - root - INFO - step: 1620 loss: 19.8652 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 3.3653 global_avg_mtp_loss: 16.4999 +[titan] 2025-07-09 13:57:53,051 - root - INFO - lr: 2.9997e-04 gnorm: 1.17 [ 0:22:57<23:14:19] +[titan] 2025-07-09 13:57:56,951 - root - INFO - step: 1625 loss: 19.8861 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 3.3709 global_avg_mtp_loss: 16.5153 +[titan] 2025-07-09 13:57:56,951 - root - INFO - lr: 2.9997e-04 gnorm: 1.17 [ 0:23:01<23:13:53] +[titan] 2025-07-09 13:58:00,870 - root - INFO - step: 1630 loss: 19.7995 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 3.3753 global_avg_mtp_loss: 16.4242 +[titan] 2025-07-09 13:58:00,871 - root - INFO - lr: 2.9997e-04 gnorm: 1.16 [ 0:23:05<23:13:29] +[titan] 2025-07-09 13:58:04,784 - root - INFO - step: 1635 loss: 19.9085 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 3.3831 global_avg_mtp_loss: 16.5254 +[titan] 2025-07-09 13:58:04,784 - root - INFO - lr: 2.9997e-04 gnorm: 1.07 [ 0:23:09<23:13:04] +[titan] 2025-07-09 13:58:08,735 - root - INFO - step: 1640 loss: 19.6869 memory: 44.58GiB(31.99%) tps: 82,928 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 3.3394 global_avg_mtp_loss: 16.3476 +[titan] 2025-07-09 13:58:08,736 - root - INFO - lr: 2.9997e-04 gnorm: 1.08 [ 0:23:13<23:12:42] +[titan] 2025-07-09 13:58:12,657 - root - INFO - step: 1645 loss: 19.9334 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 3.3783 global_avg_mtp_loss: 16.5551 +[titan] 2025-07-09 13:58:12,657 - root - INFO - lr: 2.9997e-04 gnorm: 1.21 [ 0:23:17<23:12:18] +[titan] 2025-07-09 13:58:15,773 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:58:16,560 - root - INFO - step: 1650 loss: 19.9695 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 3.3826 global_avg_mtp_loss: 16.5869 +[titan] 2025-07-09 13:58:16,560 - root - INFO - lr: 2.9997e-04 gnorm: 1.09 [ 0:23:21<23:11:54] +[titan] 2025-07-09 13:58:20,447 - root - INFO - step: 1655 loss: 19.9903 memory: 44.58GiB(31.99%) tps: 84,298 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 3.3888 global_avg_mtp_loss: 16.6015 +[titan] 2025-07-09 13:58:20,447 - root - INFO - lr: 2.9997e-04 gnorm: 1.17 [ 0:23:24<23:11:28] +[titan] 2025-07-09 13:58:24,352 - root - INFO - step: 1660 loss: 19.6613 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 3.3173 global_avg_mtp_loss: 16.3440 +[titan] 2025-07-09 13:58:24,352 - root - INFO - lr: 2.9997e-04 gnorm: 1.15 [ 0:23:28<23:11:04] +[titan] 2025-07-09 13:58:28,252 - root - INFO - step: 1665 loss: 19.8727 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 3.3712 global_avg_mtp_loss: 16.5015 +[titan] 2025-07-09 13:58:28,252 - root - INFO - lr: 2.9997e-04 gnorm: 1.15 [ 0:23:32<23:10:39] +[titan] 2025-07-09 13:58:32,153 - root - INFO - step: 1670 loss: 19.6439 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 3.3103 global_avg_mtp_loss: 16.3336 +[titan] 2025-07-09 13:58:32,153 - root - INFO - lr: 2.9997e-04 gnorm: 1.15 [ 0:23:36<23:10:15] +[titan] 2025-07-09 13:58:36,056 - root - INFO - step: 1675 loss: 19.5381 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 3.3148 global_avg_mtp_loss: 16.2233 +[titan] 2025-07-09 13:58:36,056 - root - INFO - lr: 2.9997e-04 gnorm: 1.12 [ 0:23:40<23:09:50] +[titan] 2025-07-09 13:58:39,967 - root - INFO - step: 1680 loss: 19.8524 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 3.3508 global_avg_mtp_loss: 16.5016 +[titan] 2025-07-09 13:58:39,968 - root - INFO - lr: 2.9997e-04 gnorm: 1.17 [ 0:23:44<23:09:27] +[titan] 2025-07-09 13:58:43,861 - root - INFO - step: 1685 loss: 19.9711 memory: 44.58GiB(31.99%) tps: 84,169 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 3.3918 global_avg_mtp_loss: 16.5793 +[titan] 2025-07-09 13:58:43,861 - root - INFO - lr: 2.9997e-04 gnorm: 1.02 [ 0:23:48<23:09:02] +[titan] 2025-07-09 13:58:47,797 - root - INFO - step: 1690 loss: 19.8541 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 3.3668 global_avg_mtp_loss: 16.4873 +[titan] 2025-07-09 13:58:47,797 - root - INFO - lr: 2.9997e-04 gnorm: 1.15 [ 0:23:52<23:08:41] +[titan] 2025-07-09 13:58:51,745 - root - INFO - step: 1695 loss: 19.7719 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 3.3356 global_avg_mtp_loss: 16.4363 +[titan] 2025-07-09 13:58:51,746 - root - INFO - lr: 2.9997e-04 gnorm: 1.03 [ 0:23:56<23:08:20] +[titan] 2025-07-09 13:58:54,868 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:58:55,651 - root - INFO - step: 1700 loss: 19.8014 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 3.3627 global_avg_mtp_loss: 16.4387 +[titan] 2025-07-09 13:58:55,651 - root - INFO - lr: 2.9997e-04 gnorm: 1.03 [ 0:24:00<23:07:56] +[titan] 2025-07-09 13:58:59,559 - root - INFO - step: 1705 loss: 19.7516 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 3.3439 global_avg_mtp_loss: 16.4077 +[titan] 2025-07-09 13:58:59,560 - root - INFO - lr: 2.9997e-04 gnorm: 1.08 [ 0:24:04<23:07:33] +[titan] 2025-07-09 13:59:03,497 - root - INFO - step: 1710 loss: 19.5852 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 3.3139 global_avg_mtp_loss: 16.2712 +[titan] 2025-07-09 13:59:03,497 - root - INFO - lr: 2.9997e-04 gnorm: 1.17 [ 0:24:08<23:07:12] +[titan] 2025-07-09 13:59:07,403 - root - INFO - step: 1715 loss: 19.7492 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 3.3360 global_avg_mtp_loss: 16.4133 +[titan] 2025-07-09 13:59:07,404 - root - INFO - lr: 2.9997e-04 gnorm: 1.00 [ 0:24:11<23:06:49] +[titan] 2025-07-09 13:59:11,340 - root - INFO - step: 1720 loss: 19.6047 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 3.3031 global_avg_mtp_loss: 16.3016 +[titan] 2025-07-09 13:59:11,340 - root - INFO - lr: 2.9996e-04 gnorm: 1.14 [ 0:24:15<23:06:27] +[titan] 2025-07-09 13:59:15,246 - root - INFO - step: 1725 loss: 19.5242 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 3.2849 global_avg_mtp_loss: 16.2393 +[titan] 2025-07-09 13:59:15,246 - root - INFO - lr: 2.9996e-04 gnorm: 1.01 [ 0:24:19<23:06:05] +[titan] 2025-07-09 13:59:19,180 - root - INFO - step: 1730 loss: 19.6429 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 3.3148 global_avg_mtp_loss: 16.3282 +[titan] 2025-07-09 13:59:19,180 - root - INFO - lr: 2.9996e-04 gnorm: 1.06 [ 0:24:23<23:05:43] +[titan] 2025-07-09 13:59:23,082 - root - INFO - step: 1735 loss: 19.7209 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 3.3119 global_avg_mtp_loss: 16.4091 +[titan] 2025-07-09 13:59:23,082 - root - INFO - lr: 2.9996e-04 gnorm: 1.11 [ 0:24:27<23:05:21] +[titan] 2025-07-09 13:59:27,038 - root - INFO - step: 1740 loss: 19.9505 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 3.3891 global_avg_mtp_loss: 16.5614 +[titan] 2025-07-09 13:59:27,038 - root - INFO - lr: 2.9996e-04 gnorm: 1.18 [ 0:24:31<23:05:01] +[titan] 2025-07-09 13:59:30,934 - root - INFO - step: 1745 loss: 19.5584 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 3.2935 global_avg_mtp_loss: 16.2649 +[titan] 2025-07-09 13:59:30,934 - root - INFO - lr: 2.9996e-04 gnorm: 1.08 [ 0:24:35<23:04:38] +[titan] 2025-07-09 13:59:34,060 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 13:59:34,850 - root - INFO - step: 1750 loss: 19.9411 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 3.3667 global_avg_mtp_loss: 16.5744 +[titan] 2025-07-09 13:59:34,850 - root - INFO - lr: 2.9996e-04 gnorm: 1.02 [ 0:24:39<23:04:16] +[titan] 2025-07-09 13:59:38,768 - root - INFO - step: 1755 loss: 19.7868 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 3.3416 global_avg_mtp_loss: 16.4452 +[titan] 2025-07-09 13:59:38,768 - root - INFO - lr: 2.9996e-04 gnorm: 1.18 [ 0:24:43<23:03:55] +[titan] 2025-07-09 13:59:42,675 - root - INFO - step: 1760 loss: 19.9497 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 3.3684 global_avg_mtp_loss: 16.5813 +[titan] 2025-07-09 13:59:42,676 - root - INFO - lr: 2.9996e-04 gnorm: 1.06 [ 0:24:47<23:03:33] +[titan] 2025-07-09 13:59:46,580 - root - INFO - step: 1765 loss: 19.7176 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 3.3302 global_avg_mtp_loss: 16.3874 +[titan] 2025-07-09 13:59:46,581 - root - INFO - lr: 2.9996e-04 gnorm: 1.09 [ 0:24:51<23:03:11] +[titan] 2025-07-09 13:59:50,494 - root - INFO - step: 1770 loss: 19.3535 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 3.2535 global_avg_mtp_loss: 16.1000 +[titan] 2025-07-09 13:59:50,494 - root - INFO - lr: 2.9996e-04 gnorm: 1.07 [ 0:24:55<23:02:49] +[titan] 2025-07-09 13:59:54,387 - root - INFO - step: 1775 loss: 19.6524 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 3.2906 global_avg_mtp_loss: 16.3618 +[titan] 2025-07-09 13:59:54,387 - root - INFO - lr: 2.9996e-04 gnorm: 1.15 [ 0:24:58<23:02:27] +[titan] 2025-07-09 13:59:58,317 - root - INFO - step: 1780 loss: 19.6842 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 3.3287 global_avg_mtp_loss: 16.3555 +[titan] 2025-07-09 13:59:58,317 - root - INFO - lr: 2.9996e-04 gnorm: 1.08 [ 0:25:02<23:02:06] +[titan] 2025-07-09 14:00:02,250 - root - INFO - step: 1785 loss: 19.7863 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 3.3396 global_avg_mtp_loss: 16.4468 +[titan] 2025-07-09 14:00:02,250 - root - INFO - lr: 2.9996e-04 gnorm: 1.16 [ 0:25:06<23:01:46] +[titan] 2025-07-09 14:00:06,162 - root - INFO - step: 1790 loss: 19.7334 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 3.3298 global_avg_mtp_loss: 16.4036 +[titan] 2025-07-09 14:00:06,163 - root - INFO - lr: 2.9996e-04 gnorm: 1.10 [ 0:25:10<23:01:25] +[titan] 2025-07-09 14:00:10,078 - root - INFO - step: 1795 loss: 19.5957 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 3.2928 global_avg_mtp_loss: 16.3029 +[titan] 2025-07-09 14:00:10,079 - root - INFO - lr: 2.9996e-04 gnorm: 1.01 [ 0:25:14<23:01:04] +[titan] 2025-07-09 14:00:13,216 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:00:14,004 - root - INFO - step: 1800 loss: 19.3777 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 3.2475 global_avg_mtp_loss: 16.1303 +[titan] 2025-07-09 14:00:14,004 - root - INFO - lr: 2.9996e-04 gnorm: 1.06 [ 0:25:18<23:00:44] +[titan] 2025-07-09 14:00:17,925 - root - INFO - step: 1805 loss: 19.4825 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 3.2658 global_avg_mtp_loss: 16.2167 +[titan] 2025-07-09 14:00:17,925 - root - INFO - lr: 2.9996e-04 gnorm: 1.09 [ 0:25:22<23:00:23] +[titan] 2025-07-09 14:00:21,841 - root - INFO - step: 1810 loss: 19.3425 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 3.2372 global_avg_mtp_loss: 16.1053 +[titan] 2025-07-09 14:00:21,841 - root - INFO - lr: 2.9996e-04 gnorm: 1.07 [ 0:25:26<23:00:03] +[titan] 2025-07-09 14:00:25,745 - root - INFO - step: 1815 loss: 19.4101 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 3.2687 global_avg_mtp_loss: 16.1414 +[titan] 2025-07-09 14:00:25,746 - root - INFO - lr: 2.9995e-04 gnorm: 1.08 [ 0:25:30<22:59:42] +[titan] 2025-07-09 14:00:29,694 - root - INFO - step: 1820 loss: 19.5714 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 3.2805 global_avg_mtp_loss: 16.2909 +[titan] 2025-07-09 14:00:29,694 - root - INFO - lr: 2.9995e-04 gnorm: 1.08 [ 0:25:34<22:59:23] +[titan] 2025-07-09 14:00:33,631 - root - INFO - step: 1825 loss: 19.5983 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 3.3015 global_avg_mtp_loss: 16.2968 +[titan] 2025-07-09 14:00:33,632 - root - INFO - lr: 2.9995e-04 gnorm: 1.03 [ 0:25:38<22:59:04] +[titan] 2025-07-09 14:00:37,549 - root - INFO - step: 1830 loss: 19.5545 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 3.2986 global_avg_mtp_loss: 16.2559 +[titan] 2025-07-09 14:00:37,550 - root - INFO - lr: 2.9995e-04 gnorm: 1.10 [ 0:25:42<22:58:44] +[titan] 2025-07-09 14:00:41,492 - root - INFO - step: 1835 loss: 19.4438 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 3.2540 global_avg_mtp_loss: 16.1899 +[titan] 2025-07-09 14:00:41,493 - root - INFO - lr: 2.9995e-04 gnorm: 0.98 [ 0:25:46<22:58:25] +[titan] 2025-07-09 14:00:45,409 - root - INFO - step: 1840 loss: 19.6070 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 3.3045 global_avg_mtp_loss: 16.3025 +[titan] 2025-07-09 14:00:45,409 - root - INFO - lr: 2.9995e-04 gnorm: 0.99 [ 0:25:49<22:58:05] +[titan] 2025-07-09 14:00:49,325 - root - INFO - step: 1845 loss: 19.7378 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 3.3195 global_avg_mtp_loss: 16.4182 +[titan] 2025-07-09 14:00:49,325 - root - INFO - lr: 2.9995e-04 gnorm: 1.08 [ 0:25:53<22:57:45] +[titan] 2025-07-09 14:00:52,449 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:00:53,231 - root - INFO - step: 1850 loss: 19.4284 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 3.2507 global_avg_mtp_loss: 16.1778 +[titan] 2025-07-09 14:00:53,232 - root - INFO - lr: 2.9995e-04 gnorm: 1.04 [ 0:25:57<22:57:25] +[titan] 2025-07-09 14:00:57,152 - root - INFO - step: 1855 loss: 19.6343 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 3.2981 global_avg_mtp_loss: 16.3362 +[titan] 2025-07-09 14:00:57,153 - root - INFO - lr: 2.9995e-04 gnorm: 1.03 [ 0:26:01<22:57:05] +[titan] 2025-07-09 14:01:01,061 - root - INFO - step: 1860 loss: 19.3719 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 3.2653 global_avg_mtp_loss: 16.1065 +[titan] 2025-07-09 14:01:01,061 - root - INFO - lr: 2.9995e-04 gnorm: 1.00 [ 0:26:05<22:56:45] +[titan] 2025-07-09 14:01:04,949 - root - INFO - step: 1865 loss: 19.6809 memory: 44.58GiB(31.99%) tps: 84,294 tflops: 290.91 mfu: 29.41% global_avg_ntp_loss: 3.2938 global_avg_mtp_loss: 16.3871 +[titan] 2025-07-09 14:01:04,949 - root - INFO - lr: 2.9995e-04 gnorm: 1.00 [ 0:26:09<22:56:24] +[titan] 2025-07-09 14:01:08,869 - root - INFO - step: 1870 loss: 19.5783 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 3.2888 global_avg_mtp_loss: 16.2895 +[titan] 2025-07-09 14:01:08,869 - root - INFO - lr: 2.9995e-04 gnorm: 0.95 [ 0:26:13<22:56:05] +[titan] 2025-07-09 14:01:12,790 - root - INFO - step: 1875 loss: 19.5004 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 3.2638 global_avg_mtp_loss: 16.2366 +[titan] 2025-07-09 14:01:12,790 - root - INFO - lr: 2.9995e-04 gnorm: 0.94 [ 0:26:17<22:55:45] +[titan] 2025-07-09 14:01:16,697 - root - INFO - step: 1880 loss: 19.5485 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 3.2673 global_avg_mtp_loss: 16.2812 +[titan] 2025-07-09 14:01:16,697 - root - INFO - lr: 2.9995e-04 gnorm: 1.38 [ 0:26:21<22:55:25] +[titan] 2025-07-09 14:01:20,587 - root - INFO - step: 1885 loss: 19.3611 memory: 44.58GiB(31.99%) tps: 84,229 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 3.2410 global_avg_mtp_loss: 16.1201 +[titan] 2025-07-09 14:01:20,587 - root - INFO - lr: 2.9995e-04 gnorm: 1.15 [ 0:26:25<22:55:05] +[titan] 2025-07-09 14:01:24,510 - root - INFO - step: 1890 loss: 19.4962 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 3.2594 global_avg_mtp_loss: 16.2368 +[titan] 2025-07-09 14:01:24,510 - root - INFO - lr: 2.9995e-04 gnorm: 1.07 [ 0:26:29<22:54:46] +[titan] 2025-07-09 14:01:28,406 - root - INFO - step: 1895 loss: 19.5353 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 3.2981 global_avg_mtp_loss: 16.2373 +[titan] 2025-07-09 14:01:28,406 - root - INFO - lr: 2.9995e-04 gnorm: 1.09 [ 0:26:32<22:54:26] +[titan] 2025-07-09 14:01:31,517 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:01:32,321 - root - INFO - step: 1900 loss: 19.6405 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 3.3090 global_avg_mtp_loss: 16.3314 +[titan] 2025-07-09 14:01:32,321 - root - INFO - lr: 2.9994e-04 gnorm: 1.04 [ 0:26:36<22:54:07] +[titan] 2025-07-09 14:01:36,266 - root - INFO - step: 1905 loss: 19.2108 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 3.2096 global_avg_mtp_loss: 16.0012 +[titan] 2025-07-09 14:01:36,266 - root - INFO - lr: 2.9994e-04 gnorm: 1.07 [ 0:26:40<22:53:49] +[titan] 2025-07-09 14:01:40,187 - root - INFO - step: 1910 loss: 19.4439 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 3.2544 global_avg_mtp_loss: 16.1895 +[titan] 2025-07-09 14:01:40,188 - root - INFO - lr: 2.9994e-04 gnorm: 1.02 [ 0:26:44<22:53:31] +[titan] 2025-07-09 14:01:44,130 - root - INFO - step: 1915 loss: 19.3668 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 3.2419 global_avg_mtp_loss: 16.1248 +[titan] 2025-07-09 14:01:44,130 - root - INFO - lr: 2.9994e-04 gnorm: 1.08 [ 0:26:48<22:53:13] +[titan] 2025-07-09 14:01:48,079 - root - INFO - step: 1920 loss: 19.1855 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 3.2049 global_avg_mtp_loss: 15.9806 +[titan] 2025-07-09 14:01:48,079 - root - INFO - lr: 2.9994e-04 gnorm: 1.01 [ 0:26:52<22:52:56] +[titan] 2025-07-09 14:01:51,996 - root - INFO - step: 1925 loss: 19.1665 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 3.1823 global_avg_mtp_loss: 15.9842 +[titan] 2025-07-09 14:01:51,996 - root - INFO - lr: 2.9994e-04 gnorm: 1.00 [ 0:26:56<22:52:38] +[titan] 2025-07-09 14:01:55,907 - root - INFO - step: 1930 loss: 19.5824 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 3.2800 global_avg_mtp_loss: 16.3024 +[titan] 2025-07-09 14:01:55,907 - root - INFO - lr: 2.9994e-04 gnorm: 1.02 [ 0:27:00<22:52:19] +[titan] 2025-07-09 14:01:59,810 - root - INFO - step: 1935 loss: 19.5573 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 3.2922 global_avg_mtp_loss: 16.2652 +[titan] 2025-07-09 14:01:59,811 - root - INFO - lr: 2.9994e-04 gnorm: 1.14 [ 0:27:04<22:52:00] +[titan] 2025-07-09 14:02:03,716 - root - INFO - step: 1940 loss: 19.3289 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 3.2282 global_avg_mtp_loss: 16.1007 +[titan] 2025-07-09 14:02:03,716 - root - INFO - lr: 2.9994e-04 gnorm: 1.03 [ 0:27:08<22:51:41] +[titan] 2025-07-09 14:02:07,679 - root - INFO - step: 1945 loss: 19.2739 memory: 44.58GiB(31.99%) tps: 82,700 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 3.2097 global_avg_mtp_loss: 16.0641 +[titan] 2025-07-09 14:02:07,679 - root - INFO - lr: 2.9994e-04 gnorm: 1.04 [ 0:27:12<22:51:25] +[titan] 2025-07-09 14:02:10,791 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:02:11,578 - root - INFO - step: 1950 loss: 19.5413 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 3.2751 global_avg_mtp_loss: 16.2662 +[titan] 2025-07-09 14:02:11,578 - root - INFO - lr: 2.9994e-04 gnorm: 1.08 [ 0:27:16<22:51:05] +[titan] 2025-07-09 14:02:15,480 - root - INFO - step: 1955 loss: 19.5381 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 3.2680 global_avg_mtp_loss: 16.2702 +[titan] 2025-07-09 14:02:15,480 - root - INFO - lr: 2.9994e-04 gnorm: 0.96 [ 0:27:19<22:50:47] +[titan] 2025-07-09 14:02:19,406 - root - INFO - step: 1960 loss: 19.3955 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 3.2459 global_avg_mtp_loss: 16.1496 +[titan] 2025-07-09 14:02:19,406 - root - INFO - lr: 2.9994e-04 gnorm: 0.96 [ 0:27:23<22:50:29] +[titan] 2025-07-09 14:02:23,288 - root - INFO - step: 1965 loss: 19.5217 memory: 44.58GiB(31.99%) tps: 84,428 tflops: 291.37 mfu: 29.46% global_avg_ntp_loss: 3.2591 global_avg_mtp_loss: 16.2627 +[titan] 2025-07-09 14:02:23,288 - root - INFO - lr: 2.9994e-04 gnorm: 1.07 [ 0:27:27<22:50:09] +[titan] 2025-07-09 14:02:27,173 - root - INFO - step: 1970 loss: 19.4951 memory: 44.58GiB(31.99%) tps: 84,352 tflops: 291.11 mfu: 29.43% global_avg_ntp_loss: 3.2690 global_avg_mtp_loss: 16.2261 +[titan] 2025-07-09 14:02:27,173 - root - INFO - lr: 2.9994e-04 gnorm: 1.19 [ 0:27:31<22:49:50] +[titan] 2025-07-09 14:02:31,086 - root - INFO - step: 1975 loss: 19.2700 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 3.2206 global_avg_mtp_loss: 16.0493 +[titan] 2025-07-09 14:02:31,086 - root - INFO - lr: 2.9994e-04 gnorm: 1.27 [ 0:27:35<22:49:32] +[titan] 2025-07-09 14:02:35,010 - root - INFO - step: 1980 loss: 19.4141 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 3.2473 global_avg_mtp_loss: 16.1669 +[titan] 2025-07-09 14:02:35,010 - root - INFO - lr: 2.9993e-04 gnorm: 1.07 [ 0:27:39<22:49:14] +[titan] 2025-07-09 14:02:38,923 - root - INFO - step: 1985 loss: 19.4334 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 3.2451 global_avg_mtp_loss: 16.1882 +[titan] 2025-07-09 14:02:38,923 - root - INFO - lr: 2.9993e-04 gnorm: 0.90 [ 0:27:43<22:48:56] +[titan] 2025-07-09 14:02:42,868 - root - INFO - step: 1990 loss: 19.5005 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 3.2598 global_avg_mtp_loss: 16.2406 +[titan] 2025-07-09 14:02:42,868 - root - INFO - lr: 2.9993e-04 gnorm: 1.13 [ 0:27:47<22:48:40] +[titan] 2025-07-09 14:02:46,788 - root - INFO - step: 1995 loss: 19.2693 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 3.2320 global_avg_mtp_loss: 16.0373 +[titan] 2025-07-09 14:02:46,788 - root - INFO - lr: 2.9993e-04 gnorm: 1.17 [ 0:27:51<22:48:22] +[titan] 2025-07-09 14:02:49,946 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:02:50,729 - root - INFO - step: 2000 loss: 19.3593 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 3.2289 global_avg_mtp_loss: 16.1304 +[titan] 2025-07-09 14:02:50,729 - root - INFO - lr: 2.9993e-04 gnorm: 0.96 [ 0:27:55<22:48:06] +[titan] 2025-07-09 14:02:54,624 - root - INFO - step: 2005 loss: 19.3014 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 3.2135 global_avg_mtp_loss: 16.0880 +[titan] 2025-07-09 14:02:54,624 - root - INFO - lr: 2.9993e-04 gnorm: 1.02 [ 0:27:59<22:47:48] +[titan] 2025-07-09 14:02:58,548 - root - INFO - step: 2010 loss: 19.1654 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 3.2049 global_avg_mtp_loss: 15.9605 +[titan] 2025-07-09 14:02:58,549 - root - INFO - lr: 2.9993e-04 gnorm: 1.01 [ 0:28:03<22:47:31] +[titan] 2025-07-09 14:03:02,481 - root - INFO - step: 2015 loss: 19.3765 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 3.2312 global_avg_mtp_loss: 16.1453 +[titan] 2025-07-09 14:03:02,481 - root - INFO - lr: 2.9993e-04 gnorm: 0.94 [ 0:28:06<22:47:14] +[titan] 2025-07-09 14:03:06,398 - root - INFO - step: 2020 loss: 19.3360 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 3.2148 global_avg_mtp_loss: 16.1212 +[titan] 2025-07-09 14:03:06,398 - root - INFO - lr: 2.9993e-04 gnorm: 0.99 [ 0:28:10<22:46:57] +[titan] 2025-07-09 14:03:10,355 - root - INFO - step: 2025 loss: 19.4027 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 3.2302 global_avg_mtp_loss: 16.1725 +[titan] 2025-07-09 14:03:10,355 - root - INFO - lr: 2.9993e-04 gnorm: 1.03 [ 0:28:14<22:46:41] +[titan] 2025-07-09 14:03:14,296 - root - INFO - step: 2030 loss: 19.1778 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 3.2125 global_avg_mtp_loss: 15.9653 +[titan] 2025-07-09 14:03:14,297 - root - INFO - lr: 2.9993e-04 gnorm: 1.00 [ 0:28:18<22:46:25] +[titan] 2025-07-09 14:03:18,203 - root - INFO - step: 2035 loss: 19.4461 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 3.2494 global_avg_mtp_loss: 16.1967 +[titan] 2025-07-09 14:03:18,203 - root - INFO - lr: 2.9993e-04 gnorm: 1.04 [ 0:28:22<22:46:08] +[titan] 2025-07-09 14:03:22,106 - root - INFO - step: 2040 loss: 19.0736 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 3.1832 global_avg_mtp_loss: 15.8905 +[titan] 2025-07-09 14:03:22,106 - root - INFO - lr: 2.9993e-04 gnorm: 1.07 [ 0:28:26<22:45:50] +[titan] 2025-07-09 14:03:26,087 - root - INFO - step: 2045 loss: 19.3422 memory: 44.58GiB(31.99%) tps: 82,304 tflops: 284.04 mfu: 28.72% global_avg_ntp_loss: 3.2204 global_avg_mtp_loss: 16.1218 +[titan] 2025-07-09 14:03:26,088 - root - INFO - lr: 2.9993e-04 gnorm: 1.03 [ 0:28:30<22:45:36] +[titan] 2025-07-09 14:03:28,584 - root - INFO - Dumping profiler traces at step 2048 +[titan] 2025-07-09 14:03:28,615 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:03:29,400 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:03:30,183 - root - INFO - step: 2050 loss: 19.2037 memory: 44.58GiB(31.99%) tps: 80,007 tflops: 276.12 mfu: 27.92% global_avg_ntp_loss: 3.1953 global_avg_mtp_loss: 16.0084 +[titan] 2025-07-09 14:03:30,184 - root - INFO - lr: 2.9992e-04 gnorm: 0.95 [ 0:28:34<22:45:28] +[titan] 2025-07-09 14:03:34,089 - root - INFO - step: 2055 loss: 19.2976 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 3.2227 global_avg_mtp_loss: 16.0749 +[titan] 2025-07-09 14:03:34,089 - root - INFO - lr: 2.9992e-04 gnorm: 0.98 [ 0:28:38<22:45:11] +[titan] 2025-07-09 14:03:38,044 - root - INFO - step: 2060 loss: 19.1426 memory: 44.58GiB(31.99%) tps: 82,850 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 3.1690 global_avg_mtp_loss: 15.9736 +[titan] 2025-07-09 14:03:38,044 - root - INFO - lr: 2.9992e-04 gnorm: 0.91 [ 0:28:42<22:44:56] +[titan] 2025-07-09 14:03:41,937 - root - INFO - step: 2065 loss: 19.5385 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 3.2608 global_avg_mtp_loss: 16.2777 +[titan] 2025-07-09 14:03:41,937 - root - INFO - lr: 2.9992e-04 gnorm: 1.00 [ 0:28:46<22:44:38] +[titan] 2025-07-09 14:03:45,846 - root - INFO - step: 2070 loss: 19.4863 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 3.2603 global_avg_mtp_loss: 16.2260 +[titan] 2025-07-09 14:03:45,846 - root - INFO - lr: 2.9992e-04 gnorm: 0.94 [ 0:28:50<22:44:21] +[titan] 2025-07-09 14:03:49,781 - root - INFO - step: 2075 loss: 19.2331 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 3.1977 global_avg_mtp_loss: 16.0354 +[titan] 2025-07-09 14:03:49,781 - root - INFO - lr: 2.9992e-04 gnorm: 1.03 [ 0:28:54<22:44:05] +[titan] 2025-07-09 14:03:53,728 - root - INFO - step: 2080 loss: 19.3355 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 3.2323 global_avg_mtp_loss: 16.1032 +[titan] 2025-07-09 14:03:53,728 - root - INFO - lr: 2.9992e-04 gnorm: 1.09 [ 0:28:58<22:43:50] +[titan] 2025-07-09 14:03:57,626 - root - INFO - step: 2085 loss: 19.2700 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 3.2152 global_avg_mtp_loss: 16.0548 +[titan] 2025-07-09 14:03:57,627 - root - INFO - lr: 2.9992e-04 gnorm: 1.00 [ 0:29:02<22:43:33] +[titan] 2025-07-09 14:04:01,538 - root - INFO - step: 2090 loss: 19.1105 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 3.1765 global_avg_mtp_loss: 15.9340 +[titan] 2025-07-09 14:04:01,538 - root - INFO - lr: 2.9992e-04 gnorm: 0.94 [ 0:29:06<22:43:16] +[titan] 2025-07-09 14:04:05,435 - root - INFO - step: 2095 loss: 19.2306 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 3.2025 global_avg_mtp_loss: 16.0281 +[titan] 2025-07-09 14:04:05,435 - root - INFO - lr: 2.9992e-04 gnorm: 1.08 [ 0:29:09<22:42:59] +[titan] 2025-07-09 14:04:08,543 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:04:09,333 - root - INFO - step: 2100 loss: 19.1041 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 3.1780 global_avg_mtp_loss: 15.9262 +[titan] 2025-07-09 14:04:09,333 - root - INFO - lr: 2.9992e-04 gnorm: 0.93 [ 0:29:13<22:42:41] +[titan] 2025-07-09 14:04:13,275 - root - INFO - step: 2105 loss: 19.2677 memory: 44.58GiB(31.99%) tps: 83,122 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 3.2206 global_avg_mtp_loss: 16.0471 +[titan] 2025-07-09 14:04:13,276 - root - INFO - lr: 2.9992e-04 gnorm: 1.06 [ 0:29:17<22:42:26] +[titan] 2025-07-09 14:04:17,203 - root - INFO - step: 2110 loss: 19.1620 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 3.1833 global_avg_mtp_loss: 15.9787 +[titan] 2025-07-09 14:04:17,203 - root - INFO - lr: 2.9992e-04 gnorm: 0.99 [ 0:29:21<22:42:11] +[titan] 2025-07-09 14:04:21,119 - root - INFO - step: 2115 loss: 19.1705 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 3.1836 global_avg_mtp_loss: 15.9869 +[titan] 2025-07-09 14:04:21,120 - root - INFO - lr: 2.9992e-04 gnorm: 0.98 [ 0:29:25<22:41:55] +[titan] 2025-07-09 14:04:25,021 - root - INFO - step: 2120 loss: 19.2591 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 3.1999 global_avg_mtp_loss: 16.0591 +[titan] 2025-07-09 14:04:25,021 - root - INFO - lr: 2.9991e-04 gnorm: 1.10 [ 0:29:29<22:41:38] +[titan] 2025-07-09 14:04:28,967 - root - INFO - step: 2125 loss: 19.3589 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 3.2386 global_avg_mtp_loss: 16.1203 +[titan] 2025-07-09 14:04:28,968 - root - INFO - lr: 2.9991e-04 gnorm: 0.93 [ 0:29:33<22:41:23] +[titan] 2025-07-09 14:04:32,909 - root - INFO - step: 2130 loss: 19.2509 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 3.1941 global_avg_mtp_loss: 16.0569 +[titan] 2025-07-09 14:04:32,909 - root - INFO - lr: 2.9991e-04 gnorm: 1.02 [ 0:29:37<22:41:08] +[titan] 2025-07-09 14:04:36,815 - root - INFO - step: 2135 loss: 19.2676 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 3.2108 global_avg_mtp_loss: 16.0568 +[titan] 2025-07-09 14:04:36,815 - root - INFO - lr: 2.9991e-04 gnorm: 1.00 [ 0:29:41<22:40:52] +[titan] 2025-07-09 14:04:40,733 - root - INFO - step: 2140 loss: 19.1375 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 3.1659 global_avg_mtp_loss: 15.9716 +[titan] 2025-07-09 14:04:40,734 - root - INFO - lr: 2.9991e-04 gnorm: 0.96 [ 0:29:45<22:40:36] +[titan] 2025-07-09 14:04:44,635 - root - INFO - step: 2145 loss: 19.2654 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 3.2042 global_avg_mtp_loss: 16.0612 +[titan] 2025-07-09 14:04:44,635 - root - INFO - lr: 2.9991e-04 gnorm: 0.96 [ 0:29:49<22:40:20] +[titan] 2025-07-09 14:04:47,745 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:04:48,533 - root - INFO - step: 2150 loss: 19.2199 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 3.2007 global_avg_mtp_loss: 16.0192 +[titan] 2025-07-09 14:04:48,533 - root - INFO - lr: 2.9991e-04 gnorm: 1.00 [ 0:29:53<22:40:03] +[titan] 2025-07-09 14:04:52,432 - root - INFO - step: 2155 loss: 19.5747 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 3.2826 global_avg_mtp_loss: 16.2921 +[titan] 2025-07-09 14:04:52,432 - root - INFO - lr: 2.9991e-04 gnorm: 1.03 [ 0:29:56<22:39:47] +[titan] 2025-07-09 14:04:56,330 - root - INFO - step: 2160 loss: 19.0770 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 3.1538 global_avg_mtp_loss: 15.9233 +[titan] 2025-07-09 14:04:56,330 - root - INFO - lr: 2.9991e-04 gnorm: 1.07 [ 0:30:00<22:39:30] +[titan] 2025-07-09 14:05:00,248 - root - INFO - step: 2165 loss: 19.1320 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 3.1782 global_avg_mtp_loss: 15.9537 +[titan] 2025-07-09 14:05:00,248 - root - INFO - lr: 2.9991e-04 gnorm: 0.99 [ 0:30:04<22:39:15] +[titan] 2025-07-09 14:05:04,158 - root - INFO - step: 2170 loss: 19.2352 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 3.2084 global_avg_mtp_loss: 16.0268 +[titan] 2025-07-09 14:05:04,158 - root - INFO - lr: 2.9991e-04 gnorm: 0.90 [ 0:30:08<22:38:59] +[titan] 2025-07-09 14:05:08,071 - root - INFO - step: 2175 loss: 19.1851 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 3.1839 global_avg_mtp_loss: 16.0012 +[titan] 2025-07-09 14:05:08,072 - root - INFO - lr: 2.9991e-04 gnorm: 0.93 [ 0:30:12<22:38:43] +[titan] 2025-07-09 14:05:11,985 - root - INFO - step: 2180 loss: 19.1034 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 3.1620 global_avg_mtp_loss: 15.9414 +[titan] 2025-07-09 14:05:11,985 - root - INFO - lr: 2.9991e-04 gnorm: 0.91 [ 0:30:16<22:38:28] +[titan] 2025-07-09 14:05:15,933 - root - INFO - step: 2185 loss: 19.2298 memory: 44.58GiB(31.99%) tps: 83,001 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 3.1739 global_avg_mtp_loss: 16.0559 +[titan] 2025-07-09 14:05:15,933 - root - INFO - lr: 2.9990e-04 gnorm: 1.07 [ 0:30:20<22:38:14] +[titan] 2025-07-09 14:05:19,832 - root - INFO - step: 2190 loss: 19.0765 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 3.1493 global_avg_mtp_loss: 15.9273 +[titan] 2025-07-09 14:05:19,832 - root - INFO - lr: 2.9990e-04 gnorm: 1.01 [ 0:30:24<22:37:58] +[titan] 2025-07-09 14:05:23,742 - root - INFO - step: 2195 loss: 19.2554 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 3.1854 global_avg_mtp_loss: 16.0700 +[titan] 2025-07-09 14:05:23,742 - root - INFO - lr: 2.9990e-04 gnorm: 1.09 [ 0:30:28<22:37:42] +[titan] 2025-07-09 14:05:26,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:05:27,678 - root - INFO - step: 2200 loss: 19.0370 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 3.1434 global_avg_mtp_loss: 15.8936 +[titan] 2025-07-09 14:05:27,679 - root - INFO - lr: 2.9990e-04 gnorm: 0.99 [ 0:30:32<22:37:28] +[titan] 2025-07-09 14:05:31,613 - root - INFO - step: 2205 loss: 19.2726 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 3.1949 global_avg_mtp_loss: 16.0777 +[titan] 2025-07-09 14:05:31,614 - root - INFO - lr: 2.9990e-04 gnorm: 0.92 [ 0:30:36<22:37:13] +[titan] 2025-07-09 14:05:35,576 - root - INFO - step: 2210 loss: 19.2766 memory: 44.58GiB(31.99%) tps: 82,699 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 3.1996 global_avg_mtp_loss: 16.0770 +[titan] 2025-07-09 14:05:35,576 - root - INFO - lr: 2.9990e-04 gnorm: 0.92 [ 0:30:40<22:37:00] +[titan] 2025-07-09 14:05:39,486 - root - INFO - step: 2215 loss: 18.8993 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 3.1192 global_avg_mtp_loss: 15.7802 +[titan] 2025-07-09 14:05:39,486 - root - INFO - lr: 2.9990e-04 gnorm: 0.97 [ 0:30:43<22:36:45] +[titan] 2025-07-09 14:05:43,398 - root - INFO - step: 2220 loss: 19.2728 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 3.2196 global_avg_mtp_loss: 16.0531 +[titan] 2025-07-09 14:05:43,398 - root - INFO - lr: 2.9990e-04 gnorm: 1.16 [ 0:30:47<22:36:30] +[titan] 2025-07-09 14:05:47,312 - root - INFO - step: 2225 loss: 19.1153 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 3.1459 global_avg_mtp_loss: 15.9694 +[titan] 2025-07-09 14:05:47,312 - root - INFO - lr: 2.9990e-04 gnorm: 1.00 [ 0:30:51<22:36:15] +[titan] 2025-07-09 14:05:51,213 - root - INFO - step: 2230 loss: 19.1507 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 3.1676 global_avg_mtp_loss: 15.9831 +[titan] 2025-07-09 14:05:51,213 - root - INFO - lr: 2.9990e-04 gnorm: 1.06 [ 0:30:55<22:35:59] +[titan] 2025-07-09 14:05:55,115 - root - INFO - step: 2235 loss: 18.9371 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 3.1512 global_avg_mtp_loss: 15.7859 +[titan] 2025-07-09 14:05:55,116 - root - INFO - lr: 2.9990e-04 gnorm: 1.22 [ 0:30:59<22:35:44] +[titan] 2025-07-09 14:05:59,043 - root - INFO - step: 2240 loss: 19.1514 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 3.1730 global_avg_mtp_loss: 15.9784 +[titan] 2025-07-09 14:05:59,043 - root - INFO - lr: 2.9990e-04 gnorm: 1.00 [ 0:31:03<22:35:29] +[titan] 2025-07-09 14:06:02,966 - root - INFO - step: 2245 loss: 19.1697 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 3.1847 global_avg_mtp_loss: 15.9849 +[titan] 2025-07-09 14:06:02,966 - root - INFO - lr: 2.9989e-04 gnorm: 1.01 [ 0:31:07<22:35:15] +[titan] 2025-07-09 14:06:06,094 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:06:06,879 - root - INFO - step: 2250 loss: 19.1088 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 3.1872 global_avg_mtp_loss: 15.9216 +[titan] 2025-07-09 14:06:06,879 - root - INFO - lr: 2.9989e-04 gnorm: 1.14 [ 0:31:11<22:35:00] +[titan] 2025-07-09 14:06:10,822 - root - INFO - step: 2255 loss: 19.3551 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 3.2226 global_avg_mtp_loss: 16.1325 +[titan] 2025-07-09 14:06:10,822 - root - INFO - lr: 2.9989e-04 gnorm: 0.97 [ 0:31:15<22:34:46] +[titan] 2025-07-09 14:06:14,769 - root - INFO - step: 2260 loss: 19.0886 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 3.1697 global_avg_mtp_loss: 15.9189 +[titan] 2025-07-09 14:06:14,769 - root - INFO - lr: 2.9989e-04 gnorm: 1.04 [ 0:31:19<22:34:33] +[titan] 2025-07-09 14:06:18,669 - root - INFO - step: 2265 loss: 19.2100 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 3.1923 global_avg_mtp_loss: 16.0177 +[titan] 2025-07-09 14:06:18,669 - root - INFO - lr: 2.9989e-04 gnorm: 1.07 [ 0:31:23<22:34:18] +[titan] 2025-07-09 14:06:22,595 - root - INFO - step: 2270 loss: 19.0188 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 3.1437 global_avg_mtp_loss: 15.8751 +[titan] 2025-07-09 14:06:22,596 - root - INFO - lr: 2.9989e-04 gnorm: 0.92 [ 0:31:27<22:34:04] +[titan] 2025-07-09 14:06:26,487 - root - INFO - step: 2275 loss: 19.0396 memory: 44.58GiB(31.99%) tps: 84,204 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 3.1416 global_avg_mtp_loss: 15.8981 +[titan] 2025-07-09 14:06:26,488 - root - INFO - lr: 2.9989e-04 gnorm: 1.03 [ 0:31:30<22:33:48] +[titan] 2025-07-09 14:06:30,390 - root - INFO - step: 2280 loss: 18.8042 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 3.0911 global_avg_mtp_loss: 15.7130 +[titan] 2025-07-09 14:06:30,391 - root - INFO - lr: 2.9989e-04 gnorm: 0.94 [ 0:31:34<22:33:33] +[titan] 2025-07-09 14:06:34,297 - root - INFO - step: 2285 loss: 18.9140 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 3.1302 global_avg_mtp_loss: 15.7838 +[titan] 2025-07-09 14:06:34,297 - root - INFO - lr: 2.9989e-04 gnorm: 1.03 [ 0:31:38<22:33:18] +[titan] 2025-07-09 14:06:38,195 - root - INFO - step: 2290 loss: 19.0094 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 3.1476 global_avg_mtp_loss: 15.8618 +[titan] 2025-07-09 14:06:38,195 - root - INFO - lr: 2.9989e-04 gnorm: 0.93 [ 0:31:42<22:33:03] +[titan] 2025-07-09 14:06:42,108 - root - INFO - step: 2295 loss: 19.1104 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 3.1697 global_avg_mtp_loss: 15.9407 +[titan] 2025-07-09 14:06:42,109 - root - INFO - lr: 2.9989e-04 gnorm: 0.99 [ 0:31:46<22:32:49] +[titan] 2025-07-09 14:06:45,243 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:06:46,031 - root - INFO - step: 2300 loss: 18.7709 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 3.0984 global_avg_mtp_loss: 15.6724 +[titan] 2025-07-09 14:06:46,032 - root - INFO - lr: 2.9988e-04 gnorm: 1.06 [ 0:31:50<22:32:35] +[titan] 2025-07-09 14:06:49,944 - root - INFO - step: 2305 loss: 18.9063 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 3.1158 global_avg_mtp_loss: 15.7905 +[titan] 2025-07-09 14:06:49,945 - root - INFO - lr: 2.9988e-04 gnorm: 1.01 [ 0:31:54<22:32:20] +[titan] 2025-07-09 14:06:53,848 - root - INFO - step: 2310 loss: 18.9422 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 3.1380 global_avg_mtp_loss: 15.8042 +[titan] 2025-07-09 14:06:53,848 - root - INFO - lr: 2.9988e-04 gnorm: 0.99 [ 0:31:58<22:32:06] +[titan] 2025-07-09 14:06:57,760 - root - INFO - step: 2315 loss: 19.1277 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 3.1768 global_avg_mtp_loss: 15.9509 +[titan] 2025-07-09 14:06:57,760 - root - INFO - lr: 2.9988e-04 gnorm: 1.05 [ 0:32:02<22:31:51] +[titan] 2025-07-09 14:07:01,649 - root - INFO - step: 2320 loss: 19.0773 memory: 44.58GiB(31.99%) tps: 84,265 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 3.1472 global_avg_mtp_loss: 15.9301 +[titan] 2025-07-09 14:07:01,650 - root - INFO - lr: 2.9988e-04 gnorm: 0.98 [ 0:32:06<22:31:36] +[titan] 2025-07-09 14:07:05,556 - root - INFO - step: 2325 loss: 18.9549 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 3.1295 global_avg_mtp_loss: 15.8254 +[titan] 2025-07-09 14:07:05,556 - root - INFO - lr: 2.9988e-04 gnorm: 0.93 [ 0:32:10<22:31:22] +[titan] 2025-07-09 14:07:09,465 - root - INFO - step: 2330 loss: 19.0294 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 3.1383 global_avg_mtp_loss: 15.8911 +[titan] 2025-07-09 14:07:09,466 - root - INFO - lr: 2.9988e-04 gnorm: 1.03 [ 0:32:13<22:31:07] +[titan] 2025-07-09 14:07:13,429 - root - INFO - step: 2335 loss: 19.0445 memory: 44.58GiB(31.99%) tps: 82,676 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 3.1452 global_avg_mtp_loss: 15.8993 +[titan] 2025-07-09 14:07:13,429 - root - INFO - lr: 2.9988e-04 gnorm: 1.18 [ 0:32:17<22:30:55] +[titan] 2025-07-09 14:07:17,357 - root - INFO - step: 2340 loss: 18.9716 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 3.1434 global_avg_mtp_loss: 15.8283 +[titan] 2025-07-09 14:07:17,357 - root - INFO - lr: 2.9988e-04 gnorm: 1.59 [ 0:32:21<22:30:42] +[titan] 2025-07-09 14:07:21,268 - root - INFO - step: 2345 loss: 18.6957 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 3.0700 global_avg_mtp_loss: 15.6257 +[titan] 2025-07-09 14:07:21,268 - root - INFO - lr: 2.9988e-04 gnorm: 0.94 [ 0:32:25<22:30:28] +[titan] 2025-07-09 14:07:24,380 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:07:25,166 - root - INFO - step: 2350 loss: 19.2765 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 3.1994 global_avg_mtp_loss: 16.0771 +[titan] 2025-07-09 14:07:25,166 - root - INFO - lr: 2.9988e-04 gnorm: 1.00 [ 0:32:29<22:30:13] +[titan] 2025-07-09 14:07:29,078 - root - INFO - step: 2355 loss: 18.7969 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 3.0939 global_avg_mtp_loss: 15.7029 +[titan] 2025-07-09 14:07:29,078 - root - INFO - lr: 2.9988e-04 gnorm: 1.09 [ 0:32:33<22:29:59] +[titan] 2025-07-09 14:07:32,983 - root - INFO - step: 2360 loss: 18.9877 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 3.1270 global_avg_mtp_loss: 15.8608 +[titan] 2025-07-09 14:07:32,983 - root - INFO - lr: 2.9987e-04 gnorm: 0.96 [ 0:32:37<22:29:45] +[titan] 2025-07-09 14:07:36,919 - root - INFO - step: 2365 loss: 19.0008 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 3.1218 global_avg_mtp_loss: 15.8789 +[titan] 2025-07-09 14:07:36,919 - root - INFO - lr: 2.9987e-04 gnorm: 1.13 [ 0:32:41<22:29:32] +[titan] 2025-07-09 14:07:40,831 - root - INFO - step: 2370 loss: 19.1205 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 3.1522 global_avg_mtp_loss: 15.9683 +[titan] 2025-07-09 14:07:40,831 - root - INFO - lr: 2.9987e-04 gnorm: 0.90 [ 0:32:45<22:29:18] +[titan] 2025-07-09 14:07:44,802 - root - INFO - step: 2375 loss: 19.0600 memory: 44.58GiB(31.99%) tps: 82,521 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 3.1534 global_avg_mtp_loss: 15.9066 +[titan] 2025-07-09 14:07:44,802 - root - INFO - lr: 2.9987e-04 gnorm: 1.03 [ 0:32:49<22:29:07] +[titan] 2025-07-09 14:07:48,710 - root - INFO - step: 2380 loss: 18.8653 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 3.1038 global_avg_mtp_loss: 15.7615 +[titan] 2025-07-09 14:07:48,710 - root - INFO - lr: 2.9987e-04 gnorm: 0.98 [ 0:32:53<22:28:53] +[titan] 2025-07-09 14:07:52,632 - root - INFO - step: 2385 loss: 19.0326 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 3.1446 global_avg_mtp_loss: 15.8880 +[titan] 2025-07-09 14:07:52,632 - root - INFO - lr: 2.9987e-04 gnorm: 0.94 [ 0:32:57<22:28:40] +[titan] 2025-07-09 14:07:56,562 - root - INFO - step: 2390 loss: 19.0164 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 3.1411 global_avg_mtp_loss: 15.8754 +[titan] 2025-07-09 14:07:56,562 - root - INFO - lr: 2.9987e-04 gnorm: 1.10 [ 0:33:01<22:28:27] +[titan] 2025-07-09 14:08:00,482 - root - INFO - step: 2395 loss: 18.8691 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 3.0990 global_avg_mtp_loss: 15.7702 +[titan] 2025-07-09 14:08:00,483 - root - INFO - lr: 2.9987e-04 gnorm: 0.97 [ 0:33:04<22:28:14] +[titan] 2025-07-09 14:08:03,601 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:08:04,388 - root - INFO - step: 2400 loss: 19.0869 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 3.1597 global_avg_mtp_loss: 15.9272 +[titan] 2025-07-09 14:08:04,388 - root - INFO - lr: 2.9987e-04 gnorm: 0.94 [ 0:33:08<22:28:00] +[titan] 2025-07-09 14:08:08,293 - root - INFO - step: 2405 loss: 19.0623 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 3.1463 global_avg_mtp_loss: 15.9160 +[titan] 2025-07-09 14:08:08,293 - root - INFO - lr: 2.9987e-04 gnorm: 0.92 [ 0:33:12<22:27:46] +[titan] 2025-07-09 14:08:12,198 - root - INFO - step: 2410 loss: 18.8961 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 3.1053 global_avg_mtp_loss: 15.7908 +[titan] 2025-07-09 14:08:12,198 - root - INFO - lr: 2.9986e-04 gnorm: 1.10 [ 0:33:16<22:27:32] +[titan] 2025-07-09 14:08:16,115 - root - INFO - step: 2415 loss: 18.6858 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 3.0753 global_avg_mtp_loss: 15.6105 +[titan] 2025-07-09 14:08:16,116 - root - INFO - lr: 2.9986e-04 gnorm: 1.01 [ 0:33:20<22:27:19] +[titan] 2025-07-09 14:08:20,039 - root - INFO - step: 2420 loss: 19.1330 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 3.1672 global_avg_mtp_loss: 15.9658 +[titan] 2025-07-09 14:08:20,039 - root - INFO - lr: 2.9986e-04 gnorm: 0.98 [ 0:33:24<22:27:06] +[titan] 2025-07-09 14:08:23,960 - root - INFO - step: 2425 loss: 18.8680 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 3.1144 global_avg_mtp_loss: 15.7536 +[titan] 2025-07-09 14:08:23,960 - root - INFO - lr: 2.9986e-04 gnorm: 0.99 [ 0:33:28<22:26:53] +[titan] 2025-07-09 14:08:27,864 - root - INFO - step: 2430 loss: 18.9129 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 3.1150 global_avg_mtp_loss: 15.7979 +[titan] 2025-07-09 14:08:27,864 - root - INFO - lr: 2.9986e-04 gnorm: 1.00 [ 0:33:32<22:26:39] +[titan] 2025-07-09 14:08:31,746 - root - INFO - step: 2435 loss: 18.9262 memory: 44.58GiB(31.99%) tps: 84,417 tflops: 291.34 mfu: 29.46% global_avg_ntp_loss: 3.1217 global_avg_mtp_loss: 15.8045 +[titan] 2025-07-09 14:08:31,746 - root - INFO - lr: 2.9986e-04 gnorm: 0.96 [ 0:33:36<22:26:25] +[titan] 2025-07-09 14:08:35,661 - root - INFO - step: 2440 loss: 18.9754 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 3.1392 global_avg_mtp_loss: 15.8362 +[titan] 2025-07-09 14:08:35,662 - root - INFO - lr: 2.9986e-04 gnorm: 1.12 [ 0:33:40<22:26:12] +[titan] 2025-07-09 14:08:39,564 - root - INFO - step: 2445 loss: 18.8248 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 3.0979 global_avg_mtp_loss: 15.7269 +[titan] 2025-07-09 14:08:39,565 - root - INFO - lr: 2.9986e-04 gnorm: 1.04 [ 0:33:44<22:25:58] +[titan] 2025-07-09 14:08:42,686 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:08:43,481 - root - INFO - step: 2450 loss: 18.9069 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 3.1004 global_avg_mtp_loss: 15.8066 +[titan] 2025-07-09 14:08:43,482 - root - INFO - lr: 2.9986e-04 gnorm: 0.93 [ 0:33:47<22:25:45] +[titan] 2025-07-09 14:08:47,398 - root - INFO - step: 2455 loss: 19.0265 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 3.1463 global_avg_mtp_loss: 15.8802 +[titan] 2025-07-09 14:08:47,399 - root - INFO - lr: 2.9986e-04 gnorm: 0.97 [ 0:33:51<22:25:32] +[titan] 2025-07-09 14:08:51,330 - root - INFO - step: 2460 loss: 18.8885 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 3.1118 global_avg_mtp_loss: 15.7768 +[titan] 2025-07-09 14:08:51,330 - root - INFO - lr: 2.9985e-04 gnorm: 0.98 [ 0:33:55<22:25:20] +[titan] 2025-07-09 14:08:55,241 - root - INFO - step: 2465 loss: 18.8913 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 3.1085 global_avg_mtp_loss: 15.7827 +[titan] 2025-07-09 14:08:55,241 - root - INFO - lr: 2.9985e-04 gnorm: 0.89 [ 0:33:59<22:25:06] +[titan] 2025-07-09 14:08:59,171 - root - INFO - step: 2470 loss: 18.6748 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 3.0727 global_avg_mtp_loss: 15.6021 +[titan] 2025-07-09 14:08:59,171 - root - INFO - lr: 2.9985e-04 gnorm: 1.00 [ 0:34:03<22:24:54] +[titan] 2025-07-09 14:09:03,067 - root - INFO - step: 2475 loss: 18.9113 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 3.1107 global_avg_mtp_loss: 15.8007 +[titan] 2025-07-09 14:09:03,067 - root - INFO - lr: 2.9985e-04 gnorm: 1.02 [ 0:34:07<22:24:40] +[titan] 2025-07-09 14:09:06,973 - root - INFO - step: 2480 loss: 19.0984 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 3.1506 global_avg_mtp_loss: 15.9479 +[titan] 2025-07-09 14:09:06,973 - root - INFO - lr: 2.9985e-04 gnorm: 0.87 [ 0:34:11<22:24:27] +[titan] 2025-07-09 14:09:10,875 - root - INFO - step: 2485 loss: 18.7387 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 3.0820 global_avg_mtp_loss: 15.6567 +[titan] 2025-07-09 14:09:10,876 - root - INFO - lr: 2.9985e-04 gnorm: 1.00 [ 0:34:15<22:24:14] +[titan] 2025-07-09 14:09:14,798 - root - INFO - step: 2490 loss: 18.6648 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 3.0533 global_avg_mtp_loss: 15.6114 +[titan] 2025-07-09 14:09:14,798 - root - INFO - lr: 2.9985e-04 gnorm: 0.91 [ 0:34:19<22:24:01] +[titan] 2025-07-09 14:09:18,709 - root - INFO - step: 2495 loss: 19.0477 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 3.1304 global_avg_mtp_loss: 15.9173 +[titan] 2025-07-09 14:09:18,709 - root - INFO - lr: 2.9985e-04 gnorm: 1.00 [ 0:34:23<22:23:49] +[titan] 2025-07-09 14:09:21,812 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:09:22,601 - root - INFO - step: 2500 loss: 18.7031 memory: 44.58GiB(31.99%) tps: 84,202 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 3.0627 global_avg_mtp_loss: 15.6404 +[titan] 2025-07-09 14:09:22,601 - root - INFO - lr: 2.9985e-04 gnorm: 0.90 [ 0:34:27<22:23:35] +[titan] 2025-07-09 14:09:26,488 - root - INFO - step: 2505 loss: 18.8305 memory: 44.58GiB(31.99%) tps: 84,315 tflops: 290.98 mfu: 29.42% global_avg_ntp_loss: 3.1048 global_avg_mtp_loss: 15.7257 +[titan] 2025-07-09 14:09:26,488 - root - INFO - lr: 2.9985e-04 gnorm: 0.98 [ 0:34:30<22:23:21] +[titan] 2025-07-09 14:09:30,403 - root - INFO - step: 2510 loss: 18.7460 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 3.0788 global_avg_mtp_loss: 15.6672 +[titan] 2025-07-09 14:09:30,404 - root - INFO - lr: 2.9984e-04 gnorm: 1.02 [ 0:34:34<22:23:09] +[titan] 2025-07-09 14:09:34,346 - root - INFO - step: 2515 loss: 18.7825 memory: 44.58GiB(31.99%) tps: 83,126 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 3.1007 global_avg_mtp_loss: 15.6817 +[titan] 2025-07-09 14:09:34,346 - root - INFO - lr: 2.9984e-04 gnorm: 0.99 [ 0:34:38<22:22:57] +[titan] 2025-07-09 14:09:38,305 - root - INFO - step: 2520 loss: 18.9094 memory: 44.58GiB(31.99%) tps: 82,773 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 3.1217 global_avg_mtp_loss: 15.7876 +[titan] 2025-07-09 14:09:38,305 - root - INFO - lr: 2.9984e-04 gnorm: 1.11 [ 0:34:42<22:22:46] +[titan] 2025-07-09 14:09:42,223 - root - INFO - step: 2525 loss: 19.0322 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 3.1357 global_avg_mtp_loss: 15.8965 +[titan] 2025-07-09 14:09:42,223 - root - INFO - lr: 2.9984e-04 gnorm: 1.04 [ 0:34:46<22:22:34] +[titan] 2025-07-09 14:09:46,159 - root - INFO - step: 2530 loss: 18.6972 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 3.0733 global_avg_mtp_loss: 15.6238 +[titan] 2025-07-09 14:09:46,159 - root - INFO - lr: 2.9984e-04 gnorm: 1.00 [ 0:34:50<22:22:22] +[titan] 2025-07-09 14:09:50,066 - root - INFO - step: 2535 loss: 18.7543 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 3.0735 global_avg_mtp_loss: 15.6808 +[titan] 2025-07-09 14:09:50,066 - root - INFO - lr: 2.9984e-04 gnorm: 0.97 [ 0:34:54<22:22:09] +[titan] 2025-07-09 14:09:54,005 - root - INFO - step: 2540 loss: 19.0143 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 3.1355 global_avg_mtp_loss: 15.8788 +[titan] 2025-07-09 14:09:54,005 - root - INFO - lr: 2.9984e-04 gnorm: 0.96 [ 0:34:58<22:21:58] +[titan] 2025-07-09 14:09:57,943 - root - INFO - step: 2545 loss: 18.7568 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 3.0753 global_avg_mtp_loss: 15.6815 +[titan] 2025-07-09 14:09:57,944 - root - INFO - lr: 2.9984e-04 gnorm: 0.95 [ 0:35:02<22:21:46] +[titan] 2025-07-09 14:10:01,049 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:10:01,835 - root - INFO - step: 2550 loss: 18.9639 memory: 44.58GiB(31.99%) tps: 84,201 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 3.1282 global_avg_mtp_loss: 15.8356 +[titan] 2025-07-09 14:10:01,836 - root - INFO - lr: 2.9984e-04 gnorm: 1.06 [ 0:35:06<22:21:33] +[titan] 2025-07-09 14:10:05,743 - root - INFO - step: 2555 loss: 18.4222 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 3.0354 global_avg_mtp_loss: 15.3869 +[titan] 2025-07-09 14:10:05,743 - root - INFO - lr: 2.9984e-04 gnorm: 0.98 [ 0:35:10<22:21:20] +[titan] 2025-07-09 14:10:09,722 - root - INFO - step: 2560 loss: 18.8424 memory: 44.58GiB(31.99%) tps: 82,372 tflops: 284.28 mfu: 28.74% global_avg_ntp_loss: 3.0879 global_avg_mtp_loss: 15.7545 +[titan] 2025-07-09 14:10:09,722 - root - INFO - lr: 2.9983e-04 gnorm: 0.97 [ 0:35:14<22:21:10] +[titan] 2025-07-09 14:10:09,857 - root - INFO - Dumping profiler traces at step 2560 +[titan] 2025-07-09 14:10:09,889 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:10:13,853 - root - INFO - step: 2565 loss: 18.7026 memory: 44.58GiB(31.99%) tps: 79,322 tflops: 273.75 mfu: 27.68% global_avg_ntp_loss: 3.0803 global_avg_mtp_loss: 15.6223 +[titan] 2025-07-09 14:10:13,853 - root - INFO - lr: 2.9983e-04 gnorm: 0.98 [ 0:35:18<22:21:06] +[titan] 2025-07-09 14:10:17,752 - root - INFO - step: 2570 loss: 18.6202 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 3.0484 global_avg_mtp_loss: 15.5718 +[titan] 2025-07-09 14:10:17,752 - root - INFO - lr: 2.9983e-04 gnorm: 0.97 [ 0:35:22<22:20:53] +[titan] 2025-07-09 14:10:21,652 - root - INFO - step: 2575 loss: 18.8577 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 3.1005 global_avg_mtp_loss: 15.7572 +[titan] 2025-07-09 14:10:21,652 - root - INFO - lr: 2.9983e-04 gnorm: 1.01 [ 0:35:26<22:20:41] +[titan] 2025-07-09 14:10:25,549 - root - INFO - step: 2580 loss: 18.9265 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 3.1149 global_avg_mtp_loss: 15.8116 +[titan] 2025-07-09 14:10:25,549 - root - INFO - lr: 2.9983e-04 gnorm: 0.98 [ 0:35:30<22:20:28] +[titan] 2025-07-09 14:10:29,446 - root - INFO - step: 2585 loss: 18.7927 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 3.0821 global_avg_mtp_loss: 15.7106 +[titan] 2025-07-09 14:10:29,446 - root - INFO - lr: 2.9983e-04 gnorm: 1.18 [ 0:35:33<22:20:15] +[titan] 2025-07-09 14:10:33,352 - root - INFO - step: 2590 loss: 18.6563 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 3.0600 global_avg_mtp_loss: 15.5963 +[titan] 2025-07-09 14:10:33,352 - root - INFO - lr: 2.9983e-04 gnorm: 1.13 [ 0:35:37<22:20:02] +[titan] 2025-07-09 14:10:37,261 - root - INFO - step: 2595 loss: 18.9945 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 3.1373 global_avg_mtp_loss: 15.8572 +[titan] 2025-07-09 14:10:37,261 - root - INFO - lr: 2.9983e-04 gnorm: 1.05 [ 0:35:41<22:19:50] +[titan] 2025-07-09 14:10:40,398 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:10:41,190 - root - INFO - step: 2600 loss: 19.0116 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 3.1233 global_avg_mtp_loss: 15.8884 +[titan] 2025-07-09 14:10:41,191 - root - INFO - lr: 2.9983e-04 gnorm: 0.94 [ 0:35:45<22:19:39] +[titan] 2025-07-09 14:10:45,127 - root - INFO - step: 2605 loss: 18.8464 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 3.0859 global_avg_mtp_loss: 15.7605 +[titan] 2025-07-09 14:10:45,127 - root - INFO - lr: 2.9982e-04 gnorm: 1.02 [ 0:35:49<22:19:27] +[titan] 2025-07-09 14:10:49,027 - root - INFO - step: 2610 loss: 18.8040 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 3.0833 global_avg_mtp_loss: 15.7206 +[titan] 2025-07-09 14:10:49,027 - root - INFO - lr: 2.9982e-04 gnorm: 0.97 [ 0:35:53<22:19:15] +[titan] 2025-07-09 14:10:52,920 - root - INFO - step: 2615 loss: 18.6773 memory: 44.58GiB(31.99%) tps: 84,182 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 3.0767 global_avg_mtp_loss: 15.6006 +[titan] 2025-07-09 14:10:52,920 - root - INFO - lr: 2.9982e-04 gnorm: 1.01 [ 0:35:57<22:19:02] +[titan] 2025-07-09 14:10:56,829 - root - INFO - step: 2620 loss: 18.8215 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 3.0844 global_avg_mtp_loss: 15.7371 +[titan] 2025-07-09 14:10:56,829 - root - INFO - lr: 2.9982e-04 gnorm: 1.02 [ 0:36:01<22:18:50] +[titan] 2025-07-09 14:11:00,752 - root - INFO - step: 2625 loss: 18.6073 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 3.0586 global_avg_mtp_loss: 15.5487 +[titan] 2025-07-09 14:11:00,752 - root - INFO - lr: 2.9982e-04 gnorm: 1.13 [ 0:36:05<22:18:38] +[titan] 2025-07-09 14:11:04,697 - root - INFO - step: 2630 loss: 18.8966 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 3.1006 global_avg_mtp_loss: 15.7961 +[titan] 2025-07-09 14:11:04,697 - root - INFO - lr: 2.9982e-04 gnorm: 0.95 [ 0:36:09<22:18:27] +[titan] 2025-07-09 14:11:08,613 - root - INFO - step: 2635 loss: 19.1521 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 3.1568 global_avg_mtp_loss: 15.9953 +[titan] 2025-07-09 14:11:08,614 - root - INFO - lr: 2.9982e-04 gnorm: 1.17 [ 0:36:13<22:18:16] +[titan] 2025-07-09 14:11:12,516 - root - INFO - step: 2640 loss: 18.7599 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 3.0749 global_avg_mtp_loss: 15.6849 +[titan] 2025-07-09 14:11:12,516 - root - INFO - lr: 2.9982e-04 gnorm: 0.99 [ 0:36:16<22:18:03] +[titan] 2025-07-09 14:11:16,412 - root - INFO - step: 2645 loss: 19.0045 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 3.1586 global_avg_mtp_loss: 15.8460 +[titan] 2025-07-09 14:11:16,413 - root - INFO - lr: 2.9982e-04 gnorm: 0.89 [ 0:36:20<22:17:51] +[titan] 2025-07-09 14:11:19,532 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:11:20,318 - root - INFO - step: 2650 loss: 18.8721 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 3.1058 global_avg_mtp_loss: 15.7663 +[titan] 2025-07-09 14:11:20,318 - root - INFO - lr: 2.9981e-04 gnorm: 1.01 [ 0:36:24<22:17:39] +[titan] 2025-07-09 14:11:24,225 - root - INFO - step: 2655 loss: 18.4823 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 3.0266 global_avg_mtp_loss: 15.4557 +[titan] 2025-07-09 14:11:24,226 - root - INFO - lr: 2.9981e-04 gnorm: 1.92 [ 0:36:28<22:17:27] +[titan] 2025-07-09 14:11:28,205 - root - INFO - step: 2660 loss: 18.5989 memory: 44.58GiB(31.99%) tps: 82,352 tflops: 284.21 mfu: 28.74% global_avg_ntp_loss: 3.0467 global_avg_mtp_loss: 15.5522 +[titan] 2025-07-09 14:11:28,205 - root - INFO - lr: 2.9981e-04 gnorm: 1.01 [ 0:36:32<22:17:17] +[titan] 2025-07-09 14:11:32,161 - root - INFO - step: 2665 loss: 18.8072 memory: 44.58GiB(31.99%) tps: 82,828 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 3.0897 global_avg_mtp_loss: 15.7175 +[titan] 2025-07-09 14:11:32,161 - root - INFO - lr: 2.9981e-04 gnorm: 0.99 [ 0:36:36<22:17:07] +[titan] 2025-07-09 14:11:36,104 - root - INFO - step: 2670 loss: 18.5599 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 3.0243 global_avg_mtp_loss: 15.5356 +[titan] 2025-07-09 14:11:36,105 - root - INFO - lr: 2.9981e-04 gnorm: 1.01 [ 0:36:40<22:16:57] +[titan] 2025-07-09 14:11:40,016 - root - INFO - step: 2675 loss: 18.7060 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 3.0544 global_avg_mtp_loss: 15.6516 +[titan] 2025-07-09 14:11:40,016 - root - INFO - lr: 2.9981e-04 gnorm: 1.01 [ 0:36:44<22:16:45] +[titan] 2025-07-09 14:11:43,949 - root - INFO - step: 2680 loss: 18.6697 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 3.0566 global_avg_mtp_loss: 15.6131 +[titan] 2025-07-09 14:11:43,949 - root - INFO - lr: 2.9981e-04 gnorm: 0.92 [ 0:36:48<22:16:34] +[titan] 2025-07-09 14:11:47,872 - root - INFO - step: 2685 loss: 19.0206 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 3.1346 global_avg_mtp_loss: 15.8860 +[titan] 2025-07-09 14:11:47,873 - root - INFO - lr: 2.9981e-04 gnorm: 1.02 [ 0:36:52<22:16:23] +[titan] 2025-07-09 14:11:51,823 - root - INFO - step: 2690 loss: 18.7402 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 3.0727 global_avg_mtp_loss: 15.6675 +[titan] 2025-07-09 14:11:51,824 - root - INFO - lr: 2.9981e-04 gnorm: 1.18 [ 0:36:56<22:16:12] +[titan] 2025-07-09 14:11:55,777 - root - INFO - step: 2695 loss: 18.9529 memory: 44.58GiB(31.99%) tps: 82,900 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 3.1285 global_avg_mtp_loss: 15.8244 +[titan] 2025-07-09 14:11:55,777 - root - INFO - lr: 2.9980e-04 gnorm: 0.96 [ 0:37:00<22:16:02] +[titan] 2025-07-09 14:11:58,909 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:11:59,695 - root - INFO - step: 2700 loss: 18.7713 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 3.0719 global_avg_mtp_loss: 15.6994 +[titan] 2025-07-09 14:11:59,695 - root - INFO - lr: 2.9980e-04 gnorm: 0.92 [ 0:37:04<22:15:51] +[titan] 2025-07-09 14:12:03,659 - root - INFO - step: 2705 loss: 18.7023 memory: 44.58GiB(31.99%) tps: 82,667 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 3.0624 global_avg_mtp_loss: 15.6399 +[titan] 2025-07-09 14:12:03,659 - root - INFO - lr: 2.9980e-04 gnorm: 0.91 [ 0:37:08<22:15:41] +[titan] 2025-07-09 14:12:07,619 - root - INFO - step: 2710 loss: 18.8473 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 3.0972 global_avg_mtp_loss: 15.7501 +[titan] 2025-07-09 14:12:07,620 - root - INFO - lr: 2.9980e-04 gnorm: 1.03 [ 0:37:12<22:15:31] +[titan] 2025-07-09 14:12:11,526 - root - INFO - step: 2715 loss: 18.6447 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 3.0640 global_avg_mtp_loss: 15.5807 +[titan] 2025-07-09 14:12:11,526 - root - INFO - lr: 2.9980e-04 gnorm: 1.00 [ 0:37:15<22:15:20] +[titan] 2025-07-09 14:12:15,445 - root - INFO - step: 2720 loss: 18.7056 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 3.0693 global_avg_mtp_loss: 15.6363 +[titan] 2025-07-09 14:12:15,445 - root - INFO - lr: 2.9980e-04 gnorm: 0.97 [ 0:37:19<22:15:08] +[titan] 2025-07-09 14:12:19,359 - root - INFO - step: 2725 loss: 18.5651 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 3.0447 global_avg_mtp_loss: 15.5204 +[titan] 2025-07-09 14:12:19,359 - root - INFO - lr: 2.9980e-04 gnorm: 0.93 [ 0:37:23<22:14:57] +[titan] 2025-07-09 14:12:23,274 - root - INFO - step: 2730 loss: 18.7175 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 3.0706 global_avg_mtp_loss: 15.6470 +[titan] 2025-07-09 14:12:23,274 - root - INFO - lr: 2.9980e-04 gnorm: 0.92 [ 0:37:27<22:14:46] +[titan] 2025-07-09 14:12:27,166 - root - INFO - step: 2735 loss: 18.9085 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 3.1031 global_avg_mtp_loss: 15.8055 +[titan] 2025-07-09 14:12:27,166 - root - INFO - lr: 2.9980e-04 gnorm: 0.97 [ 0:37:31<22:14:33] +[titan] 2025-07-09 14:12:31,092 - root - INFO - step: 2740 loss: 18.7508 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 3.0718 global_avg_mtp_loss: 15.6790 +[titan] 2025-07-09 14:12:31,092 - root - INFO - lr: 2.9979e-04 gnorm: 0.94 [ 0:37:35<22:14:23] +[titan] 2025-07-09 14:12:34,994 - root - INFO - step: 2745 loss: 18.6535 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 3.0502 global_avg_mtp_loss: 15.6032 +[titan] 2025-07-09 14:12:34,994 - root - INFO - lr: 2.9979e-04 gnorm: 1.07 [ 0:37:39<22:14:11] +[titan] 2025-07-09 14:12:38,118 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:12:38,909 - root - INFO - step: 2750 loss: 18.5151 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 3.0159 global_avg_mtp_loss: 15.4991 +[titan] 2025-07-09 14:12:38,909 - root - INFO - lr: 2.9979e-04 gnorm: 0.92 [ 0:37:43<22:14:00] +[titan] 2025-07-09 14:12:42,820 - root - INFO - step: 2755 loss: 18.8006 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 3.0719 global_avg_mtp_loss: 15.7288 +[titan] 2025-07-09 14:12:42,820 - root - INFO - lr: 2.9979e-04 gnorm: 1.03 [ 0:37:47<22:13:48] +[titan] 2025-07-09 14:12:46,720 - root - INFO - step: 2760 loss: 18.7456 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 3.0557 global_avg_mtp_loss: 15.6899 +[titan] 2025-07-09 14:12:46,720 - root - INFO - lr: 2.9979e-04 gnorm: 0.92 [ 0:37:51<22:13:37] +[titan] 2025-07-09 14:12:50,624 - root - INFO - step: 2765 loss: 18.8174 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 3.0898 global_avg_mtp_loss: 15.7276 +[titan] 2025-07-09 14:12:50,624 - root - INFO - lr: 2.9979e-04 gnorm: 0.91 [ 0:37:55<22:13:25] +[titan] 2025-07-09 14:12:54,526 - root - INFO - step: 2770 loss: 18.7202 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 3.0618 global_avg_mtp_loss: 15.6584 +[titan] 2025-07-09 14:12:54,526 - root - INFO - lr: 2.9979e-04 gnorm: 0.98 [ 0:37:58<22:13:14] +[titan] 2025-07-09 14:12:58,444 - root - INFO - step: 2775 loss: 18.5572 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 3.0303 global_avg_mtp_loss: 15.5268 +[titan] 2025-07-09 14:12:58,444 - root - INFO - lr: 2.9979e-04 gnorm: 0.87 [ 0:38:02<22:13:03] +[titan] 2025-07-09 14:13:02,349 - root - INFO - step: 2780 loss: 18.5995 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 3.0410 global_avg_mtp_loss: 15.5585 +[titan] 2025-07-09 14:13:02,349 - root - INFO - lr: 2.9978e-04 gnorm: 0.98 [ 0:38:06<22:12:51] +[titan] 2025-07-09 14:13:06,321 - root - INFO - step: 2785 loss: 18.7067 memory: 44.58GiB(31.99%) tps: 82,517 tflops: 284.78 mfu: 28.79% global_avg_ntp_loss: 3.0580 global_avg_mtp_loss: 15.6487 +[titan] 2025-07-09 14:13:06,321 - root - INFO - lr: 2.9978e-04 gnorm: 0.91 [ 0:38:10<22:12:42] +[titan] 2025-07-09 14:13:10,262 - root - INFO - step: 2790 loss: 18.5262 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 3.0380 global_avg_mtp_loss: 15.4882 +[titan] 2025-07-09 14:13:10,262 - root - INFO - lr: 2.9978e-04 gnorm: 1.04 [ 0:38:14<22:12:32] +[titan] 2025-07-09 14:13:14,193 - root - INFO - step: 2795 loss: 18.5697 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 3.0250 global_avg_mtp_loss: 15.5448 +[titan] 2025-07-09 14:13:14,193 - root - INFO - lr: 2.9978e-04 gnorm: 0.97 [ 0:38:18<22:12:22] +[titan] 2025-07-09 14:13:17,344 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:13:18,151 - root - INFO - step: 2800 loss: 18.5008 memory: 44.58GiB(31.99%) tps: 82,799 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 3.0096 global_avg_mtp_loss: 15.4912 +[titan] 2025-07-09 14:13:18,151 - root - INFO - lr: 2.9978e-04 gnorm: 1.00 [ 0:38:22<22:12:12] +[titan] 2025-07-09 14:13:22,056 - root - INFO - step: 2805 loss: 18.6248 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 3.0328 global_avg_mtp_loss: 15.5920 +[titan] 2025-07-09 14:13:22,056 - root - INFO - lr: 2.9978e-04 gnorm: 1.02 [ 0:38:26<22:12:01] +[titan] 2025-07-09 14:13:25,956 - root - INFO - step: 2810 loss: 18.7886 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 3.0847 global_avg_mtp_loss: 15.7039 +[titan] 2025-07-09 14:13:25,956 - root - INFO - lr: 2.9978e-04 gnorm: 1.01 [ 0:38:30<22:11:49] +[titan] 2025-07-09 14:13:29,860 - root - INFO - step: 2815 loss: 18.7359 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 3.0706 global_avg_mtp_loss: 15.6652 +[titan] 2025-07-09 14:13:29,860 - root - INFO - lr: 2.9978e-04 gnorm: 0.96 [ 0:38:34<22:11:38] +[titan] 2025-07-09 14:13:33,772 - root - INFO - step: 2820 loss: 18.7813 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 3.0857 global_avg_mtp_loss: 15.6956 +[titan] 2025-07-09 14:13:33,772 - root - INFO - lr: 2.9977e-04 gnorm: 1.06 [ 0:38:38<22:11:27] +[titan] 2025-07-09 14:13:37,726 - root - INFO - step: 2825 loss: 18.6660 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 3.0580 global_avg_mtp_loss: 15.6079 +[titan] 2025-07-09 14:13:37,726 - root - INFO - lr: 2.9977e-04 gnorm: 0.96 [ 0:38:42<22:11:18] +[titan] 2025-07-09 14:13:41,637 - root - INFO - step: 2830 loss: 18.8034 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 3.0821 global_avg_mtp_loss: 15.7212 +[titan] 2025-07-09 14:13:41,638 - root - INFO - lr: 2.9977e-04 gnorm: 1.06 [ 0:38:46<22:11:07] +[titan] 2025-07-09 14:13:45,560 - root - INFO - step: 2835 loss: 18.6695 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 3.0555 global_avg_mtp_loss: 15.6140 +[titan] 2025-07-09 14:13:45,560 - root - INFO - lr: 2.9977e-04 gnorm: 0.88 [ 0:38:49<22:10:56] +[titan] 2025-07-09 14:13:49,465 - root - INFO - step: 2840 loss: 18.4930 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 3.0227 global_avg_mtp_loss: 15.4703 +[titan] 2025-07-09 14:13:49,465 - root - INFO - lr: 2.9977e-04 gnorm: 1.07 [ 0:38:53<22:10:45] +[titan] 2025-07-09 14:13:53,372 - root - INFO - step: 2845 loss: 18.5813 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 3.0361 global_avg_mtp_loss: 15.5452 +[titan] 2025-07-09 14:13:53,372 - root - INFO - lr: 2.9977e-04 gnorm: 0.99 [ 0:38:57<22:10:34] +[titan] 2025-07-09 14:13:56,491 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:13:57,276 - root - INFO - step: 2850 loss: 18.6480 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 3.0427 global_avg_mtp_loss: 15.6053 +[titan] 2025-07-09 14:13:57,277 - root - INFO - lr: 2.9977e-04 gnorm: 0.94 [ 0:39:01<22:10:23] +[titan] 2025-07-09 14:14:01,174 - root - INFO - step: 2855 loss: 18.9171 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 3.1129 global_avg_mtp_loss: 15.8042 +[titan] 2025-07-09 14:14:01,174 - root - INFO - lr: 2.9977e-04 gnorm: 1.01 [ 0:39:05<22:10:12] +[titan] 2025-07-09 14:14:05,095 - root - INFO - step: 2860 loss: 18.5172 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 3.0257 global_avg_mtp_loss: 15.4915 +[titan] 2025-07-09 14:14:05,095 - root - INFO - lr: 2.9976e-04 gnorm: 0.95 [ 0:39:09<22:10:01] +[titan] 2025-07-09 14:14:08,990 - root - INFO - step: 2865 loss: 18.6398 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 3.0423 global_avg_mtp_loss: 15.5975 +[titan] 2025-07-09 14:14:08,990 - root - INFO - lr: 2.9976e-04 gnorm: 0.87 [ 0:39:13<22:09:50] +[titan] 2025-07-09 14:14:12,954 - root - INFO - step: 2870 loss: 18.8189 memory: 44.58GiB(31.99%) tps: 82,669 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 3.0770 global_avg_mtp_loss: 15.7419 +[titan] 2025-07-09 14:14:12,955 - root - INFO - lr: 2.9976e-04 gnorm: 0.90 [ 0:39:17<22:09:41] +[titan] 2025-07-09 14:14:16,881 - root - INFO - step: 2875 loss: 18.6212 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 3.0403 global_avg_mtp_loss: 15.5809 +[titan] 2025-07-09 14:14:16,881 - root - INFO - lr: 2.9976e-04 gnorm: 0.90 [ 0:39:21<22:09:31] +[titan] 2025-07-09 14:14:20,765 - root - INFO - step: 2880 loss: 18.7340 memory: 44.58GiB(31.99%) tps: 84,366 tflops: 291.16 mfu: 29.44% global_avg_ntp_loss: 3.0710 global_avg_mtp_loss: 15.6630 +[titan] 2025-07-09 14:14:20,766 - root - INFO - lr: 2.9976e-04 gnorm: 1.07 [ 0:39:25<22:09:19] +[titan] 2025-07-09 14:14:24,689 - root - INFO - step: 2885 loss: 18.2482 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.9836 global_avg_mtp_loss: 15.2645 +[titan] 2025-07-09 14:14:24,689 - root - INFO - lr: 2.9976e-04 gnorm: 1.15 [ 0:39:29<22:09:09] +[titan] 2025-07-09 14:14:28,603 - root - INFO - step: 2890 loss: 18.8140 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 3.0785 global_avg_mtp_loss: 15.7355 +[titan] 2025-07-09 14:14:28,604 - root - INFO - lr: 2.9976e-04 gnorm: 1.04 [ 0:39:33<22:08:58] +[titan] 2025-07-09 14:14:32,550 - root - INFO - step: 2895 loss: 18.7198 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 3.0681 global_avg_mtp_loss: 15.6517 +[titan] 2025-07-09 14:14:32,550 - root - INFO - lr: 2.9976e-04 gnorm: 1.03 [ 0:39:36<22:08:49] +[titan] 2025-07-09 14:14:35,718 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:14:36,509 - root - INFO - step: 2900 loss: 18.3841 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 3.0064 global_avg_mtp_loss: 15.3777 +[titan] 2025-07-09 14:14:36,510 - root - INFO - lr: 2.9975e-04 gnorm: 1.06 [ 0:39:40<22:08:40] +[titan] 2025-07-09 14:14:40,412 - root - INFO - step: 2905 loss: 18.6191 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 3.0406 global_avg_mtp_loss: 15.5785 +[titan] 2025-07-09 14:14:40,413 - root - INFO - lr: 2.9975e-04 gnorm: 1.00 [ 0:39:44<22:08:29] +[titan] 2025-07-09 14:14:44,333 - root - INFO - step: 2910 loss: 18.6285 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 3.0418 global_avg_mtp_loss: 15.5867 +[titan] 2025-07-09 14:14:44,333 - root - INFO - lr: 2.9975e-04 gnorm: 0.97 [ 0:39:48<22:08:19] +[titan] 2025-07-09 14:14:48,244 - root - INFO - step: 2915 loss: 18.3896 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.9887 global_avg_mtp_loss: 15.4009 +[titan] 2025-07-09 14:14:48,244 - root - INFO - lr: 2.9975e-04 gnorm: 0.95 [ 0:39:52<22:08:08] +[titan] 2025-07-09 14:14:52,179 - root - INFO - step: 2920 loss: 18.8280 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 3.0812 global_avg_mtp_loss: 15.7468 +[titan] 2025-07-09 14:14:52,179 - root - INFO - lr: 2.9975e-04 gnorm: 1.00 [ 0:39:56<22:07:58] +[titan] 2025-07-09 14:14:56,098 - root - INFO - step: 2925 loss: 18.8400 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 3.0928 global_avg_mtp_loss: 15.7472 +[titan] 2025-07-09 14:14:56,098 - root - INFO - lr: 2.9975e-04 gnorm: 1.06 [ 0:40:00<22:07:48] +[titan] 2025-07-09 14:15:00,017 - root - INFO - step: 2930 loss: 18.6330 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 3.0421 global_avg_mtp_loss: 15.5909 +[titan] 2025-07-09 14:15:00,018 - root - INFO - lr: 2.9975e-04 gnorm: 1.00 [ 0:40:04<22:07:38] +[titan] 2025-07-09 14:15:03,927 - root - INFO - step: 2935 loss: 18.5443 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 3.0353 global_avg_mtp_loss: 15.5090 +[titan] 2025-07-09 14:15:03,928 - root - INFO - lr: 2.9975e-04 gnorm: 1.08 [ 0:40:08<22:07:27] +[titan] 2025-07-09 14:15:07,850 - root - INFO - step: 2940 loss: 18.4775 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 3.0208 global_avg_mtp_loss: 15.4567 +[titan] 2025-07-09 14:15:07,850 - root - INFO - lr: 2.9974e-04 gnorm: 0.92 [ 0:40:12<22:07:17] +[titan] 2025-07-09 14:15:11,754 - root - INFO - step: 2945 loss: 18.5620 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 3.0369 global_avg_mtp_loss: 15.5251 +[titan] 2025-07-09 14:15:11,754 - root - INFO - lr: 2.9974e-04 gnorm: 0.98 [ 0:40:16<22:07:07] +[titan] 2025-07-09 14:15:14,884 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:15:15,701 - root - INFO - step: 2950 loss: 18.6154 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 3.0434 global_avg_mtp_loss: 15.5719 +[titan] 2025-07-09 14:15:15,701 - root - INFO - lr: 2.9974e-04 gnorm: 0.97 [ 0:40:20<22:06:57] +[titan] 2025-07-09 14:15:19,588 - root - INFO - step: 2955 loss: 18.6604 memory: 44.58GiB(31.99%) tps: 84,298 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 3.0497 global_avg_mtp_loss: 15.6107 +[titan] 2025-07-09 14:15:19,588 - root - INFO - lr: 2.9974e-04 gnorm: 0.93 [ 0:40:24<22:06:46] +[titan] 2025-07-09 14:15:23,481 - root - INFO - step: 2960 loss: 18.6699 memory: 44.58GiB(31.99%) tps: 84,183 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 3.0455 global_avg_mtp_loss: 15.6244 +[titan] 2025-07-09 14:15:23,481 - root - INFO - lr: 2.9974e-04 gnorm: 0.99 [ 0:40:27<22:06:35] +[titan] 2025-07-09 14:15:27,400 - root - INFO - step: 2965 loss: 18.7858 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 3.0528 global_avg_mtp_loss: 15.7330 +[titan] 2025-07-09 14:15:27,400 - root - INFO - lr: 2.9974e-04 gnorm: 1.29 [ 0:40:31<22:06:25] +[titan] 2025-07-09 14:15:31,307 - root - INFO - step: 2970 loss: 18.6626 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 3.0554 global_avg_mtp_loss: 15.6072 +[titan] 2025-07-09 14:15:31,307 - root - INFO - lr: 2.9974e-04 gnorm: 1.01 [ 0:40:35<22:06:15] +[titan] 2025-07-09 14:15:35,224 - root - INFO - step: 2975 loss: 18.5415 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 3.0317 global_avg_mtp_loss: 15.5098 +[titan] 2025-07-09 14:15:35,224 - root - INFO - lr: 2.9973e-04 gnorm: 1.03 [ 0:40:39<22:06:05] +[titan] 2025-07-09 14:15:39,148 - root - INFO - step: 2980 loss: 18.5465 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 3.0338 global_avg_mtp_loss: 15.5127 +[titan] 2025-07-09 14:15:39,148 - root - INFO - lr: 2.9973e-04 gnorm: 0.99 [ 0:40:43<22:05:55] +[titan] 2025-07-09 14:15:43,036 - root - INFO - step: 2985 loss: 18.9173 memory: 44.58GiB(31.99%) tps: 84,290 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 3.0932 global_avg_mtp_loss: 15.8241 +[titan] 2025-07-09 14:15:43,036 - root - INFO - lr: 2.9973e-04 gnorm: 0.97 [ 0:40:47<22:05:44] +[titan] 2025-07-09 14:15:46,940 - root - INFO - step: 2990 loss: 18.9622 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 3.1104 global_avg_mtp_loss: 15.8518 +[titan] 2025-07-09 14:15:46,940 - root - INFO - lr: 2.9973e-04 gnorm: 2.50 [ 0:40:51<22:05:33] +[titan] 2025-07-09 14:15:50,832 - root - INFO - step: 2995 loss: 18.4997 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 3.0166 global_avg_mtp_loss: 15.4831 +[titan] 2025-07-09 14:15:50,832 - root - INFO - lr: 2.9973e-04 gnorm: 0.98 [ 0:40:55<22:05:22] +[titan] 2025-07-09 14:15:53,941 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:15:54,728 - root - INFO - step: 3000 loss: 18.6192 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 3.0348 global_avg_mtp_loss: 15.5844 +[titan] 2025-07-09 14:15:54,728 - root - INFO - lr: 2.9973e-04 gnorm: 0.99 [ 0:40:59<22:05:12] +[titan] 2025-07-09 14:15:58,634 - root - INFO - step: 3005 loss: 18.5888 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 3.0384 global_avg_mtp_loss: 15.5503 +[titan] 2025-07-09 14:15:58,635 - root - INFO - lr: 2.9973e-04 gnorm: 0.93 [ 0:41:03<22:05:01] +[titan] 2025-07-09 14:16:02,539 - root - INFO - step: 3010 loss: 18.6145 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 3.0322 global_avg_mtp_loss: 15.5823 +[titan] 2025-07-09 14:16:02,540 - root - INFO - lr: 2.9973e-04 gnorm: 0.93 [ 0:41:06<22:04:51] +[titan] 2025-07-09 14:16:06,444 - root - INFO - step: 3015 loss: 18.4876 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 3.0104 global_avg_mtp_loss: 15.4773 +[titan] 2025-07-09 14:16:06,445 - root - INFO - lr: 2.9972e-04 gnorm: 0.96 [ 0:41:10<22:04:41] +[titan] 2025-07-09 14:16:10,368 - root - INFO - step: 3020 loss: 18.3881 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.9981 global_avg_mtp_loss: 15.3900 +[titan] 2025-07-09 14:16:10,368 - root - INFO - lr: 2.9972e-04 gnorm: 0.98 [ 0:41:14<22:04:31] +[titan] 2025-07-09 14:16:14,283 - root - INFO - step: 3025 loss: 18.7452 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 3.0527 global_avg_mtp_loss: 15.6925 +[titan] 2025-07-09 14:16:14,283 - root - INFO - lr: 2.9972e-04 gnorm: 1.17 [ 0:41:18<22:04:21] +[titan] 2025-07-09 14:16:18,193 - root - INFO - step: 3030 loss: 18.3985 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.9919 global_avg_mtp_loss: 15.4066 +[titan] 2025-07-09 14:16:18,194 - root - INFO - lr: 2.9972e-04 gnorm: 1.04 [ 0:41:22<22:04:11] +[titan] 2025-07-09 14:16:22,143 - root - INFO - step: 3035 loss: 18.5706 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 3.0273 global_avg_mtp_loss: 15.5434 +[titan] 2025-07-09 14:16:22,143 - root - INFO - lr: 2.9972e-04 gnorm: 0.98 [ 0:41:26<22:04:02] +[titan] 2025-07-09 14:16:26,091 - root - INFO - step: 3040 loss: 18.7309 memory: 44.58GiB(31.99%) tps: 82,995 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 3.0666 global_avg_mtp_loss: 15.6644 +[titan] 2025-07-09 14:16:26,092 - root - INFO - lr: 2.9972e-04 gnorm: 0.96 [ 0:41:30<22:03:53] +[titan] 2025-07-09 14:16:29,995 - root - INFO - step: 3045 loss: 18.4003 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 3.0031 global_avg_mtp_loss: 15.3972 +[titan] 2025-07-09 14:16:29,995 - root - INFO - lr: 2.9972e-04 gnorm: 0.98 [ 0:41:34<22:03:43] +[titan] 2025-07-09 14:16:33,121 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:16:33,921 - root - INFO - step: 3050 loss: 18.6084 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 3.0249 global_avg_mtp_loss: 15.5835 +[titan] 2025-07-09 14:16:33,921 - root - INFO - lr: 2.9971e-04 gnorm: 1.03 [ 0:41:38<22:03:34] +[titan] 2025-07-09 14:16:37,845 - root - INFO - step: 3055 loss: 18.5905 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 3.0394 global_avg_mtp_loss: 15.5510 +[titan] 2025-07-09 14:16:37,846 - root - INFO - lr: 2.9971e-04 gnorm: 1.00 [ 0:41:42<22:03:24] +[titan] 2025-07-09 14:16:41,777 - root - INFO - step: 3060 loss: 18.4914 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 3.0087 global_avg_mtp_loss: 15.4827 +[titan] 2025-07-09 14:16:41,777 - root - INFO - lr: 2.9971e-04 gnorm: 0.89 [ 0:41:46<22:03:15] +[titan] 2025-07-09 14:16:45,693 - root - INFO - step: 3065 loss: 18.8241 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 3.0839 global_avg_mtp_loss: 15.7402 +[titan] 2025-07-09 14:16:45,693 - root - INFO - lr: 2.9971e-04 gnorm: 0.96 [ 0:41:50<22:03:05] +[titan] 2025-07-09 14:16:49,641 - root - INFO - step: 3070 loss: 18.6338 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 3.0446 global_avg_mtp_loss: 15.5892 +[titan] 2025-07-09 14:16:49,641 - root - INFO - lr: 2.9971e-04 gnorm: 0.98 [ 0:41:54<22:02:56] +[titan] 2025-07-09 14:16:51,355 - root - INFO - Dumping profiler traces at step 3072 +[titan] 2025-07-09 14:16:51,386 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:16:53,723 - root - INFO - step: 3075 loss: 18.5001 memory: 44.58GiB(31.99%) tps: 80,285 tflops: 277.08 mfu: 28.02% global_avg_ntp_loss: 3.0149 global_avg_mtp_loss: 15.4853 +[titan] 2025-07-09 14:16:53,723 - root - INFO - lr: 2.9971e-04 gnorm: 0.96 [ 0:41:58<22:02:52] +[titan] 2025-07-09 14:16:57,629 - root - INFO - step: 3080 loss: 18.3786 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.9870 global_avg_mtp_loss: 15.3916 +[titan] 2025-07-09 14:16:57,629 - root - INFO - lr: 2.9971e-04 gnorm: 0.92 [ 0:42:02<22:02:42] +[titan] 2025-07-09 14:17:01,514 - root - INFO - step: 3085 loss: 18.6137 memory: 44.58GiB(31.99%) tps: 84,358 tflops: 291.13 mfu: 29.44% global_avg_ntp_loss: 3.0263 global_avg_mtp_loss: 15.5874 +[titan] 2025-07-09 14:17:01,514 - root - INFO - lr: 2.9970e-04 gnorm: 1.07 [ 0:42:05<22:02:31] +[titan] 2025-07-09 14:17:05,416 - root - INFO - step: 3090 loss: 18.5019 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 3.0231 global_avg_mtp_loss: 15.4788 +[titan] 2025-07-09 14:17:05,416 - root - INFO - lr: 2.9970e-04 gnorm: 0.91 [ 0:42:09<22:02:21] +[titan] 2025-07-09 14:17:09,328 - root - INFO - step: 3095 loss: 18.6120 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 3.0510 global_avg_mtp_loss: 15.5611 +[titan] 2025-07-09 14:17:09,329 - root - INFO - lr: 2.9970e-04 gnorm: 1.03 [ 0:42:13<22:02:11] +[titan] 2025-07-09 14:17:12,448 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:17:13,232 - root - INFO - step: 3100 loss: 18.5580 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 3.0242 global_avg_mtp_loss: 15.5338 +[titan] 2025-07-09 14:17:13,232 - root - INFO - lr: 2.9970e-04 gnorm: 0.98 [ 0:42:17<22:02:01] +[titan] 2025-07-09 14:17:17,136 - root - INFO - step: 3105 loss: 18.3297 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.9808 global_avg_mtp_loss: 15.3489 +[titan] 2025-07-09 14:17:17,136 - root - INFO - lr: 2.9970e-04 gnorm: 1.06 [ 0:42:21<22:01:51] +[titan] 2025-07-09 14:17:21,058 - root - INFO - step: 3110 loss: 18.6961 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 3.0472 global_avg_mtp_loss: 15.6489 +[titan] 2025-07-09 14:17:21,058 - root - INFO - lr: 2.9970e-04 gnorm: 1.03 [ 0:42:25<22:01:42] +[titan] 2025-07-09 14:17:24,990 - root - INFO - step: 3115 loss: 18.4956 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 3.0155 global_avg_mtp_loss: 15.4801 +[titan] 2025-07-09 14:17:24,990 - root - INFO - lr: 2.9970e-04 gnorm: 1.01 [ 0:42:29<22:01:33] +[titan] 2025-07-09 14:17:28,910 - root - INFO - step: 3120 loss: 18.5078 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 3.0174 global_avg_mtp_loss: 15.4904 +[titan] 2025-07-09 14:17:28,910 - root - INFO - lr: 2.9969e-04 gnorm: 1.26 [ 0:42:33<22:01:23] +[titan] 2025-07-09 14:17:32,844 - root - INFO - step: 3125 loss: 18.6401 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 3.0321 global_avg_mtp_loss: 15.6081 +[titan] 2025-07-09 14:17:32,844 - root - INFO - lr: 2.9969e-04 gnorm: 1.08 [ 0:42:37<22:01:14] +[titan] 2025-07-09 14:17:36,744 - root - INFO - step: 3130 loss: 18.4497 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.9923 global_avg_mtp_loss: 15.4573 +[titan] 2025-07-09 14:17:36,744 - root - INFO - lr: 2.9969e-04 gnorm: 1.01 [ 0:42:41<22:01:04] +[titan] 2025-07-09 14:17:40,641 - root - INFO - step: 3135 loss: 18.4626 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 3.0031 global_avg_mtp_loss: 15.4595 +[titan] 2025-07-09 14:17:40,642 - root - INFO - lr: 2.9969e-04 gnorm: 1.11 [ 0:42:45<22:00:54] +[titan] 2025-07-09 14:17:44,584 - root - INFO - step: 3140 loss: 18.6823 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 3.0456 global_avg_mtp_loss: 15.6368 +[titan] 2025-07-09 14:17:44,584 - root - INFO - lr: 2.9969e-04 gnorm: 0.92 [ 0:42:48<22:00:45] +[titan] 2025-07-09 14:17:48,484 - root - INFO - step: 3145 loss: 18.6710 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 3.0537 global_avg_mtp_loss: 15.6174 +[titan] 2025-07-09 14:17:48,485 - root - INFO - lr: 2.9969e-04 gnorm: 1.03 [ 0:42:52<22:00:35] +[titan] 2025-07-09 14:17:51,661 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:17:52,455 - root - INFO - step: 3150 loss: 18.6917 memory: 44.58GiB(31.99%) tps: 82,526 tflops: 284.81 mfu: 28.80% global_avg_ntp_loss: 3.0484 global_avg_mtp_loss: 15.6433 +[titan] 2025-07-09 14:17:52,456 - root - INFO - lr: 2.9969e-04 gnorm: 0.93 [ 0:42:56<22:00:28] +[titan] 2025-07-09 14:17:56,408 - root - INFO - step: 3155 loss: 18.5090 memory: 44.58GiB(31.99%) tps: 82,921 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 3.0265 global_avg_mtp_loss: 15.4824 +[titan] 2025-07-09 14:17:56,408 - root - INFO - lr: 2.9968e-04 gnorm: 1.02 [ 0:43:00<22:00:19] +[titan] 2025-07-09 14:18:00,328 - root - INFO - step: 3160 loss: 18.6125 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 3.0410 global_avg_mtp_loss: 15.5715 +[titan] 2025-07-09 14:18:00,328 - root - INFO - lr: 2.9968e-04 gnorm: 1.22 [ 0:43:04<22:00:10] +[titan] 2025-07-09 14:18:04,245 - root - INFO - step: 3165 loss: 18.3196 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.9736 global_avg_mtp_loss: 15.3460 +[titan] 2025-07-09 14:18:04,245 - root - INFO - lr: 2.9968e-04 gnorm: 0.97 [ 0:43:08<22:00:01] +[titan] 2025-07-09 14:18:08,146 - root - INFO - step: 3170 loss: 18.4084 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 3.0029 global_avg_mtp_loss: 15.4055 +[titan] 2025-07-09 14:18:08,147 - root - INFO - lr: 2.9968e-04 gnorm: 1.04 [ 0:43:12<21:59:51] +[titan] 2025-07-09 14:18:12,065 - root - INFO - step: 3175 loss: 18.6412 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 3.0505 global_avg_mtp_loss: 15.5908 +[titan] 2025-07-09 14:18:12,065 - root - INFO - lr: 2.9968e-04 gnorm: 0.94 [ 0:43:16<21:59:41] +[titan] 2025-07-09 14:18:15,971 - root - INFO - step: 3180 loss: 18.5654 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 3.0283 global_avg_mtp_loss: 15.5371 +[titan] 2025-07-09 14:18:15,971 - root - INFO - lr: 2.9968e-04 gnorm: 0.96 [ 0:43:20<21:59:32] +[titan] 2025-07-09 14:18:19,890 - root - INFO - step: 3185 loss: 18.3977 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.9818 global_avg_mtp_loss: 15.4159 +[titan] 2025-07-09 14:18:19,890 - root - INFO - lr: 2.9968e-04 gnorm: 0.95 [ 0:43:24<21:59:22] +[titan] 2025-07-09 14:18:23,795 - root - INFO - step: 3190 loss: 18.4921 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.9907 global_avg_mtp_loss: 15.5014 +[titan] 2025-07-09 14:18:23,795 - root - INFO - lr: 2.9967e-04 gnorm: 1.00 [ 0:43:28<21:59:13] +[titan] 2025-07-09 14:18:27,706 - root - INFO - step: 3195 loss: 18.5805 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 3.0357 global_avg_mtp_loss: 15.5448 +[titan] 2025-07-09 14:18:27,706 - root - INFO - lr: 2.9967e-04 gnorm: 0.93 [ 0:43:32<21:59:03] +[titan] 2025-07-09 14:18:30,866 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:18:31,653 - root - INFO - step: 3200 loss: 18.7675 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 3.0728 global_avg_mtp_loss: 15.6947 +[titan] 2025-07-09 14:18:31,653 - root - INFO - lr: 2.9967e-04 gnorm: 0.93 [ 0:43:36<21:58:55] +[titan] 2025-07-09 14:18:35,533 - root - INFO - step: 3205 loss: 18.2831 memory: 44.58GiB(31.99%) tps: 84,450 tflops: 291.45 mfu: 29.47% global_avg_ntp_loss: 2.9689 global_avg_mtp_loss: 15.3142 +[titan] 2025-07-09 14:18:35,534 - root - INFO - lr: 2.9967e-04 gnorm: 0.89 [ 0:43:39<21:58:45] +[titan] 2025-07-09 14:18:39,464 - root - INFO - step: 3210 loss: 18.5494 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 3.0177 global_avg_mtp_loss: 15.5316 +[titan] 2025-07-09 14:18:39,464 - root - INFO - lr: 2.9967e-04 gnorm: 0.94 [ 0:43:43<21:58:36] +[titan] 2025-07-09 14:18:43,378 - root - INFO - step: 3215 loss: 18.6089 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 3.0516 global_avg_mtp_loss: 15.5574 +[titan] 2025-07-09 14:18:43,378 - root - INFO - lr: 2.9967e-04 gnorm: 1.17 [ 0:43:47<21:58:26] +[titan] 2025-07-09 14:18:47,284 - root - INFO - step: 3220 loss: 18.3713 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 3.0035 global_avg_mtp_loss: 15.3679 +[titan] 2025-07-09 14:18:47,285 - root - INFO - lr: 2.9966e-04 gnorm: 1.00 [ 0:43:51<21:58:17] +[titan] 2025-07-09 14:18:51,190 - root - INFO - step: 3225 loss: 18.6951 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 3.0514 global_avg_mtp_loss: 15.6437 +[titan] 2025-07-09 14:18:51,190 - root - INFO - lr: 2.9966e-04 gnorm: 0.90 [ 0:43:55<21:58:07] +[titan] 2025-07-09 14:18:55,127 - root - INFO - step: 3230 loss: 18.5374 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 3.0026 global_avg_mtp_loss: 15.5348 +[titan] 2025-07-09 14:18:55,127 - root - INFO - lr: 2.9966e-04 gnorm: 1.05 [ 0:43:59<21:57:59] +[titan] 2025-07-09 14:18:59,096 - root - INFO - step: 3235 loss: 18.4133 memory: 44.58GiB(31.99%) tps: 82,573 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.9773 global_avg_mtp_loss: 15.4360 +[titan] 2025-07-09 14:18:59,096 - root - INFO - lr: 2.9966e-04 gnorm: 0.96 [ 0:44:03<21:57:51] +[titan] 2025-07-09 14:19:03,004 - root - INFO - step: 3240 loss: 18.6468 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 3.0322 global_avg_mtp_loss: 15.6146 +[titan] 2025-07-09 14:19:03,004 - root - INFO - lr: 2.9966e-04 gnorm: 0.88 [ 0:44:07<21:57:42] +[titan] 2025-07-09 14:19:06,958 - root - INFO - step: 3245 loss: 18.6221 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 3.0233 global_avg_mtp_loss: 15.5988 +[titan] 2025-07-09 14:19:06,958 - root - INFO - lr: 2.9966e-04 gnorm: 1.05 [ 0:44:11<21:57:34] +[titan] 2025-07-09 14:19:10,089 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:19:10,877 - root - INFO - step: 3250 loss: 18.6263 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 3.0417 global_avg_mtp_loss: 15.5845 +[titan] 2025-07-09 14:19:10,878 - root - INFO - lr: 2.9966e-04 gnorm: 0.96 [ 0:44:15<21:57:25] +[titan] 2025-07-09 14:19:14,803 - root - INFO - step: 3255 loss: 18.6873 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 3.0533 global_avg_mtp_loss: 15.6341 +[titan] 2025-07-09 14:19:14,803 - root - INFO - lr: 2.9965e-04 gnorm: 1.07 [ 0:44:19<21:57:16] +[titan] 2025-07-09 14:19:18,720 - root - INFO - step: 3260 loss: 18.4623 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 3.0023 global_avg_mtp_loss: 15.4600 +[titan] 2025-07-09 14:19:18,720 - root - INFO - lr: 2.9965e-04 gnorm: 0.95 [ 0:44:23<21:57:07] +[titan] 2025-07-09 14:19:22,619 - root - INFO - step: 3265 loss: 18.4323 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.9997 global_avg_mtp_loss: 15.4326 +[titan] 2025-07-09 14:19:22,619 - root - INFO - lr: 2.9965e-04 gnorm: 0.98 [ 0:44:27<21:56:57] +[titan] 2025-07-09 14:19:26,577 - root - INFO - step: 3270 loss: 18.5603 memory: 44.58GiB(31.99%) tps: 82,784 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 3.0383 global_avg_mtp_loss: 15.5220 +[titan] 2025-07-09 14:19:26,578 - root - INFO - lr: 2.9965e-04 gnorm: 0.97 [ 0:44:30<21:56:49] +[titan] 2025-07-09 14:19:30,494 - root - INFO - step: 3275 loss: 18.5615 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 3.0143 global_avg_mtp_loss: 15.5472 +[titan] 2025-07-09 14:19:30,494 - root - INFO - lr: 2.9965e-04 gnorm: 0.95 [ 0:44:34<21:56:40] +[titan] 2025-07-09 14:19:34,387 - root - INFO - step: 3280 loss: 18.8030 memory: 44.58GiB(31.99%) tps: 84,191 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 3.0728 global_avg_mtp_loss: 15.7302 +[titan] 2025-07-09 14:19:34,387 - root - INFO - lr: 2.9965e-04 gnorm: 0.99 [ 0:44:38<21:56:31] +[titan] 2025-07-09 14:19:38,281 - root - INFO - step: 3285 loss: 18.5698 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 3.0251 global_avg_mtp_loss: 15.5447 +[titan] 2025-07-09 14:19:38,282 - root - INFO - lr: 2.9964e-04 gnorm: 0.99 [ 0:44:42<21:56:21] +[titan] 2025-07-09 14:19:42,183 - root - INFO - step: 3290 loss: 18.3398 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.9658 global_avg_mtp_loss: 15.3740 +[titan] 2025-07-09 14:19:42,183 - root - INFO - lr: 2.9964e-04 gnorm: 1.03 [ 0:44:46<21:56:12] +[titan] 2025-07-09 14:19:46,068 - root - INFO - step: 3295 loss: 18.3932 memory: 44.58GiB(31.99%) tps: 84,342 tflops: 291.08 mfu: 29.43% global_avg_ntp_loss: 2.9911 global_avg_mtp_loss: 15.4021 +[titan] 2025-07-09 14:19:46,068 - root - INFO - lr: 2.9964e-04 gnorm: 1.07 [ 0:44:50<21:56:02] +[titan] 2025-07-09 14:19:49,185 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:19:49,977 - root - INFO - step: 3300 loss: 18.2561 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.9670 global_avg_mtp_loss: 15.2891 +[titan] 2025-07-09 14:19:49,978 - root - INFO - lr: 2.9964e-04 gnorm: 1.04 [ 0:44:54<21:55:53] +[titan] 2025-07-09 14:19:53,905 - root - INFO - step: 3305 loss: 18.4695 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.9861 global_avg_mtp_loss: 15.4834 +[titan] 2025-07-09 14:19:53,905 - root - INFO - lr: 2.9964e-04 gnorm: 0.92 [ 0:44:58<21:55:44] +[titan] 2025-07-09 14:19:57,795 - root - INFO - step: 3310 loss: 18.4624 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 3.0046 global_avg_mtp_loss: 15.4579 +[titan] 2025-07-09 14:19:57,795 - root - INFO - lr: 2.9964e-04 gnorm: 0.92 [ 0:45:02<21:55:34] +[titan] 2025-07-09 14:20:01,678 - root - INFO - step: 3315 loss: 18.2407 memory: 44.58GiB(31.99%) tps: 84,391 tflops: 291.25 mfu: 29.45% global_avg_ntp_loss: 2.9692 global_avg_mtp_loss: 15.2715 +[titan] 2025-07-09 14:20:01,679 - root - INFO - lr: 2.9964e-04 gnorm: 1.11 [ 0:45:06<21:55:24] +[titan] 2025-07-09 14:20:05,592 - root - INFO - step: 3320 loss: 18.4071 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.9852 global_avg_mtp_loss: 15.4219 +[titan] 2025-07-09 14:20:05,593 - root - INFO - lr: 2.9963e-04 gnorm: 0.92 [ 0:45:09<21:55:15] +[titan] 2025-07-09 14:20:09,504 - root - INFO - step: 3325 loss: 18.3400 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.9820 global_avg_mtp_loss: 15.3581 +[titan] 2025-07-09 14:20:09,504 - root - INFO - lr: 2.9963e-04 gnorm: 0.96 [ 0:45:13<21:55:06] +[titan] 2025-07-09 14:20:13,419 - root - INFO - step: 3330 loss: 18.5398 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 3.0254 global_avg_mtp_loss: 15.5144 +[titan] 2025-07-09 14:20:13,419 - root - INFO - lr: 2.9963e-04 gnorm: 0.97 [ 0:45:17<21:54:57] +[titan] 2025-07-09 14:20:17,338 - root - INFO - step: 3335 loss: 18.2656 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.9668 global_avg_mtp_loss: 15.2987 +[titan] 2025-07-09 14:20:17,339 - root - INFO - lr: 2.9963e-04 gnorm: 1.00 [ 0:45:21<21:54:49] +[titan] 2025-07-09 14:20:21,275 - root - INFO - step: 3340 loss: 18.3118 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.9797 global_avg_mtp_loss: 15.3321 +[titan] 2025-07-09 14:20:21,276 - root - INFO - lr: 2.9963e-04 gnorm: 1.01 [ 0:45:25<21:54:40] +[titan] 2025-07-09 14:20:25,178 - root - INFO - step: 3345 loss: 18.3838 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.9785 global_avg_mtp_loss: 15.4053 +[titan] 2025-07-09 14:20:25,178 - root - INFO - lr: 2.9963e-04 gnorm: 0.96 [ 0:45:29<21:54:31] +[titan] 2025-07-09 14:20:28,352 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:20:29,141 - root - INFO - step: 3350 loss: 18.4265 memory: 44.58GiB(31.99%) tps: 82,690 tflops: 285.38 mfu: 28.85% global_avg_ntp_loss: 2.9855 global_avg_mtp_loss: 15.4410 +[titan] 2025-07-09 14:20:29,141 - root - INFO - lr: 2.9962e-04 gnorm: 0.99 [ 0:45:33<21:54:24] +[titan] 2025-07-09 14:20:33,079 - root - INFO - step: 3355 loss: 18.5339 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 3.0028 global_avg_mtp_loss: 15.5311 +[titan] 2025-07-09 14:20:33,079 - root - INFO - lr: 2.9962e-04 gnorm: 1.08 [ 0:45:37<21:54:15] +[titan] 2025-07-09 14:20:36,992 - root - INFO - step: 3360 loss: 18.4262 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.9867 global_avg_mtp_loss: 15.4395 +[titan] 2025-07-09 14:20:36,992 - root - INFO - lr: 2.9962e-04 gnorm: 1.03 [ 0:45:41<21:54:07] +[titan] 2025-07-09 14:20:40,894 - root - INFO - step: 3365 loss: 18.2953 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.9778 global_avg_mtp_loss: 15.3175 +[titan] 2025-07-09 14:20:40,894 - root - INFO - lr: 2.9962e-04 gnorm: 0.98 [ 0:45:45<21:53:57] +[titan] 2025-07-09 14:20:44,832 - root - INFO - step: 3370 loss: 18.4001 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.9773 global_avg_mtp_loss: 15.4227 +[titan] 2025-07-09 14:20:44,833 - root - INFO - lr: 2.9962e-04 gnorm: 0.97 [ 0:45:49<21:53:49] +[titan] 2025-07-09 14:20:48,743 - root - INFO - step: 3375 loss: 18.3887 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.9779 global_avg_mtp_loss: 15.4108 +[titan] 2025-07-09 14:20:48,743 - root - INFO - lr: 2.9962e-04 gnorm: 1.06 [ 0:45:53<21:53:40] +[titan] 2025-07-09 14:20:52,681 - root - INFO - step: 3380 loss: 18.3699 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.9674 global_avg_mtp_loss: 15.4025 +[titan] 2025-07-09 14:20:52,681 - root - INFO - lr: 2.9961e-04 gnorm: 0.99 [ 0:45:57<21:53:32] +[titan] 2025-07-09 14:20:56,620 - root - INFO - step: 3385 loss: 18.4244 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.9880 global_avg_mtp_loss: 15.4364 +[titan] 2025-07-09 14:20:56,620 - root - INFO - lr: 2.9961e-04 gnorm: 0.96 [ 0:46:00<21:53:24] +[titan] 2025-07-09 14:21:00,550 - root - INFO - step: 3390 loss: 18.8135 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 3.0587 global_avg_mtp_loss: 15.7548 +[titan] 2025-07-09 14:21:00,551 - root - INFO - lr: 2.9961e-04 gnorm: 1.04 [ 0:46:04<21:53:16] +[titan] 2025-07-09 14:21:04,464 - root - INFO - step: 3395 loss: 18.5582 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 3.0206 global_avg_mtp_loss: 15.5376 +[titan] 2025-07-09 14:21:04,464 - root - INFO - lr: 2.9961e-04 gnorm: 1.01 [ 0:46:08<21:53:07] +[titan] 2025-07-09 14:21:07,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:21:08,392 - root - INFO - step: 3400 loss: 18.2393 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.9737 global_avg_mtp_loss: 15.2656 +[titan] 2025-07-09 14:21:08,393 - root - INFO - lr: 2.9961e-04 gnorm: 0.95 [ 0:46:12<21:52:59] +[titan] 2025-07-09 14:21:12,322 - root - INFO - step: 3405 loss: 18.4462 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.9831 global_avg_mtp_loss: 15.4631 +[titan] 2025-07-09 14:21:12,322 - root - INFO - lr: 2.9961e-04 gnorm: 0.98 [ 0:46:16<21:52:50] +[titan] 2025-07-09 14:21:16,252 - root - INFO - step: 3410 loss: 18.4760 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.9893 global_avg_mtp_loss: 15.4867 +[titan] 2025-07-09 14:21:16,252 - root - INFO - lr: 2.9961e-04 gnorm: 0.96 [ 0:46:20<21:52:42] +[titan] 2025-07-09 14:21:20,148 - root - INFO - step: 3415 loss: 18.4615 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 3.0087 global_avg_mtp_loss: 15.4528 +[titan] 2025-07-09 14:21:20,149 - root - INFO - lr: 2.9960e-04 gnorm: 0.85 [ 0:46:24<21:52:33] +[titan] 2025-07-09 14:21:24,058 - root - INFO - step: 3420 loss: 18.4086 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.9953 global_avg_mtp_loss: 15.4133 +[titan] 2025-07-09 14:21:24,059 - root - INFO - lr: 2.9960e-04 gnorm: 0.92 [ 0:46:28<21:52:24] +[titan] 2025-07-09 14:21:27,956 - root - INFO - step: 3425 loss: 18.6556 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 3.0979 global_avg_mtp_loss: 15.5577 +[titan] 2025-07-09 14:21:27,956 - root - INFO - lr: 2.9960e-04 gnorm: 2.66 [ 0:46:32<21:52:15] +[titan] 2025-07-09 14:21:31,902 - root - INFO - step: 3430 loss: 18.5448 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 3.0046 global_avg_mtp_loss: 15.5402 +[titan] 2025-07-09 14:21:31,902 - root - INFO - lr: 2.9960e-04 gnorm: 1.01 [ 0:46:36<21:52:07] +[titan] 2025-07-09 14:21:35,822 - root - INFO - step: 3435 loss: 18.4438 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.9874 global_avg_mtp_loss: 15.4564 +[titan] 2025-07-09 14:21:35,823 - root - INFO - lr: 2.9960e-04 gnorm: 0.99 [ 0:46:40<21:51:59] +[titan] 2025-07-09 14:21:39,739 - root - INFO - step: 3440 loss: 18.5358 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 3.0048 global_avg_mtp_loss: 15.5310 +[titan] 2025-07-09 14:21:39,739 - root - INFO - lr: 2.9960e-04 gnorm: 0.90 [ 0:46:44<21:51:50] +[titan] 2025-07-09 14:21:43,667 - root - INFO - step: 3445 loss: 18.3291 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.9792 global_avg_mtp_loss: 15.3499 +[titan] 2025-07-09 14:21:43,667 - root - INFO - lr: 2.9959e-04 gnorm: 0.97 [ 0:46:48<21:51:42] +[titan] 2025-07-09 14:21:46,828 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:21:47,616 - root - INFO - step: 3450 loss: 18.3031 memory: 44.58GiB(31.99%) tps: 82,982 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.9657 global_avg_mtp_loss: 15.3374 +[titan] 2025-07-09 14:21:47,616 - root - INFO - lr: 2.9959e-04 gnorm: 1.15 [ 0:46:51<21:51:34] +[titan] 2025-07-09 14:21:51,544 - root - INFO - step: 3455 loss: 18.7396 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 3.0579 global_avg_mtp_loss: 15.6817 +[titan] 2025-07-09 14:21:51,544 - root - INFO - lr: 2.9959e-04 gnorm: 0.92 [ 0:46:55<21:51:26] +[titan] 2025-07-09 14:21:55,458 - root - INFO - step: 3460 loss: 18.4088 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 3.0065 global_avg_mtp_loss: 15.4023 +[titan] 2025-07-09 14:21:55,458 - root - INFO - lr: 2.9959e-04 gnorm: 0.94 [ 0:46:59<21:51:18] +[titan] 2025-07-09 14:21:59,392 - root - INFO - step: 3465 loss: 18.2061 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.9505 global_avg_mtp_loss: 15.2555 +[titan] 2025-07-09 14:21:59,393 - root - INFO - lr: 2.9959e-04 gnorm: 1.05 [ 0:47:03<21:51:10] +[titan] 2025-07-09 14:22:03,302 - root - INFO - step: 3470 loss: 18.1443 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.9307 global_avg_mtp_loss: 15.2136 +[titan] 2025-07-09 14:22:03,302 - root - INFO - lr: 2.9959e-04 gnorm: 1.00 [ 0:47:07<21:51:01] +[titan] 2025-07-09 14:22:07,213 - root - INFO - step: 3475 loss: 18.4698 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 3.0012 global_avg_mtp_loss: 15.4686 +[titan] 2025-07-09 14:22:07,213 - root - INFO - lr: 2.9958e-04 gnorm: 1.08 [ 0:47:11<21:50:52] +[titan] 2025-07-09 14:22:11,137 - root - INFO - step: 3480 loss: 18.7380 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 3.0896 global_avg_mtp_loss: 15.6484 +[titan] 2025-07-09 14:22:11,137 - root - INFO - lr: 2.9958e-04 gnorm: 1.02 [ 0:47:15<21:50:44] +[titan] 2025-07-09 14:22:15,062 - root - INFO - step: 3485 loss: 18.5454 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 3.0270 global_avg_mtp_loss: 15.5184 +[titan] 2025-07-09 14:22:15,062 - root - INFO - lr: 2.9958e-04 gnorm: 1.11 [ 0:47:19<21:50:36] +[titan] 2025-07-09 14:22:18,982 - root - INFO - step: 3490 loss: 18.4768 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.9860 global_avg_mtp_loss: 15.4907 +[titan] 2025-07-09 14:22:18,982 - root - INFO - lr: 2.9958e-04 gnorm: 1.04 [ 0:47:23<21:50:27] +[titan] 2025-07-09 14:22:22,909 - root - INFO - step: 3495 loss: 18.6823 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 3.0465 global_avg_mtp_loss: 15.6359 +[titan] 2025-07-09 14:22:22,910 - root - INFO - lr: 2.9958e-04 gnorm: 0.94 [ 0:47:27<21:50:19] +[titan] 2025-07-09 14:22:26,041 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:22:26,839 - root - INFO - step: 3500 loss: 18.4833 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.9852 global_avg_mtp_loss: 15.4981 +[titan] 2025-07-09 14:22:26,839 - root - INFO - lr: 2.9958e-04 gnorm: 0.93 [ 0:47:31<21:50:11] +[titan] 2025-07-09 14:22:30,761 - root - INFO - step: 3505 loss: 18.3329 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.9817 global_avg_mtp_loss: 15.3512 +[titan] 2025-07-09 14:22:30,761 - root - INFO - lr: 2.9957e-04 gnorm: 0.99 [ 0:47:35<21:50:03] +[titan] 2025-07-09 14:22:34,694 - root - INFO - step: 3510 loss: 18.3526 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.9658 global_avg_mtp_loss: 15.3868 +[titan] 2025-07-09 14:22:34,694 - root - INFO - lr: 2.9957e-04 gnorm: 0.98 [ 0:47:39<21:49:55] +[titan] 2025-07-09 14:22:38,628 - root - INFO - step: 3515 loss: 18.3138 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.9593 global_avg_mtp_loss: 15.3545 +[titan] 2025-07-09 14:22:38,628 - root - INFO - lr: 2.9957e-04 gnorm: 1.07 [ 0:47:42<21:49:47] +[titan] 2025-07-09 14:22:42,593 - root - INFO - step: 3520 loss: 18.3274 memory: 44.58GiB(31.99%) tps: 82,657 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.9644 global_avg_mtp_loss: 15.3630 +[titan] 2025-07-09 14:22:42,593 - root - INFO - lr: 2.9957e-04 gnorm: 1.00 [ 0:47:46<21:49:40] +[titan] 2025-07-09 14:22:46,492 - root - INFO - step: 3525 loss: 18.4527 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.9906 global_avg_mtp_loss: 15.4621 +[titan] 2025-07-09 14:22:46,492 - root - INFO - lr: 2.9957e-04 gnorm: 0.94 [ 0:47:50<21:49:31] +[titan] 2025-07-09 14:22:50,410 - root - INFO - step: 3530 loss: 18.6407 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 3.0480 global_avg_mtp_loss: 15.5927 +[titan] 2025-07-09 14:22:50,410 - root - INFO - lr: 2.9956e-04 gnorm: 0.96 [ 0:47:54<21:49:23] +[titan] 2025-07-09 14:22:54,342 - root - INFO - step: 3535 loss: 18.5147 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 3.0021 global_avg_mtp_loss: 15.5126 +[titan] 2025-07-09 14:22:54,343 - root - INFO - lr: 2.9956e-04 gnorm: 1.08 [ 0:47:58<21:49:15] +[titan] 2025-07-09 14:22:58,269 - root - INFO - step: 3540 loss: 18.5010 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 3.0049 global_avg_mtp_loss: 15.4961 +[titan] 2025-07-09 14:22:58,269 - root - INFO - lr: 2.9956e-04 gnorm: 1.20 [ 0:48:02<21:49:07] +[titan] 2025-07-09 14:23:02,199 - root - INFO - step: 3545 loss: 18.5401 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 3.0139 global_avg_mtp_loss: 15.5262 +[titan] 2025-07-09 14:23:02,199 - root - INFO - lr: 2.9956e-04 gnorm: 1.02 [ 0:48:06<21:48:59] +[titan] 2025-07-09 14:23:05,317 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:23:06,105 - root - INFO - step: 3550 loss: 18.3516 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.9776 global_avg_mtp_loss: 15.3739 +[titan] 2025-07-09 14:23:06,105 - root - INFO - lr: 2.9956e-04 gnorm: 1.00 [ 0:48:10<21:48:51] +[titan] 2025-07-09 14:23:10,004 - root - INFO - step: 3555 loss: 18.4530 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.9931 global_avg_mtp_loss: 15.4599 +[titan] 2025-07-09 14:23:10,004 - root - INFO - lr: 2.9956e-04 gnorm: 0.96 [ 0:48:14<21:48:42] +[titan] 2025-07-09 14:23:13,912 - root - INFO - step: 3560 loss: 18.4033 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 3.0035 global_avg_mtp_loss: 15.3997 +[titan] 2025-07-09 14:23:13,912 - root - INFO - lr: 2.9955e-04 gnorm: 1.07 [ 0:48:18<21:48:33] +[titan] 2025-07-09 14:23:17,819 - root - INFO - step: 3565 loss: 18.2383 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.9461 global_avg_mtp_loss: 15.2922 +[titan] 2025-07-09 14:23:17,819 - root - INFO - lr: 2.9955e-04 gnorm: 1.04 [ 0:48:22<21:48:25] +[titan] 2025-07-09 14:23:21,734 - root - INFO - step: 3570 loss: 18.4059 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.9828 global_avg_mtp_loss: 15.4231 +[titan] 2025-07-09 14:23:21,734 - root - INFO - lr: 2.9955e-04 gnorm: 1.02 [ 0:48:26<21:48:17] +[titan] 2025-07-09 14:23:25,637 - root - INFO - step: 3575 loss: 18.2601 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.9584 global_avg_mtp_loss: 15.3017 +[titan] 2025-07-09 14:23:25,638 - root - INFO - lr: 2.9955e-04 gnorm: 1.07 [ 0:48:29<21:48:08] +[titan] 2025-07-09 14:23:29,536 - root - INFO - step: 3580 loss: 18.3036 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.9734 global_avg_mtp_loss: 15.3302 +[titan] 2025-07-09 14:23:29,536 - root - INFO - lr: 2.9955e-04 gnorm: 0.89 [ 0:48:33<21:47:59] +[titan] 2025-07-09 14:23:32,868 - root - INFO - Dumping profiler traces at step 3584 +[titan] 2025-07-09 14:23:32,900 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:23:33,682 - root - INFO - step: 3585 loss: 18.3507 memory: 44.58GiB(31.99%) tps: 79,042 tflops: 272.79 mfu: 27.58% global_avg_ntp_loss: 2.9864 global_avg_mtp_loss: 15.3643 +[titan] 2025-07-09 14:23:33,682 - root - INFO - lr: 2.9955e-04 gnorm: 1.02 [ 0:48:38<21:47:57] +[titan] 2025-07-09 14:23:37,576 - root - INFO - step: 3590 loss: 18.2082 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.9371 global_avg_mtp_loss: 15.2711 +[titan] 2025-07-09 14:23:37,576 - root - INFO - lr: 2.9954e-04 gnorm: 0.88 [ 0:48:41<21:47:49] +[titan] 2025-07-09 14:23:41,504 - root - INFO - step: 3595 loss: 18.2852 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.9711 global_avg_mtp_loss: 15.3141 +[titan] 2025-07-09 14:23:41,504 - root - INFO - lr: 2.9954e-04 gnorm: 1.04 [ 0:48:45<21:47:41] +[titan] 2025-07-09 14:23:44,608 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:23:45,399 - root - INFO - step: 3600 loss: 18.2785 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.9684 global_avg_mtp_loss: 15.3102 +[titan] 2025-07-09 14:23:45,399 - root - INFO - lr: 2.9954e-04 gnorm: 0.91 [ 0:48:49<21:47:32] +[titan] 2025-07-09 14:23:49,300 - root - INFO - step: 3605 loss: 18.2429 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.9478 global_avg_mtp_loss: 15.2951 +[titan] 2025-07-09 14:23:49,300 - root - INFO - lr: 2.9954e-04 gnorm: 1.02 [ 0:48:53<21:47:23] +[titan] 2025-07-09 14:23:53,190 - root - INFO - step: 3610 loss: 18.5452 memory: 44.58GiB(31.99%) tps: 84,234 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 3.0015 global_avg_mtp_loss: 15.5437 +[titan] 2025-07-09 14:23:53,191 - root - INFO - lr: 2.9954e-04 gnorm: 1.06 [ 0:48:57<21:47:14] +[titan] 2025-07-09 14:23:57,103 - root - INFO - step: 3615 loss: 18.2506 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.9359 global_avg_mtp_loss: 15.3147 +[titan] 2025-07-09 14:23:57,104 - root - INFO - lr: 2.9954e-04 gnorm: 1.01 [ 0:49:01<21:47:06] +[titan] 2025-07-09 14:24:00,994 - root - INFO - step: 3620 loss: 18.1394 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.9353 global_avg_mtp_loss: 15.2041 +[titan] 2025-07-09 14:24:00,995 - root - INFO - lr: 2.9953e-04 gnorm: 0.98 [ 0:49:05<21:46:57] +[titan] 2025-07-09 14:24:04,888 - root - INFO - step: 3625 loss: 18.3140 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.9646 global_avg_mtp_loss: 15.3494 +[titan] 2025-07-09 14:24:04,888 - root - INFO - lr: 2.9953e-04 gnorm: 0.92 [ 0:49:09<21:46:49] +[titan] 2025-07-09 14:24:08,788 - root - INFO - step: 3630 loss: 18.2621 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.9493 global_avg_mtp_loss: 15.3128 +[titan] 2025-07-09 14:24:08,789 - root - INFO - lr: 2.9953e-04 gnorm: 0.96 [ 0:49:13<21:46:40] +[titan] 2025-07-09 14:24:12,681 - root - INFO - step: 3635 loss: 18.3036 memory: 44.58GiB(31.99%) tps: 84,183 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.9666 global_avg_mtp_loss: 15.3369 +[titan] 2025-07-09 14:24:12,682 - root - INFO - lr: 2.9953e-04 gnorm: 0.91 [ 0:49:17<21:46:31] +[titan] 2025-07-09 14:24:16,590 - root - INFO - step: 3640 loss: 18.2751 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.9478 global_avg_mtp_loss: 15.3273 +[titan] 2025-07-09 14:24:16,591 - root - INFO - lr: 2.9953e-04 gnorm: 0.98 [ 0:49:20<21:46:23] +[titan] 2025-07-09 14:24:20,484 - root - INFO - step: 3645 loss: 18.4192 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.9737 global_avg_mtp_loss: 15.4455 +[titan] 2025-07-09 14:24:20,484 - root - INFO - lr: 2.9952e-04 gnorm: 1.00 [ 0:49:24<21:46:15] +[titan] 2025-07-09 14:24:23,598 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:24:24,395 - root - INFO - step: 3650 loss: 18.3138 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.9765 global_avg_mtp_loss: 15.3373 +[titan] 2025-07-09 14:24:24,395 - root - INFO - lr: 2.9952e-04 gnorm: 0.92 [ 0:49:28<21:46:06] +[titan] 2025-07-09 14:24:28,305 - root - INFO - step: 3655 loss: 18.3792 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.9732 global_avg_mtp_loss: 15.4059 +[titan] 2025-07-09 14:24:28,305 - root - INFO - lr: 2.9952e-04 gnorm: 0.92 [ 0:49:32<21:45:58] +[titan] 2025-07-09 14:24:32,223 - root - INFO - step: 3660 loss: 18.4396 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.9844 global_avg_mtp_loss: 15.4552 +[titan] 2025-07-09 14:24:32,224 - root - INFO - lr: 2.9952e-04 gnorm: 1.07 [ 0:49:36<21:45:50] +[titan] 2025-07-09 14:24:36,162 - root - INFO - step: 3665 loss: 18.4208 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.9810 global_avg_mtp_loss: 15.4398 +[titan] 2025-07-09 14:24:36,162 - root - INFO - lr: 2.9952e-04 gnorm: 0.99 [ 0:49:40<21:45:43] +[titan] 2025-07-09 14:24:40,167 - root - INFO - step: 3670 loss: 18.1964 memory: 44.58GiB(31.99%) tps: 81,832 tflops: 282.42 mfu: 28.56% global_avg_ntp_loss: 2.9468 global_avg_mtp_loss: 15.2496 +[titan] 2025-07-09 14:24:40,167 - root - INFO - lr: 2.9952e-04 gnorm: 1.01 [ 0:49:44<21:45:37] +[titan] 2025-07-09 14:24:44,168 - root - INFO - step: 3675 loss: 18.1145 memory: 44.58GiB(31.99%) tps: 81,893 tflops: 282.63 mfu: 28.58% global_avg_ntp_loss: 2.9361 global_avg_mtp_loss: 15.1784 +[titan] 2025-07-09 14:24:44,168 - root - INFO - lr: 2.9951e-04 gnorm: 1.13 [ 0:49:48<21:45:31] +[titan] 2025-07-09 14:24:48,084 - root - INFO - step: 3680 loss: 18.1089 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.9219 global_avg_mtp_loss: 15.1870 +[titan] 2025-07-09 14:24:48,084 - root - INFO - lr: 2.9951e-04 gnorm: 1.06 [ 0:49:52<21:45:23] +[titan] 2025-07-09 14:24:51,981 - root - INFO - step: 3685 loss: 18.3129 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.9642 global_avg_mtp_loss: 15.3487 +[titan] 2025-07-09 14:24:51,981 - root - INFO - lr: 2.9951e-04 gnorm: 1.06 [ 0:49:56<21:45:15] +[titan] 2025-07-09 14:24:55,868 - root - INFO - step: 3690 loss: 18.1688 memory: 44.58GiB(31.99%) tps: 84,296 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.9373 global_avg_mtp_loss: 15.2314 +[titan] 2025-07-09 14:24:55,868 - root - INFO - lr: 2.9951e-04 gnorm: 1.03 [ 0:50:00<21:45:06] +[titan] 2025-07-09 14:24:59,807 - root - INFO - step: 3695 loss: 18.4790 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 3.0155 global_avg_mtp_loss: 15.4635 +[titan] 2025-07-09 14:24:59,808 - root - INFO - lr: 2.9951e-04 gnorm: 1.08 [ 0:50:04<21:44:59] +[titan] 2025-07-09 14:25:02,912 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:25:03,711 - root - INFO - step: 3700 loss: 18.3680 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.9626 global_avg_mtp_loss: 15.4054 +[titan] 2025-07-09 14:25:03,711 - root - INFO - lr: 2.9950e-04 gnorm: 1.02 [ 0:50:08<21:44:50] +[titan] 2025-07-09 14:25:07,646 - root - INFO - step: 3705 loss: 18.4314 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.9930 global_avg_mtp_loss: 15.4384 +[titan] 2025-07-09 14:25:07,646 - root - INFO - lr: 2.9950e-04 gnorm: 1.02 [ 0:50:11<21:44:43] +[titan] 2025-07-09 14:25:11,571 - root - INFO - step: 3710 loss: 18.3346 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.9647 global_avg_mtp_loss: 15.3699 +[titan] 2025-07-09 14:25:11,572 - root - INFO - lr: 2.9950e-04 gnorm: 0.95 [ 0:50:15<21:44:35] +[titan] 2025-07-09 14:25:15,582 - root - INFO - step: 3715 loss: 18.1613 memory: 44.58GiB(31.99%) tps: 81,716 tflops: 282.02 mfu: 28.52% global_avg_ntp_loss: 2.9332 global_avg_mtp_loss: 15.2281 +[titan] 2025-07-09 14:25:15,582 - root - INFO - lr: 2.9950e-04 gnorm: 0.99 [ 0:50:19<21:44:30] +[titan] 2025-07-09 14:25:19,506 - root - INFO - step: 3720 loss: 18.4004 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.9885 global_avg_mtp_loss: 15.4119 +[titan] 2025-07-09 14:25:19,506 - root - INFO - lr: 2.9950e-04 gnorm: 1.08 [ 0:50:23<21:44:22] +[titan] 2025-07-09 14:25:23,456 - root - INFO - step: 3725 loss: 18.1977 memory: 44.58GiB(31.99%) tps: 82,958 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.9370 global_avg_mtp_loss: 15.2608 +[titan] 2025-07-09 14:25:23,456 - root - INFO - lr: 2.9950e-04 gnorm: 1.02 [ 0:50:27<21:44:15] +[titan] 2025-07-09 14:25:27,369 - root - INFO - step: 3730 loss: 18.3217 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.9616 global_avg_mtp_loss: 15.3601 +[titan] 2025-07-09 14:25:27,369 - root - INFO - lr: 2.9949e-04 gnorm: 0.96 [ 0:50:31<21:44:07] +[titan] 2025-07-09 14:25:31,287 - root - INFO - step: 3735 loss: 18.2441 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.9457 global_avg_mtp_loss: 15.2984 +[titan] 2025-07-09 14:25:31,288 - root - INFO - lr: 2.9949e-04 gnorm: 1.00 [ 0:50:35<21:43:59] +[titan] 2025-07-09 14:25:35,242 - root - INFO - step: 3740 loss: 18.3217 memory: 44.58GiB(31.99%) tps: 82,872 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.9547 global_avg_mtp_loss: 15.3670 +[titan] 2025-07-09 14:25:35,242 - root - INFO - lr: 2.9949e-04 gnorm: 0.94 [ 0:50:39<21:43:52] +[titan] 2025-07-09 14:25:39,148 - root - INFO - step: 3745 loss: 18.4440 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.9848 global_avg_mtp_loss: 15.4592 +[titan] 2025-07-09 14:25:39,149 - root - INFO - lr: 2.9949e-04 gnorm: 0.92 [ 0:50:43<21:43:44] +[titan] 2025-07-09 14:25:42,274 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:25:43,085 - root - INFO - step: 3750 loss: 18.0964 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.9174 global_avg_mtp_loss: 15.1789 +[titan] 2025-07-09 14:25:43,085 - root - INFO - lr: 2.9949e-04 gnorm: 1.00 [ 0:50:47<21:43:37] +[titan] 2025-07-09 14:25:46,986 - root - INFO - step: 3755 loss: 18.3320 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.9636 global_avg_mtp_loss: 15.3684 +[titan] 2025-07-09 14:25:46,986 - root - INFO - lr: 2.9948e-04 gnorm: 0.96 [ 0:50:51<21:43:29] +[titan] 2025-07-09 14:25:50,904 - root - INFO - step: 3760 loss: 18.0351 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.9159 global_avg_mtp_loss: 15.1192 +[titan] 2025-07-09 14:25:50,904 - root - INFO - lr: 2.9948e-04 gnorm: 1.25 [ 0:50:55<21:43:21] +[titan] 2025-07-09 14:25:54,847 - root - INFO - step: 3765 loss: 18.2450 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.9584 global_avg_mtp_loss: 15.2866 +[titan] 2025-07-09 14:25:54,847 - root - INFO - lr: 2.9948e-04 gnorm: 0.97 [ 0:50:59<21:43:14] +[titan] 2025-07-09 14:25:58,814 - root - INFO - step: 3770 loss: 18.3966 memory: 44.58GiB(31.99%) tps: 82,612 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.9844 global_avg_mtp_loss: 15.4122 +[titan] 2025-07-09 14:25:58,814 - root - INFO - lr: 2.9948e-04 gnorm: 0.95 [ 0:51:03<21:43:07] +[titan] 2025-07-09 14:26:02,716 - root - INFO - step: 3775 loss: 18.3095 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.9662 global_avg_mtp_loss: 15.3432 +[titan] 2025-07-09 14:26:02,716 - root - INFO - lr: 2.9948e-04 gnorm: 0.94 [ 0:51:07<21:42:59] +[titan] 2025-07-09 14:26:06,707 - root - INFO - step: 3780 loss: 18.2410 memory: 44.58GiB(31.99%) tps: 82,110 tflops: 283.38 mfu: 28.65% global_avg_ntp_loss: 2.9549 global_avg_mtp_loss: 15.2861 +[titan] 2025-07-09 14:26:06,707 - root - INFO - lr: 2.9947e-04 gnorm: 0.94 [ 0:51:11<21:42:53] +[titan] 2025-07-09 14:26:10,622 - root - INFO - step: 3785 loss: 18.0062 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.9102 global_avg_mtp_loss: 15.0960 +[titan] 2025-07-09 14:26:10,623 - root - INFO - lr: 2.9947e-04 gnorm: 0.89 [ 0:51:14<21:42:45] +[titan] 2025-07-09 14:26:14,543 - root - INFO - step: 3790 loss: 18.5313 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 3.0352 global_avg_mtp_loss: 15.4961 +[titan] 2025-07-09 14:26:14,543 - root - INFO - lr: 2.9947e-04 gnorm: 1.14 [ 0:51:18<21:42:38] +[titan] 2025-07-09 14:26:18,437 - root - INFO - step: 3795 loss: 18.2517 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.9478 global_avg_mtp_loss: 15.3039 +[titan] 2025-07-09 14:26:18,437 - root - INFO - lr: 2.9947e-04 gnorm: 1.03 [ 0:51:22<21:42:29] +[titan] 2025-07-09 14:26:21,585 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:26:22,368 - root - INFO - step: 3800 loss: 18.2815 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.9613 global_avg_mtp_loss: 15.3202 +[titan] 2025-07-09 14:26:22,368 - root - INFO - lr: 2.9947e-04 gnorm: 0.93 [ 0:51:26<21:42:22] +[titan] 2025-07-09 14:26:26,286 - root - INFO - step: 3805 loss: 17.9390 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.8850 global_avg_mtp_loss: 15.0540 +[titan] 2025-07-09 14:26:26,286 - root - INFO - lr: 2.9947e-04 gnorm: 1.02 [ 0:51:30<21:42:14] +[titan] 2025-07-09 14:26:30,276 - root - INFO - step: 3810 loss: 18.3280 memory: 44.58GiB(31.99%) tps: 82,118 tflops: 283.40 mfu: 28.66% global_avg_ntp_loss: 2.9498 global_avg_mtp_loss: 15.3782 +[titan] 2025-07-09 14:26:30,277 - root - INFO - lr: 2.9946e-04 gnorm: 1.05 [ 0:51:34<21:42:08] +[titan] 2025-07-09 14:26:34,199 - root - INFO - step: 3815 loss: 18.2656 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.9476 global_avg_mtp_loss: 15.3180 +[titan] 2025-07-09 14:26:34,199 - root - INFO - lr: 2.9946e-04 gnorm: 0.96 [ 0:51:38<21:42:01] +[titan] 2025-07-09 14:26:38,128 - root - INFO - step: 3820 loss: 18.2470 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.9465 global_avg_mtp_loss: 15.3005 +[titan] 2025-07-09 14:26:38,129 - root - INFO - lr: 2.9946e-04 gnorm: 1.03 [ 0:51:42<21:41:53] +[titan] 2025-07-09 14:26:42,110 - root - INFO - step: 3825 loss: 18.1848 memory: 44.58GiB(31.99%) tps: 82,310 tflops: 284.07 mfu: 28.72% global_avg_ntp_loss: 2.9491 global_avg_mtp_loss: 15.2357 +[titan] 2025-07-09 14:26:42,110 - root - INFO - lr: 2.9946e-04 gnorm: 1.14 [ 0:51:46<21:41:47] +[titan] 2025-07-09 14:26:46,070 - root - INFO - step: 3830 loss: 18.2147 memory: 44.58GiB(31.99%) tps: 82,743 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.9360 global_avg_mtp_loss: 15.2786 +[titan] 2025-07-09 14:26:46,071 - root - INFO - lr: 2.9946e-04 gnorm: 0.99 [ 0:51:50<21:41:41] +[titan] 2025-07-09 14:26:49,987 - root - INFO - step: 3835 loss: 18.2797 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.9617 global_avg_mtp_loss: 15.3180 +[titan] 2025-07-09 14:26:49,987 - root - INFO - lr: 2.9945e-04 gnorm: 0.93 [ 0:51:54<21:41:33] +[titan] 2025-07-09 14:26:53,953 - root - INFO - step: 3840 loss: 18.1800 memory: 44.58GiB(31.99%) tps: 82,631 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.9281 global_avg_mtp_loss: 15.2519 +[titan] 2025-07-09 14:26:53,953 - root - INFO - lr: 2.9945e-04 gnorm: 1.00 [ 0:51:58<21:41:27] +[titan] 2025-07-09 14:26:57,902 - root - INFO - step: 3845 loss: 18.1901 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.9481 global_avg_mtp_loss: 15.2420 +[titan] 2025-07-09 14:26:57,902 - root - INFO - lr: 2.9945e-04 gnorm: 1.05 [ 0:52:02<21:41:20] +[titan] 2025-07-09 14:27:01,013 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:27:01,854 - root - INFO - step: 3850 loss: 18.0688 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.9155 global_avg_mtp_loss: 15.1533 +[titan] 2025-07-09 14:27:01,854 - root - INFO - lr: 2.9945e-04 gnorm: 0.96 [ 0:52:06<21:41:13] +[titan] 2025-07-09 14:27:05,783 - root - INFO - step: 3855 loss: 18.3595 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.9733 global_avg_mtp_loss: 15.3862 +[titan] 2025-07-09 14:27:05,784 - root - INFO - lr: 2.9945e-04 gnorm: 0.99 [ 0:52:10<21:41:06] +[titan] 2025-07-09 14:27:09,685 - root - INFO - step: 3860 loss: 18.0974 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.9053 global_avg_mtp_loss: 15.1921 +[titan] 2025-07-09 14:27:09,685 - root - INFO - lr: 2.9944e-04 gnorm: 0.97 [ 0:52:14<21:40:58] +[titan] 2025-07-09 14:27:13,614 - root - INFO - step: 3865 loss: 18.1942 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.9366 global_avg_mtp_loss: 15.2576 +[titan] 2025-07-09 14:27:13,615 - root - INFO - lr: 2.9944e-04 gnorm: 1.01 [ 0:52:17<21:40:50] +[titan] 2025-07-09 14:27:17,542 - root - INFO - step: 3870 loss: 18.2354 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.9585 global_avg_mtp_loss: 15.2769 +[titan] 2025-07-09 14:27:17,543 - root - INFO - lr: 2.9944e-04 gnorm: 1.04 [ 0:52:21<21:40:43] +[titan] 2025-07-09 14:27:21,451 - root - INFO - step: 3875 loss: 18.3164 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.9715 global_avg_mtp_loss: 15.3449 +[titan] 2025-07-09 14:27:21,451 - root - INFO - lr: 2.9944e-04 gnorm: 0.94 [ 0:52:25<21:40:35] +[titan] 2025-07-09 14:27:25,350 - root - INFO - step: 3880 loss: 18.1568 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.9232 global_avg_mtp_loss: 15.2336 +[titan] 2025-07-09 14:27:25,351 - root - INFO - lr: 2.9944e-04 gnorm: 0.99 [ 0:52:29<21:40:27] +[titan] 2025-07-09 14:27:29,339 - root - INFO - step: 3885 loss: 18.4497 memory: 44.58GiB(31.99%) tps: 82,152 tflops: 283.52 mfu: 28.67% global_avg_ntp_loss: 2.9910 global_avg_mtp_loss: 15.4587 +[titan] 2025-07-09 14:27:29,340 - root - INFO - lr: 2.9943e-04 gnorm: 0.93 [ 0:52:33<21:40:21] +[titan] 2025-07-09 14:27:33,237 - root - INFO - step: 3890 loss: 18.2479 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.9473 global_avg_mtp_loss: 15.3007 +[titan] 2025-07-09 14:27:33,238 - root - INFO - lr: 2.9943e-04 gnorm: 0.89 [ 0:52:37<21:40:13] +[titan] 2025-07-09 14:27:37,163 - root - INFO - step: 3895 loss: 18.2589 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.9447 global_avg_mtp_loss: 15.3143 +[titan] 2025-07-09 14:27:37,163 - root - INFO - lr: 2.9943e-04 gnorm: 1.12 [ 0:52:41<21:40:06] +[titan] 2025-07-09 14:27:40,290 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:27:41,078 - root - INFO - step: 3900 loss: 18.0005 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.9191 global_avg_mtp_loss: 15.0815 +[titan] 2025-07-09 14:27:41,078 - root - INFO - lr: 2.9943e-04 gnorm: 0.99 [ 0:52:45<21:39:58] +[titan] 2025-07-09 14:27:45,000 - root - INFO - step: 3905 loss: 18.2983 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.9518 global_avg_mtp_loss: 15.3465 +[titan] 2025-07-09 14:27:45,000 - root - INFO - lr: 2.9943e-04 gnorm: 0.95 [ 0:52:49<21:39:51] +[titan] 2025-07-09 14:27:48,902 - root - INFO - step: 3910 loss: 18.2489 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.9443 global_avg_mtp_loss: 15.3046 +[titan] 2025-07-09 14:27:48,902 - root - INFO - lr: 2.9942e-04 gnorm: 1.00 [ 0:52:53<21:39:43] +[titan] 2025-07-09 14:27:52,831 - root - INFO - step: 3915 loss: 18.0291 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.9002 global_avg_mtp_loss: 15.1289 +[titan] 2025-07-09 14:27:52,831 - root - INFO - lr: 2.9942e-04 gnorm: 0.98 [ 0:52:57<21:39:36] +[titan] 2025-07-09 14:27:56,776 - root - INFO - step: 3920 loss: 18.2517 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.9391 global_avg_mtp_loss: 15.3126 +[titan] 2025-07-09 14:27:56,776 - root - INFO - lr: 2.9942e-04 gnorm: 0.92 [ 0:53:01<21:39:29] +[titan] 2025-07-09 14:28:00,717 - root - INFO - step: 3925 loss: 18.2281 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.9472 global_avg_mtp_loss: 15.2808 +[titan] 2025-07-09 14:28:00,717 - root - INFO - lr: 2.9942e-04 gnorm: 0.95 [ 0:53:05<21:39:22] +[titan] 2025-07-09 14:28:04,625 - root - INFO - step: 3930 loss: 18.5043 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.9990 global_avg_mtp_loss: 15.5053 +[titan] 2025-07-09 14:28:04,626 - root - INFO - lr: 2.9942e-04 gnorm: 0.98 [ 0:53:08<21:39:14] +[titan] 2025-07-09 14:28:08,530 - root - INFO - step: 3935 loss: 18.1134 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.9187 global_avg_mtp_loss: 15.1947 +[titan] 2025-07-09 14:28:08,530 - root - INFO - lr: 2.9941e-04 gnorm: 1.00 [ 0:53:12<21:39:07] +[titan] 2025-07-09 14:28:12,467 - root - INFO - step: 3940 loss: 18.1684 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.9417 global_avg_mtp_loss: 15.2268 +[titan] 2025-07-09 14:28:12,467 - root - INFO - lr: 2.9941e-04 gnorm: 1.02 [ 0:53:16<21:39:00] +[titan] 2025-07-09 14:28:16,407 - root - INFO - step: 3945 loss: 18.2402 memory: 44.58GiB(31.99%) tps: 83,174 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.9457 global_avg_mtp_loss: 15.2945 +[titan] 2025-07-09 14:28:16,407 - root - INFO - lr: 2.9941e-04 gnorm: 1.06 [ 0:53:20<21:38:53] +[titan] 2025-07-09 14:28:19,517 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:28:20,305 - root - INFO - step: 3950 loss: 18.2198 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.9387 global_avg_mtp_loss: 15.2811 +[titan] 2025-07-09 14:28:20,305 - root - INFO - lr: 2.9941e-04 gnorm: 1.02 [ 0:53:24<21:38:45] +[titan] 2025-07-09 14:28:24,214 - root - INFO - step: 3955 loss: 18.0377 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.9056 global_avg_mtp_loss: 15.1321 +[titan] 2025-07-09 14:28:24,214 - root - INFO - lr: 2.9941e-04 gnorm: 0.89 [ 0:53:28<21:38:37] +[titan] 2025-07-09 14:28:28,104 - root - INFO - step: 3960 loss: 18.3639 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.9874 global_avg_mtp_loss: 15.3764 +[titan] 2025-07-09 14:28:28,104 - root - INFO - lr: 2.9940e-04 gnorm: 1.04 [ 0:53:32<21:38:29] +[titan] 2025-07-09 14:28:32,024 - root - INFO - step: 3965 loss: 18.2183 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.9379 global_avg_mtp_loss: 15.2803 +[titan] 2025-07-09 14:28:32,024 - root - INFO - lr: 2.9940e-04 gnorm: 0.99 [ 0:53:36<21:38:22] +[titan] 2025-07-09 14:28:35,972 - root - INFO - step: 3970 loss: 18.0067 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.9116 global_avg_mtp_loss: 15.0951 +[titan] 2025-07-09 14:28:35,972 - root - INFO - lr: 2.9940e-04 gnorm: 1.08 [ 0:53:40<21:38:15] +[titan] 2025-07-09 14:28:39,902 - root - INFO - step: 3975 loss: 18.2950 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.9461 global_avg_mtp_loss: 15.3489 +[titan] 2025-07-09 14:28:39,903 - root - INFO - lr: 2.9940e-04 gnorm: 0.96 [ 0:53:44<21:38:08] +[titan] 2025-07-09 14:28:43,825 - root - INFO - step: 3980 loss: 18.0470 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.9062 global_avg_mtp_loss: 15.1408 +[titan] 2025-07-09 14:28:43,825 - root - INFO - lr: 2.9940e-04 gnorm: 0.92 [ 0:53:48<21:38:01] +[titan] 2025-07-09 14:28:47,743 - root - INFO - step: 3985 loss: 18.0177 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.8993 global_avg_mtp_loss: 15.1183 +[titan] 2025-07-09 14:28:47,744 - root - INFO - lr: 2.9939e-04 gnorm: 1.05 [ 0:53:52<21:37:53] +[titan] 2025-07-09 14:28:51,648 - root - INFO - step: 3990 loss: 18.0776 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.9058 global_avg_mtp_loss: 15.1719 +[titan] 2025-07-09 14:28:51,649 - root - INFO - lr: 2.9939e-04 gnorm: 0.97 [ 0:53:55<21:37:46] +[titan] 2025-07-09 14:28:55,618 - root - INFO - step: 3995 loss: 18.1557 memory: 44.58GiB(31.99%) tps: 82,548 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.9290 global_avg_mtp_loss: 15.2266 +[titan] 2025-07-09 14:28:55,619 - root - INFO - lr: 2.9939e-04 gnorm: 1.00 [ 0:53:59<21:37:39] +[titan] 2025-07-09 14:28:58,744 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:28:59,534 - root - INFO - step: 4000 loss: 18.4186 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.9711 global_avg_mtp_loss: 15.4475 +[titan] 2025-07-09 14:28:59,534 - root - INFO - lr: 2.9939e-04 gnorm: 1.01 [ 0:54:03<21:37:32] +[titan] 2025-07-09 14:29:03,443 - root - INFO - step: 4005 loss: 18.2900 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.9704 global_avg_mtp_loss: 15.3196 +[titan] 2025-07-09 14:29:03,444 - root - INFO - lr: 2.9939e-04 gnorm: 0.98 [ 0:54:07<21:37:24] +[titan] 2025-07-09 14:29:07,363 - root - INFO - step: 4010 loss: 18.1928 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.9377 global_avg_mtp_loss: 15.2552 +[titan] 2025-07-09 14:29:07,363 - root - INFO - lr: 2.9938e-04 gnorm: 1.02 [ 0:54:11<21:37:17] +[titan] 2025-07-09 14:29:11,259 - root - INFO - step: 4015 loss: 17.9567 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.9090 global_avg_mtp_loss: 15.0477 +[titan] 2025-07-09 14:29:11,259 - root - INFO - lr: 2.9938e-04 gnorm: 1.01 [ 0:54:15<21:37:09] +[titan] 2025-07-09 14:29:15,167 - root - INFO - step: 4020 loss: 18.2083 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.9343 global_avg_mtp_loss: 15.2741 +[titan] 2025-07-09 14:29:15,167 - root - INFO - lr: 2.9938e-04 gnorm: 1.04 [ 0:54:19<21:37:02] +[titan] 2025-07-09 14:29:19,083 - root - INFO - step: 4025 loss: 18.1952 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.9408 global_avg_mtp_loss: 15.2544 +[titan] 2025-07-09 14:29:19,083 - root - INFO - lr: 2.9938e-04 gnorm: 1.00 [ 0:54:23<21:36:54] +[titan] 2025-07-09 14:29:23,016 - root - INFO - step: 4030 loss: 18.1668 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.9198 global_avg_mtp_loss: 15.2469 +[titan] 2025-07-09 14:29:23,017 - root - INFO - lr: 2.9938e-04 gnorm: 0.99 [ 0:54:27<21:36:47] +[titan] 2025-07-09 14:29:26,933 - root - INFO - step: 4035 loss: 18.1461 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.9174 global_avg_mtp_loss: 15.2287 +[titan] 2025-07-09 14:29:26,933 - root - INFO - lr: 2.9937e-04 gnorm: 1.01 [ 0:54:31<21:36:40] +[titan] 2025-07-09 14:29:30,869 - root - INFO - step: 4040 loss: 18.1656 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.06% global_avg_ntp_loss: 2.9170 global_avg_mtp_loss: 15.2486 +[titan] 2025-07-09 14:29:30,869 - root - INFO - lr: 2.9937e-04 gnorm: 0.98 [ 0:54:35<21:36:33] +[titan] 2025-07-09 14:29:34,787 - root - INFO - step: 4045 loss: 18.2904 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.9637 global_avg_mtp_loss: 15.3267 +[titan] 2025-07-09 14:29:34,787 - root - INFO - lr: 2.9937e-04 gnorm: 0.99 [ 0:54:39<21:36:26] +[titan] 2025-07-09 14:29:37,951 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:29:38,759 - root - INFO - step: 4050 loss: 18.3327 memory: 44.58GiB(31.99%) tps: 82,504 tflops: 284.73 mfu: 28.79% global_avg_ntp_loss: 2.9669 global_avg_mtp_loss: 15.3658 +[titan] 2025-07-09 14:29:38,759 - root - INFO - lr: 2.9937e-04 gnorm: 0.91 [ 0:54:43<21:36:20] +[titan] 2025-07-09 14:29:42,678 - root - INFO - step: 4055 loss: 18.3946 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.9616 global_avg_mtp_loss: 15.4330 +[titan] 2025-07-09 14:29:42,678 - root - INFO - lr: 2.9937e-04 gnorm: 1.09 [ 0:54:46<21:36:13] +[titan] 2025-07-09 14:29:46,655 - root - INFO - step: 4060 loss: 18.0516 memory: 44.58GiB(31.99%) tps: 82,409 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.9003 global_avg_mtp_loss: 15.1512 +[titan] 2025-07-09 14:29:46,655 - root - INFO - lr: 2.9936e-04 gnorm: 0.99 [ 0:54:50<21:36:07] +[titan] 2025-07-09 14:29:50,581 - root - INFO - step: 4065 loss: 18.1893 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.9268 global_avg_mtp_loss: 15.2626 +[titan] 2025-07-09 14:29:50,581 - root - INFO - lr: 2.9936e-04 gnorm: 0.96 [ 0:54:54<21:36:00] +[titan] 2025-07-09 14:29:54,508 - root - INFO - step: 4070 loss: 17.9432 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.8688 global_avg_mtp_loss: 15.0744 +[titan] 2025-07-09 14:29:54,508 - root - INFO - lr: 2.9936e-04 gnorm: 0.98 [ 0:54:58<21:35:53] +[titan] 2025-07-09 14:29:58,462 - root - INFO - step: 4075 loss: 18.1089 memory: 44.58GiB(31.99%) tps: 82,887 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.9234 global_avg_mtp_loss: 15.1855 +[titan] 2025-07-09 14:29:58,462 - root - INFO - lr: 2.9936e-04 gnorm: 0.93 [ 0:55:02<21:35:46] +[titan] 2025-07-09 14:30:02,414 - root - INFO - step: 4080 loss: 18.3437 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.9714 global_avg_mtp_loss: 15.3723 +[titan] 2025-07-09 14:30:02,414 - root - INFO - lr: 2.9936e-04 gnorm: 0.95 [ 0:55:06<21:35:40] +[titan] 2025-07-09 14:30:06,317 - root - INFO - step: 4085 loss: 18.2696 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.9382 global_avg_mtp_loss: 15.3315 +[titan] 2025-07-09 14:30:06,318 - root - INFO - lr: 2.9935e-04 gnorm: 1.01 [ 0:55:10<21:35:32] +[titan] 2025-07-09 14:30:10,241 - root - INFO - step: 4090 loss: 18.1469 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.9177 global_avg_mtp_loss: 15.2293 +[titan] 2025-07-09 14:30:10,241 - root - INFO - lr: 2.9935e-04 gnorm: 0.91 [ 0:55:14<21:35:25] +[titan] 2025-07-09 14:30:14,225 - root - INFO - step: 4095 loss: 18.3371 memory: 44.58GiB(31.99%) tps: 82,262 tflops: 283.90 mfu: 28.71% global_avg_ntp_loss: 2.9648 global_avg_mtp_loss: 15.3723 +[titan] 2025-07-09 14:30:14,225 - root - INFO - lr: 2.9935e-04 gnorm: 0.93 [ 0:55:18<21:35:20] +[titan] 2025-07-09 14:30:15,144 - root - INFO - Dumping profiler traces at step 4096 +[titan] 2025-07-09 14:30:15,176 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:30:17,518 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:30:18,304 - root - INFO - step: 4100 loss: 18.3777 memory: 44.58GiB(31.99%) tps: 80,331 tflops: 277.24 mfu: 28.03% global_avg_ntp_loss: 2.9789 global_avg_mtp_loss: 15.3988 +[titan] 2025-07-09 14:30:18,305 - root - INFO - lr: 2.9935e-04 gnorm: 0.91 [ 0:55:22<21:35:16] +[titan] 2025-07-09 14:30:22,211 - root - INFO - step: 4105 loss: 18.2074 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.9297 global_avg_mtp_loss: 15.2778 +[titan] 2025-07-09 14:30:22,211 - root - INFO - lr: 2.9934e-04 gnorm: 0.98 [ 0:55:26<21:35:09] +[titan] 2025-07-09 14:30:26,137 - root - INFO - step: 4110 loss: 18.3855 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.9774 global_avg_mtp_loss: 15.4082 +[titan] 2025-07-09 14:30:26,137 - root - INFO - lr: 2.9934e-04 gnorm: 0.96 [ 0:55:30<21:35:02] +[titan] 2025-07-09 14:30:30,055 - root - INFO - step: 4115 loss: 18.2064 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.9294 global_avg_mtp_loss: 15.2770 +[titan] 2025-07-09 14:30:30,056 - root - INFO - lr: 2.9934e-04 gnorm: 1.03 [ 0:55:34<21:34:55] +[titan] 2025-07-09 14:30:33,955 - root - INFO - step: 4120 loss: 18.0708 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.9194 global_avg_mtp_loss: 15.1514 +[titan] 2025-07-09 14:30:33,956 - root - INFO - lr: 2.9934e-04 gnorm: 1.11 [ 0:55:38<21:34:47] +[titan] 2025-07-09 14:30:37,857 - root - INFO - step: 4125 loss: 18.4791 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 3.0057 global_avg_mtp_loss: 15.4734 +[titan] 2025-07-09 14:30:37,857 - root - INFO - lr: 2.9934e-04 gnorm: 1.03 [ 0:55:42<21:34:40] +[titan] 2025-07-09 14:30:41,784 - root - INFO - step: 4130 loss: 18.1343 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.9190 global_avg_mtp_loss: 15.2152 +[titan] 2025-07-09 14:30:41,785 - root - INFO - lr: 2.9933e-04 gnorm: 0.97 [ 0:55:46<21:34:33] +[titan] 2025-07-09 14:30:45,699 - root - INFO - step: 4135 loss: 18.1465 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.9155 global_avg_mtp_loss: 15.2310 +[titan] 2025-07-09 14:30:45,699 - root - INFO - lr: 2.9933e-04 gnorm: 0.96 [ 0:55:50<21:34:25] +[titan] 2025-07-09 14:30:49,629 - root - INFO - step: 4140 loss: 18.1748 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.9381 global_avg_mtp_loss: 15.2368 +[titan] 2025-07-09 14:30:49,630 - root - INFO - lr: 2.9933e-04 gnorm: 0.99 [ 0:55:53<21:34:19] +[titan] 2025-07-09 14:30:53,618 - root - INFO - step: 4145 loss: 18.2552 memory: 44.58GiB(31.99%) tps: 82,163 tflops: 283.56 mfu: 28.67% global_avg_ntp_loss: 2.9379 global_avg_mtp_loss: 15.3173 +[titan] 2025-07-09 14:30:53,618 - root - INFO - lr: 2.9933e-04 gnorm: 0.98 [ 0:55:57<21:34:13] +[titan] 2025-07-09 14:30:56,756 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:30:57,555 - root - INFO - step: 4150 loss: 18.2838 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.9610 global_avg_mtp_loss: 15.3228 +[titan] 2025-07-09 14:30:57,556 - root - INFO - lr: 2.9933e-04 gnorm: 1.15 [ 0:56:01<21:34:06] +[titan] 2025-07-09 14:31:01,464 - root - INFO - step: 4155 loss: 18.0810 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.9063 global_avg_mtp_loss: 15.1747 +[titan] 2025-07-09 14:31:01,464 - root - INFO - lr: 2.9932e-04 gnorm: 0.90 [ 0:56:05<21:33:59] +[titan] 2025-07-09 14:31:05,370 - root - INFO - step: 4160 loss: 18.0311 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.9038 global_avg_mtp_loss: 15.1273 +[titan] 2025-07-09 14:31:05,370 - root - INFO - lr: 2.9932e-04 gnorm: 1.05 [ 0:56:09<21:33:52] +[titan] 2025-07-09 14:31:09,298 - root - INFO - step: 4165 loss: 18.1228 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.9159 global_avg_mtp_loss: 15.2069 +[titan] 2025-07-09 14:31:09,298 - root - INFO - lr: 2.9932e-04 gnorm: 0.98 [ 0:56:13<21:33:45] +[titan] 2025-07-09 14:31:13,281 - root - INFO - step: 4170 loss: 18.2560 memory: 44.58GiB(31.99%) tps: 82,279 tflops: 283.96 mfu: 28.71% global_avg_ntp_loss: 2.9549 global_avg_mtp_loss: 15.3011 +[titan] 2025-07-09 14:31:13,281 - root - INFO - lr: 2.9932e-04 gnorm: 0.97 [ 0:56:17<21:33:39] +[titan] 2025-07-09 14:31:17,186 - root - INFO - step: 4175 loss: 18.2315 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.9463 global_avg_mtp_loss: 15.2852 +[titan] 2025-07-09 14:31:17,187 - root - INFO - lr: 2.9931e-04 gnorm: 0.92 [ 0:56:21<21:33:32] +[titan] 2025-07-09 14:31:21,083 - root - INFO - step: 4180 loss: 18.1198 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.9139 global_avg_mtp_loss: 15.2059 +[titan] 2025-07-09 14:31:21,084 - root - INFO - lr: 2.9931e-04 gnorm: 0.99 [ 0:56:25<21:33:24] +[titan] 2025-07-09 14:31:24,991 - root - INFO - step: 4185 loss: 18.0464 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.9065 global_avg_mtp_loss: 15.1399 +[titan] 2025-07-09 14:31:24,991 - root - INFO - lr: 2.9931e-04 gnorm: 0.98 [ 0:56:29<21:33:17] +[titan] 2025-07-09 14:31:28,940 - root - INFO - step: 4190 loss: 18.1344 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.9164 global_avg_mtp_loss: 15.2180 +[titan] 2025-07-09 14:31:28,940 - root - INFO - lr: 2.9931e-04 gnorm: 0.91 [ 0:56:33<21:33:11] +[titan] 2025-07-09 14:31:32,850 - root - INFO - step: 4195 loss: 17.9898 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.8900 global_avg_mtp_loss: 15.0998 +[titan] 2025-07-09 14:31:32,850 - root - INFO - lr: 2.9931e-04 gnorm: 0.85 [ 0:56:37<21:33:03] +[titan] 2025-07-09 14:31:35,972 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:31:36,774 - root - INFO - step: 4200 loss: 18.0204 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.8874 global_avg_mtp_loss: 15.1330 +[titan] 2025-07-09 14:31:36,774 - root - INFO - lr: 2.9930e-04 gnorm: 0.95 [ 0:56:41<21:32:56] +[titan] 2025-07-09 14:31:40,716 - root - INFO - step: 4205 loss: 18.2115 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.9383 global_avg_mtp_loss: 15.2731 +[titan] 2025-07-09 14:31:40,716 - root - INFO - lr: 2.9930e-04 gnorm: 0.94 [ 0:56:45<21:32:50] +[titan] 2025-07-09 14:31:44,641 - root - INFO - step: 4210 loss: 17.9797 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.8904 global_avg_mtp_loss: 15.0893 +[titan] 2025-07-09 14:31:44,641 - root - INFO - lr: 2.9930e-04 gnorm: 1.38 [ 0:56:48<21:32:43] +[titan] 2025-07-09 14:31:48,565 - root - INFO - step: 4215 loss: 18.1548 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.9215 global_avg_mtp_loss: 15.2333 +[titan] 2025-07-09 14:31:48,566 - root - INFO - lr: 2.9930e-04 gnorm: 0.97 [ 0:56:52<21:32:36] +[titan] 2025-07-09 14:31:52,488 - root - INFO - step: 4220 loss: 18.2152 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.9242 global_avg_mtp_loss: 15.2910 +[titan] 2025-07-09 14:31:52,488 - root - INFO - lr: 2.9930e-04 gnorm: 0.96 [ 0:56:56<21:32:29] +[titan] 2025-07-09 14:31:56,415 - root - INFO - step: 4225 loss: 17.8210 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.8517 global_avg_mtp_loss: 14.9693 +[titan] 2025-07-09 14:31:56,415 - root - INFO - lr: 2.9929e-04 gnorm: 1.08 [ 0:57:00<21:32:22] +[titan] 2025-07-09 14:32:00,364 - root - INFO - step: 4230 loss: 18.0992 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.9124 global_avg_mtp_loss: 15.1868 +[titan] 2025-07-09 14:32:00,364 - root - INFO - lr: 2.9929e-04 gnorm: 0.90 [ 0:57:04<21:32:16] +[titan] 2025-07-09 14:32:04,283 - root - INFO - step: 4235 loss: 17.9108 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.8731 global_avg_mtp_loss: 15.0377 +[titan] 2025-07-09 14:32:04,283 - root - INFO - lr: 2.9929e-04 gnorm: 0.93 [ 0:57:08<21:32:09] +[titan] 2025-07-09 14:32:08,204 - root - INFO - step: 4240 loss: 18.0359 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.9037 global_avg_mtp_loss: 15.1322 +[titan] 2025-07-09 14:32:08,204 - root - INFO - lr: 2.9929e-04 gnorm: 0.96 [ 0:57:12<21:32:02] +[titan] 2025-07-09 14:32:12,126 - root - INFO - step: 4245 loss: 18.2691 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.9399 global_avg_mtp_loss: 15.3292 +[titan] 2025-07-09 14:32:12,126 - root - INFO - lr: 2.9928e-04 gnorm: 0.87 [ 0:57:16<21:31:55] +[titan] 2025-07-09 14:32:15,235 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:32:16,036 - root - INFO - step: 4250 loss: 17.9956 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.8968 global_avg_mtp_loss: 15.0989 +[titan] 2025-07-09 14:32:16,037 - root - INFO - lr: 2.9928e-04 gnorm: 0.89 [ 0:57:20<21:31:48] +[titan] 2025-07-09 14:32:19,967 - root - INFO - step: 4255 loss: 18.0864 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.9184 global_avg_mtp_loss: 15.1679 +[titan] 2025-07-09 14:32:19,967 - root - INFO - lr: 2.9928e-04 gnorm: 1.01 [ 0:57:24<21:31:41] +[titan] 2025-07-09 14:32:23,871 - root - INFO - step: 4260 loss: 18.4309 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.9846 global_avg_mtp_loss: 15.4463 +[titan] 2025-07-09 14:32:23,872 - root - INFO - lr: 2.9928e-04 gnorm: 0.92 [ 0:57:28<21:31:34] +[titan] 2025-07-09 14:32:27,793 - root - INFO - step: 4265 loss: 18.2444 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.9382 global_avg_mtp_loss: 15.3063 +[titan] 2025-07-09 14:32:27,794 - root - INFO - lr: 2.9928e-04 gnorm: 0.95 [ 0:57:32<21:31:27] +[titan] 2025-07-09 14:32:31,707 - root - INFO - step: 4270 loss: 18.2467 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.9388 global_avg_mtp_loss: 15.3079 +[titan] 2025-07-09 14:32:31,707 - root - INFO - lr: 2.9927e-04 gnorm: 1.05 [ 0:57:36<21:31:20] +[titan] 2025-07-09 14:32:35,597 - root - INFO - step: 4275 loss: 18.0891 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.9152 global_avg_mtp_loss: 15.1738 +[titan] 2025-07-09 14:32:35,597 - root - INFO - lr: 2.9927e-04 gnorm: 1.06 [ 0:57:39<21:31:13] +[titan] 2025-07-09 14:32:39,506 - root - INFO - step: 4280 loss: 18.2466 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.9238 global_avg_mtp_loss: 15.3229 +[titan] 2025-07-09 14:32:39,506 - root - INFO - lr: 2.9927e-04 gnorm: 0.96 [ 0:57:43<21:31:06] +[titan] 2025-07-09 14:32:43,421 - root - INFO - step: 4285 loss: 18.2437 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.9355 global_avg_mtp_loss: 15.3083 +[titan] 2025-07-09 14:32:43,421 - root - INFO - lr: 2.9927e-04 gnorm: 0.95 [ 0:57:47<21:30:59] +[titan] 2025-07-09 14:32:47,333 - root - INFO - step: 4290 loss: 18.1716 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.9392 global_avg_mtp_loss: 15.2325 +[titan] 2025-07-09 14:32:47,333 - root - INFO - lr: 2.9926e-04 gnorm: 0.95 [ 0:57:51<21:30:52] +[titan] 2025-07-09 14:32:51,248 - root - INFO - step: 4295 loss: 17.9815 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.8951 global_avg_mtp_loss: 15.0864 +[titan] 2025-07-09 14:32:51,249 - root - INFO - lr: 2.9926e-04 gnorm: 1.25 [ 0:57:55<21:30:45] +[titan] 2025-07-09 14:32:54,394 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:32:55,200 - root - INFO - step: 4300 loss: 18.3032 memory: 44.58GiB(31.99%) tps: 82,936 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.9535 global_avg_mtp_loss: 15.3497 +[titan] 2025-07-09 14:32:55,200 - root - INFO - lr: 2.9926e-04 gnorm: 0.91 [ 0:57:59<21:30:38] +[titan] 2025-07-09 14:32:59,135 - root - INFO - step: 4305 loss: 18.0330 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.9089 global_avg_mtp_loss: 15.1241 +[titan] 2025-07-09 14:32:59,135 - root - INFO - lr: 2.9926e-04 gnorm: 0.94 [ 0:58:03<21:30:32] +[titan] 2025-07-09 14:33:03,060 - root - INFO - step: 4310 loss: 18.5386 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 3.0039 global_avg_mtp_loss: 15.5347 +[titan] 2025-07-09 14:33:03,060 - root - INFO - lr: 2.9926e-04 gnorm: 1.05 [ 0:58:07<21:30:25] +[titan] 2025-07-09 14:33:06,980 - root - INFO - step: 4315 loss: 18.1866 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.9400 global_avg_mtp_loss: 15.2466 +[titan] 2025-07-09 14:33:06,980 - root - INFO - lr: 2.9925e-04 gnorm: 1.06 [ 0:58:11<21:30:18] +[titan] 2025-07-09 14:33:10,912 - root - INFO - step: 4320 loss: 18.2503 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.9302 global_avg_mtp_loss: 15.3200 +[titan] 2025-07-09 14:33:10,912 - root - INFO - lr: 2.9925e-04 gnorm: 1.01 [ 0:58:15<21:30:12] +[titan] 2025-07-09 14:33:14,849 - root - INFO - step: 4325 loss: 18.0208 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.8984 global_avg_mtp_loss: 15.1224 +[titan] 2025-07-09 14:33:14,849 - root - INFO - lr: 2.9925e-04 gnorm: 0.99 [ 0:58:19<21:30:05] +[titan] 2025-07-09 14:33:18,756 - root - INFO - step: 4330 loss: 18.2436 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.9302 global_avg_mtp_loss: 15.3134 +[titan] 2025-07-09 14:33:18,756 - root - INFO - lr: 2.9925e-04 gnorm: 1.08 [ 0:58:23<21:29:58] +[titan] 2025-07-09 14:33:22,650 - root - INFO - step: 4335 loss: 17.8291 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.8736 global_avg_mtp_loss: 14.9554 +[titan] 2025-07-09 14:33:22,650 - root - INFO - lr: 2.9924e-04 gnorm: 0.94 [ 0:58:26<21:29:51] +[titan] 2025-07-09 14:33:26,564 - root - INFO - step: 4340 loss: 18.0565 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.9115 global_avg_mtp_loss: 15.1450 +[titan] 2025-07-09 14:33:26,564 - root - INFO - lr: 2.9924e-04 gnorm: 1.00 [ 0:58:30<21:29:44] +[titan] 2025-07-09 14:33:30,485 - root - INFO - step: 4345 loss: 17.9476 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.8923 global_avg_mtp_loss: 15.0553 +[titan] 2025-07-09 14:33:30,485 - root - INFO - lr: 2.9924e-04 gnorm: 1.10 [ 0:58:34<21:29:37] +[titan] 2025-07-09 14:33:33,599 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:33:34,386 - root - INFO - step: 4350 loss: 18.2510 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.9430 global_avg_mtp_loss: 15.3079 +[titan] 2025-07-09 14:33:34,387 - root - INFO - lr: 2.9924e-04 gnorm: 0.98 [ 0:58:38<21:29:30] +[titan] 2025-07-09 14:33:38,344 - root - INFO - step: 4355 loss: 18.0177 memory: 44.58GiB(31.99%) tps: 82,801 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.8942 global_avg_mtp_loss: 15.1235 +[titan] 2025-07-09 14:33:38,344 - root - INFO - lr: 2.9924e-04 gnorm: 1.03 [ 0:58:42<21:29:24] +[titan] 2025-07-09 14:33:42,276 - root - INFO - step: 4360 loss: 17.9554 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.8765 global_avg_mtp_loss: 15.0789 +[titan] 2025-07-09 14:33:42,277 - root - INFO - lr: 2.9923e-04 gnorm: 0.98 [ 0:58:46<21:29:17] +[titan] 2025-07-09 14:33:46,222 - root - INFO - step: 4365 loss: 18.0413 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.9007 global_avg_mtp_loss: 15.1406 +[titan] 2025-07-09 14:33:46,223 - root - INFO - lr: 2.9923e-04 gnorm: 1.02 [ 0:58:50<21:29:11] +[titan] 2025-07-09 14:33:50,155 - root - INFO - step: 4370 loss: 18.1890 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.9367 global_avg_mtp_loss: 15.2523 +[titan] 2025-07-09 14:33:50,155 - root - INFO - lr: 2.9923e-04 gnorm: 1.07 [ 0:58:54<21:29:05] +[titan] 2025-07-09 14:33:54,117 - root - INFO - step: 4375 loss: 18.1156 memory: 44.58GiB(31.99%) tps: 82,711 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.9097 global_avg_mtp_loss: 15.2059 +[titan] 2025-07-09 14:33:54,117 - root - INFO - lr: 2.9923e-04 gnorm: 1.00 [ 0:58:58<21:28:59] +[titan] 2025-07-09 14:33:58,048 - root - INFO - step: 4380 loss: 18.0645 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.9072 global_avg_mtp_loss: 15.1573 +[titan] 2025-07-09 14:33:58,048 - root - INFO - lr: 2.9922e-04 gnorm: 0.97 [ 0:59:02<21:28:52] +[titan] 2025-07-09 14:34:01,999 - root - INFO - step: 4385 loss: 18.1871 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.9337 global_avg_mtp_loss: 15.2534 +[titan] 2025-07-09 14:34:01,999 - root - INFO - lr: 2.9922e-04 gnorm: 1.04 [ 0:59:06<21:28:46] +[titan] 2025-07-09 14:34:05,927 - root - INFO - step: 4390 loss: 18.1505 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.9193 global_avg_mtp_loss: 15.2312 +[titan] 2025-07-09 14:34:05,928 - root - INFO - lr: 2.9922e-04 gnorm: 1.00 [ 0:59:10<21:28:40] +[titan] 2025-07-09 14:34:09,841 - root - INFO - step: 4395 loss: 18.0242 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.9062 global_avg_mtp_loss: 15.1180 +[titan] 2025-07-09 14:34:09,842 - root - INFO - lr: 2.9922e-04 gnorm: 1.07 [ 0:59:14<21:28:33] +[titan] 2025-07-09 14:34:12,975 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:34:13,767 - root - INFO - step: 4400 loss: 18.1366 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.9026 global_avg_mtp_loss: 15.2339 +[titan] 2025-07-09 14:34:13,767 - root - INFO - lr: 2.9921e-04 gnorm: 0.92 [ 0:59:18<21:28:26] +[titan] 2025-07-09 14:34:17,670 - root - INFO - step: 4405 loss: 17.7957 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.8612 global_avg_mtp_loss: 14.9345 +[titan] 2025-07-09 14:34:17,671 - root - INFO - lr: 2.9921e-04 gnorm: 0.98 [ 0:59:21<21:28:19] +[titan] 2025-07-09 14:34:21,599 - root - INFO - step: 4410 loss: 17.9882 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.8989 global_avg_mtp_loss: 15.0893 +[titan] 2025-07-09 14:34:21,599 - root - INFO - lr: 2.9921e-04 gnorm: 1.08 [ 0:59:25<21:28:13] +[titan] 2025-07-09 14:34:25,536 - root - INFO - step: 4415 loss: 18.2321 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.9536 global_avg_mtp_loss: 15.2784 +[titan] 2025-07-09 14:34:25,536 - root - INFO - lr: 2.9921e-04 gnorm: 0.98 [ 0:59:29<21:28:06] +[titan] 2025-07-09 14:34:29,477 - root - INFO - step: 4420 loss: 18.0021 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.8807 global_avg_mtp_loss: 15.1214 +[titan] 2025-07-09 14:34:29,477 - root - INFO - lr: 2.9921e-04 gnorm: 1.01 [ 0:59:33<21:28:00] +[titan] 2025-07-09 14:34:33,387 - root - INFO - step: 4425 loss: 17.9180 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.8741 global_avg_mtp_loss: 15.0439 +[titan] 2025-07-09 14:34:33,387 - root - INFO - lr: 2.9920e-04 gnorm: 0.88 [ 0:59:37<21:27:53] +[titan] 2025-07-09 14:34:37,310 - root - INFO - step: 4430 loss: 17.8843 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.8793 global_avg_mtp_loss: 15.0050 +[titan] 2025-07-09 14:34:37,311 - root - INFO - lr: 2.9920e-04 gnorm: 1.01 [ 0:59:41<21:27:46] +[titan] 2025-07-09 14:34:41,213 - root - INFO - step: 4435 loss: 18.0578 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.8972 global_avg_mtp_loss: 15.1606 +[titan] 2025-07-09 14:34:41,213 - root - INFO - lr: 2.9920e-04 gnorm: 0.96 [ 0:59:45<21:27:39] +[titan] 2025-07-09 14:34:45,156 - root - INFO - step: 4440 loss: 18.1991 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.9243 global_avg_mtp_loss: 15.2749 +[titan] 2025-07-09 14:34:45,156 - root - INFO - lr: 2.9920e-04 gnorm: 0.98 [ 0:59:49<21:27:33] +[titan] 2025-07-09 14:34:49,068 - root - INFO - step: 4445 loss: 18.2325 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.9210 global_avg_mtp_loss: 15.3115 +[titan] 2025-07-09 14:34:49,068 - root - INFO - lr: 2.9919e-04 gnorm: 1.02 [ 0:59:53<21:27:26] +[titan] 2025-07-09 14:34:52,195 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:34:52,988 - root - INFO - step: 4450 loss: 17.9765 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.8945 global_avg_mtp_loss: 15.0821 +[titan] 2025-07-09 14:34:52,988 - root - INFO - lr: 2.9919e-04 gnorm: 1.00 [ 0:59:57<21:27:20] +[titan] 2025-07-09 14:34:56,950 - root - INFO - step: 4455 loss: 18.1046 memory: 44.58GiB(31.99%) tps: 82,728 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.9250 global_avg_mtp_loss: 15.1796 +[titan] 2025-07-09 14:34:56,950 - root - INFO - lr: 2.9919e-04 gnorm: 1.07 [ 1:00:01<21:27:14] +[titan] 2025-07-09 14:35:00,865 - root - INFO - step: 4460 loss: 17.9795 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.8812 global_avg_mtp_loss: 15.0983 +[titan] 2025-07-09 14:35:00,865 - root - INFO - lr: 2.9919e-04 gnorm: 1.15 [ 1:00:05<21:27:07] +[titan] 2025-07-09 14:35:04,780 - root - INFO - step: 4465 loss: 18.0881 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.8919 global_avg_mtp_loss: 15.1963 +[titan] 2025-07-09 14:35:04,780 - root - INFO - lr: 2.9918e-04 gnorm: 0.95 [ 1:00:09<21:27:00] +[titan] 2025-07-09 14:35:08,676 - root - INFO - step: 4470 loss: 18.1751 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.9281 global_avg_mtp_loss: 15.2470 +[titan] 2025-07-09 14:35:08,677 - root - INFO - lr: 2.9918e-04 gnorm: 1.02 [ 1:00:12<21:26:53] +[titan] 2025-07-09 14:35:12,582 - root - INFO - step: 4475 loss: 18.2265 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.9343 global_avg_mtp_loss: 15.2922 +[titan] 2025-07-09 14:35:12,582 - root - INFO - lr: 2.9918e-04 gnorm: 0.99 [ 1:00:16<21:26:46] +[titan] 2025-07-09 14:35:16,491 - root - INFO - step: 4480 loss: 18.0484 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.9010 global_avg_mtp_loss: 15.1474 +[titan] 2025-07-09 14:35:16,491 - root - INFO - lr: 2.9918e-04 gnorm: 0.88 [ 1:00:20<21:26:39] +[titan] 2025-07-09 14:35:20,392 - root - INFO - step: 4485 loss: 17.8404 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.8569 global_avg_mtp_loss: 14.9835 +[titan] 2025-07-09 14:35:20,392 - root - INFO - lr: 2.9917e-04 gnorm: 1.07 [ 1:00:24<21:26:32] +[titan] 2025-07-09 14:35:24,292 - root - INFO - step: 4490 loss: 18.0652 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.8999 global_avg_mtp_loss: 15.1653 +[titan] 2025-07-09 14:35:24,292 - root - INFO - lr: 2.9917e-04 gnorm: 1.02 [ 1:00:28<21:26:25] +[titan] 2025-07-09 14:35:28,252 - root - INFO - step: 4495 loss: 17.9361 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.8838 global_avg_mtp_loss: 15.0522 +[titan] 2025-07-09 14:35:28,252 - root - INFO - lr: 2.9917e-04 gnorm: 0.97 [ 1:00:32<21:26:20] +[titan] 2025-07-09 14:35:31,391 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:35:32,174 - root - INFO - step: 4500 loss: 18.2371 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.9339 global_avg_mtp_loss: 15.3032 +[titan] 2025-07-09 14:35:32,175 - root - INFO - lr: 2.9917e-04 gnorm: 1.08 [ 1:00:36<21:26:13] +[titan] 2025-07-09 14:35:36,065 - root - INFO - step: 4505 loss: 18.1035 memory: 44.58GiB(31.99%) tps: 84,232 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.9073 global_avg_mtp_loss: 15.1962 +[titan] 2025-07-09 14:35:36,065 - root - INFO - lr: 2.9917e-04 gnorm: 1.01 [ 1:00:40<21:26:06] +[titan] 2025-07-09 14:35:39,960 - root - INFO - step: 4510 loss: 18.0292 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.8792 global_avg_mtp_loss: 15.1500 +[titan] 2025-07-09 14:35:39,960 - root - INFO - lr: 2.9916e-04 gnorm: 1.03 [ 1:00:44<21:25:59] +[titan] 2025-07-09 14:35:43,863 - root - INFO - step: 4515 loss: 18.1423 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.8997 global_avg_mtp_loss: 15.2427 +[titan] 2025-07-09 14:35:43,863 - root - INFO - lr: 2.9916e-04 gnorm: 0.91 [ 1:00:48<21:25:52] +[titan] 2025-07-09 14:35:47,764 - root - INFO - step: 4520 loss: 18.1795 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.9226 global_avg_mtp_loss: 15.2569 +[titan] 2025-07-09 14:35:47,764 - root - INFO - lr: 2.9916e-04 gnorm: 0.86 [ 1:00:52<21:25:45] +[titan] 2025-07-09 14:35:51,692 - root - INFO - step: 4525 loss: 18.1286 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.9082 global_avg_mtp_loss: 15.2204 +[titan] 2025-07-09 14:35:51,693 - root - INFO - lr: 2.9916e-04 gnorm: 0.92 [ 1:00:55<21:25:38] +[titan] 2025-07-09 14:35:55,672 - root - INFO - step: 4530 loss: 18.0296 memory: 44.58GiB(31.99%) tps: 82,345 tflops: 284.19 mfu: 28.73% global_avg_ntp_loss: 2.8822 global_avg_mtp_loss: 15.1474 +[titan] 2025-07-09 14:35:55,672 - root - INFO - lr: 2.9915e-04 gnorm: 0.95 [ 1:00:59<21:25:33] +[titan] 2025-07-09 14:35:59,580 - root - INFO - step: 4535 loss: 18.2179 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.9509 global_avg_mtp_loss: 15.2670 +[titan] 2025-07-09 14:35:59,580 - root - INFO - lr: 2.9915e-04 gnorm: 1.08 [ 1:01:03<21:25:26] +[titan] 2025-07-09 14:36:03,505 - root - INFO - step: 4540 loss: 18.0857 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.9149 global_avg_mtp_loss: 15.1708 +[titan] 2025-07-09 14:36:03,505 - root - INFO - lr: 2.9915e-04 gnorm: 1.09 [ 1:01:07<21:25:20] +[titan] 2025-07-09 14:36:07,409 - root - INFO - step: 4545 loss: 17.9022 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.8634 global_avg_mtp_loss: 15.0387 +[titan] 2025-07-09 14:36:07,410 - root - INFO - lr: 2.9915e-04 gnorm: 0.98 [ 1:01:11<21:25:13] +[titan] 2025-07-09 14:36:10,546 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:36:11,338 - root - INFO - step: 4550 loss: 18.0267 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.8919 global_avg_mtp_loss: 15.1349 +[titan] 2025-07-09 14:36:11,339 - root - INFO - lr: 2.9914e-04 gnorm: 0.94 [ 1:01:15<21:25:06] +[titan] 2025-07-09 14:36:15,256 - root - INFO - step: 4555 loss: 18.1558 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.9133 global_avg_mtp_loss: 15.2424 +[titan] 2025-07-09 14:36:15,256 - root - INFO - lr: 2.9914e-04 gnorm: 0.95 [ 1:01:19<21:25:00] +[titan] 2025-07-09 14:36:19,155 - root - INFO - step: 4560 loss: 17.7908 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.8399 global_avg_mtp_loss: 14.9510 +[titan] 2025-07-09 14:36:19,155 - root - INFO - lr: 2.9914e-04 gnorm: 1.00 [ 1:01:23<21:24:53] +[titan] 2025-07-09 14:36:23,137 - root - INFO - step: 4565 loss: 17.9797 memory: 44.58GiB(31.99%) tps: 82,296 tflops: 284.02 mfu: 28.72% global_avg_ntp_loss: 2.8851 global_avg_mtp_loss: 15.0945 +[titan] 2025-07-09 14:36:23,137 - root - INFO - lr: 2.9914e-04 gnorm: 0.92 [ 1:01:27<21:24:48] +[titan] 2025-07-09 14:36:27,039 - root - INFO - step: 4570 loss: 18.1387 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.9126 global_avg_mtp_loss: 15.2262 +[titan] 2025-07-09 14:36:27,040 - root - INFO - lr: 2.9913e-04 gnorm: 0.97 [ 1:01:31<21:24:41] +[titan] 2025-07-09 14:36:30,938 - root - INFO - step: 4575 loss: 18.3742 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.9799 global_avg_mtp_loss: 15.3943 +[titan] 2025-07-09 14:36:30,938 - root - INFO - lr: 2.9913e-04 gnorm: 0.89 [ 1:01:35<21:24:34] +[titan] 2025-07-09 14:36:34,889 - root - INFO - step: 4580 loss: 17.8161 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.8509 global_avg_mtp_loss: 14.9653 +[titan] 2025-07-09 14:36:34,889 - root - INFO - lr: 2.9913e-04 gnorm: 1.06 [ 1:01:39<21:24:28] +[titan] 2025-07-09 14:36:38,825 - root - INFO - step: 4585 loss: 18.0691 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.9035 global_avg_mtp_loss: 15.1656 +[titan] 2025-07-09 14:36:38,826 - root - INFO - lr: 2.9913e-04 gnorm: 0.88 [ 1:01:43<21:24:22] +[titan] 2025-07-09 14:36:42,764 - root - INFO - step: 4590 loss: 17.7768 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.8425 global_avg_mtp_loss: 14.9343 +[titan] 2025-07-09 14:36:42,765 - root - INFO - lr: 2.9912e-04 gnorm: 0.93 [ 1:01:47<21:24:16] +[titan] 2025-07-09 14:36:46,691 - root - INFO - step: 4595 loss: 18.0537 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.8983 global_avg_mtp_loss: 15.1554 +[titan] 2025-07-09 14:36:46,691 - root - INFO - lr: 2.9912e-04 gnorm: 1.01 [ 1:01:50<21:24:09] +[titan] 2025-07-09 14:36:49,826 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:36:50,616 - root - INFO - step: 4600 loss: 17.9251 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.8756 global_avg_mtp_loss: 15.0495 +[titan] 2025-07-09 14:36:50,616 - root - INFO - lr: 2.9912e-04 gnorm: 0.98 [ 1:01:54<21:24:03] +[titan] 2025-07-09 14:36:54,571 - root - INFO - step: 4605 loss: 17.8966 memory: 44.58GiB(31.99%) tps: 82,852 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.8635 global_avg_mtp_loss: 15.0331 +[titan] 2025-07-09 14:36:54,572 - root - INFO - lr: 2.9912e-04 gnorm: 0.99 [ 1:01:58<21:23:57] +[titan] 2025-07-09 14:36:57,061 - root - INFO - Dumping profiler traces at step 4608 +[titan] 2025-07-09 14:36:57,093 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:36:58,667 - root - INFO - step: 4610 loss: 17.7684 memory: 44.58GiB(31.99%) tps: 80,021 tflops: 276.16 mfu: 27.92% global_avg_ntp_loss: 2.8376 global_avg_mtp_loss: 14.9309 +[titan] 2025-07-09 14:36:58,667 - root - INFO - lr: 2.9911e-04 gnorm: 0.93 [ 1:02:02<21:23:54] +[titan] 2025-07-09 14:37:02,601 - root - INFO - step: 4615 loss: 17.9029 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.8678 global_avg_mtp_loss: 15.0351 +[titan] 2025-07-09 14:37:02,602 - root - INFO - lr: 2.9911e-04 gnorm: 1.00 [ 1:02:06<21:23:48] +[titan] 2025-07-09 14:37:06,509 - root - INFO - step: 4620 loss: 18.4081 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.9644 global_avg_mtp_loss: 15.4437 +[titan] 2025-07-09 14:37:06,509 - root - INFO - lr: 2.9911e-04 gnorm: 1.11 [ 1:02:10<21:23:41] +[titan] 2025-07-09 14:37:10,417 - root - INFO - step: 4625 loss: 18.0162 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.8843 global_avg_mtp_loss: 15.1318 +[titan] 2025-07-09 14:37:10,417 - root - INFO - lr: 2.9911e-04 gnorm: 0.98 [ 1:02:14<21:23:35] +[titan] 2025-07-09 14:37:14,333 - root - INFO - step: 4630 loss: 18.4341 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.9721 global_avg_mtp_loss: 15.4620 +[titan] 2025-07-09 14:37:14,333 - root - INFO - lr: 2.9910e-04 gnorm: 0.99 [ 1:02:18<21:23:28] +[titan] 2025-07-09 14:37:18,273 - root - INFO - step: 4635 loss: 17.8772 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.8502 global_avg_mtp_loss: 15.0270 +[titan] 2025-07-09 14:37:18,273 - root - INFO - lr: 2.9910e-04 gnorm: 1.07 [ 1:02:22<21:23:22] +[titan] 2025-07-09 14:37:22,175 - root - INFO - step: 4640 loss: 17.7765 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.8462 global_avg_mtp_loss: 14.9304 +[titan] 2025-07-09 14:37:22,176 - root - INFO - lr: 2.9910e-04 gnorm: 0.93 [ 1:02:26<21:23:15] +[titan] 2025-07-09 14:37:26,090 - root - INFO - step: 4645 loss: 17.7600 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.8470 global_avg_mtp_loss: 14.9130 +[titan] 2025-07-09 14:37:26,090 - root - INFO - lr: 2.9910e-04 gnorm: 0.96 [ 1:02:30<21:23:09] +[titan] 2025-07-09 14:37:29,222 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:37:30,021 - root - INFO - step: 4650 loss: 17.9460 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.8798 global_avg_mtp_loss: 15.0663 +[titan] 2025-07-09 14:37:30,021 - root - INFO - lr: 2.9909e-04 gnorm: 0.94 [ 1:02:34<21:23:02] +[titan] 2025-07-09 14:37:33,939 - root - INFO - step: 4655 loss: 18.0172 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.8857 global_avg_mtp_loss: 15.1315 +[titan] 2025-07-09 14:37:33,939 - root - INFO - lr: 2.9909e-04 gnorm: 1.10 [ 1:02:38<21:22:56] +[titan] 2025-07-09 14:37:37,871 - root - INFO - step: 4660 loss: 18.0826 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.9105 global_avg_mtp_loss: 15.1721 +[titan] 2025-07-09 14:37:37,871 - root - INFO - lr: 2.9909e-04 gnorm: 1.00 [ 1:02:42<21:22:50] +[titan] 2025-07-09 14:37:41,800 - root - INFO - step: 4665 loss: 17.9254 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.8721 global_avg_mtp_loss: 15.0534 +[titan] 2025-07-09 14:37:41,800 - root - INFO - lr: 2.9909e-04 gnorm: 0.96 [ 1:02:46<21:22:43] +[titan] 2025-07-09 14:37:45,705 - root - INFO - step: 4670 loss: 18.0592 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.8985 global_avg_mtp_loss: 15.1607 +[titan] 2025-07-09 14:37:45,706 - root - INFO - lr: 2.9909e-04 gnorm: 0.96 [ 1:02:49<21:22:37] +[titan] 2025-07-09 14:37:49,682 - root - INFO - step: 4675 loss: 18.0407 memory: 44.58GiB(31.99%) tps: 82,417 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.8946 global_avg_mtp_loss: 15.1460 +[titan] 2025-07-09 14:37:49,682 - root - INFO - lr: 2.9908e-04 gnorm: 0.94 [ 1:02:53<21:22:31] +[titan] 2025-07-09 14:37:53,589 - root - INFO - step: 4680 loss: 18.0217 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.8747 global_avg_mtp_loss: 15.1470 +[titan] 2025-07-09 14:37:53,589 - root - INFO - lr: 2.9908e-04 gnorm: 0.97 [ 1:02:57<21:22:25] +[titan] 2025-07-09 14:37:57,512 - root - INFO - step: 4685 loss: 17.7201 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.8530 global_avg_mtp_loss: 14.8671 +[titan] 2025-07-09 14:37:57,512 - root - INFO - lr: 2.9908e-04 gnorm: 4.29 [ 1:03:01<21:22:18] +[titan] 2025-07-09 14:38:01,425 - root - INFO - step: 4690 loss: 18.1569 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.9168 global_avg_mtp_loss: 15.2400 +[titan] 2025-07-09 14:38:01,425 - root - INFO - lr: 2.9908e-04 gnorm: 1.05 [ 1:03:05<21:22:12] +[titan] 2025-07-09 14:38:05,346 - root - INFO - step: 4695 loss: 17.9944 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.8895 global_avg_mtp_loss: 15.1050 +[titan] 2025-07-09 14:38:05,346 - root - INFO - lr: 2.9907e-04 gnorm: 1.09 [ 1:03:09<21:22:06] +[titan] 2025-07-09 14:38:08,509 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:38:09,302 - root - INFO - step: 4700 loss: 17.9904 memory: 44.58GiB(31.99%) tps: 82,833 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.8932 global_avg_mtp_loss: 15.0972 +[titan] 2025-07-09 14:38:09,303 - root - INFO - lr: 2.9907e-04 gnorm: 0.96 [ 1:03:13<21:22:00] +[titan] 2025-07-09 14:38:13,223 - root - INFO - step: 4705 loss: 17.8814 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.8543 global_avg_mtp_loss: 15.0270 +[titan] 2025-07-09 14:38:13,223 - root - INFO - lr: 2.9907e-04 gnorm: 0.97 [ 1:03:17<21:21:54] +[titan] 2025-07-09 14:38:17,133 - root - INFO - step: 4710 loss: 18.0478 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.8947 global_avg_mtp_loss: 15.1531 +[titan] 2025-07-09 14:38:17,133 - root - INFO - lr: 2.9907e-04 gnorm: 1.01 [ 1:03:21<21:21:47] +[titan] 2025-07-09 14:38:21,055 - root - INFO - step: 4715 loss: 17.9992 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.8852 global_avg_mtp_loss: 15.1141 +[titan] 2025-07-09 14:38:21,055 - root - INFO - lr: 2.9906e-04 gnorm: 1.00 [ 1:03:25<21:21:41] +[titan] 2025-07-09 14:38:24,971 - root - INFO - step: 4720 loss: 17.8159 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.8441 global_avg_mtp_loss: 14.9717 +[titan] 2025-07-09 14:38:24,971 - root - INFO - lr: 2.9906e-04 gnorm: 1.06 [ 1:03:29<21:21:34] +[titan] 2025-07-09 14:38:28,930 - root - INFO - step: 4725 loss: 17.9370 memory: 44.58GiB(31.99%) tps: 82,786 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.8713 global_avg_mtp_loss: 15.0657 +[titan] 2025-07-09 14:38:28,930 - root - INFO - lr: 2.9906e-04 gnorm: 0.95 [ 1:03:33<21:21:29] +[titan] 2025-07-09 14:38:32,854 - root - INFO - step: 4730 loss: 17.9113 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.8711 global_avg_mtp_loss: 15.0401 +[titan] 2025-07-09 14:38:32,854 - root - INFO - lr: 2.9905e-04 gnorm: 1.05 [ 1:03:37<21:21:22] +[titan] 2025-07-09 14:38:36,778 - root - INFO - step: 4735 loss: 18.1720 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.9128 global_avg_mtp_loss: 15.2593 +[titan] 2025-07-09 14:38:36,778 - root - INFO - lr: 2.9905e-04 gnorm: 1.05 [ 1:03:41<21:21:16] +[titan] 2025-07-09 14:38:40,681 - root - INFO - step: 4740 loss: 18.2782 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.9400 global_avg_mtp_loss: 15.3381 +[titan] 2025-07-09 14:38:40,682 - root - INFO - lr: 2.9905e-04 gnorm: 0.96 [ 1:03:44<21:21:09] +[titan] 2025-07-09 14:38:44,589 - root - INFO - step: 4745 loss: 17.9087 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.8671 global_avg_mtp_loss: 15.0416 +[titan] 2025-07-09 14:38:44,589 - root - INFO - lr: 2.9905e-04 gnorm: 0.92 [ 1:03:48<21:21:03] +[titan] 2025-07-09 14:38:47,708 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:38:48,495 - root - INFO - step: 4750 loss: 17.9604 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8741 global_avg_mtp_loss: 15.0863 +[titan] 2025-07-09 14:38:48,495 - root - INFO - lr: 2.9904e-04 gnorm: 0.99 [ 1:03:52<21:20:56] +[titan] 2025-07-09 14:38:52,413 - root - INFO - step: 4755 loss: 17.8807 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.8776 global_avg_mtp_loss: 15.0031 +[titan] 2025-07-09 14:38:52,413 - root - INFO - lr: 2.9904e-04 gnorm: 1.06 [ 1:03:56<21:20:50] +[titan] 2025-07-09 14:38:56,348 - root - INFO - step: 4760 loss: 18.0375 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.8932 global_avg_mtp_loss: 15.1443 +[titan] 2025-07-09 14:38:56,348 - root - INFO - lr: 2.9904e-04 gnorm: 1.08 [ 1:04:00<21:20:44] +[titan] 2025-07-09 14:39:00,277 - root - INFO - step: 4765 loss: 18.1750 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.9129 global_avg_mtp_loss: 15.2621 +[titan] 2025-07-09 14:39:00,277 - root - INFO - lr: 2.9904e-04 gnorm: 0.98 [ 1:04:04<21:20:38] +[titan] 2025-07-09 14:39:04,195 - root - INFO - step: 4770 loss: 17.8106 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.8537 global_avg_mtp_loss: 14.9569 +[titan] 2025-07-09 14:39:04,195 - root - INFO - lr: 2.9903e-04 gnorm: 0.93 [ 1:04:08<21:20:31] +[titan] 2025-07-09 14:39:08,143 - root - INFO - step: 4775 loss: 18.0821 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.9185 global_avg_mtp_loss: 15.1635 +[titan] 2025-07-09 14:39:08,143 - root - INFO - lr: 2.9903e-04 gnorm: 0.91 [ 1:04:12<21:20:25] +[titan] 2025-07-09 14:39:12,110 - root - INFO - step: 4780 loss: 18.0771 memory: 44.58GiB(31.99%) tps: 82,601 tflops: 285.07 mfu: 28.82% global_avg_ntp_loss: 2.8994 global_avg_mtp_loss: 15.1777 +[titan] 2025-07-09 14:39:12,111 - root - INFO - lr: 2.9903e-04 gnorm: 1.02 [ 1:04:16<21:20:20] +[titan] 2025-07-09 14:39:16,017 - root - INFO - step: 4785 loss: 17.8965 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.8630 global_avg_mtp_loss: 15.0335 +[titan] 2025-07-09 14:39:16,017 - root - INFO - lr: 2.9903e-04 gnorm: 0.98 [ 1:04:20<21:20:13] +[titan] 2025-07-09 14:39:19,913 - root - INFO - step: 4790 loss: 18.0978 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.9120 global_avg_mtp_loss: 15.1858 +[titan] 2025-07-09 14:39:19,913 - root - INFO - lr: 2.9902e-04 gnorm: 0.90 [ 1:04:24<21:20:07] +[titan] 2025-07-09 14:39:23,811 - root - INFO - step: 4795 loss: 18.1601 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.9166 global_avg_mtp_loss: 15.2435 +[titan] 2025-07-09 14:39:23,811 - root - INFO - lr: 2.9902e-04 gnorm: 0.93 [ 1:04:28<21:20:00] +[titan] 2025-07-09 14:39:26,935 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:39:27,720 - root - INFO - step: 4800 loss: 18.0812 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.9008 global_avg_mtp_loss: 15.1803 +[titan] 2025-07-09 14:39:27,721 - root - INFO - lr: 2.9902e-04 gnorm: 0.98 [ 1:04:31<21:19:53] +[titan] 2025-07-09 14:39:31,868 - root - INFO - step: 4805 loss: 17.8983 memory: 44.58GiB(31.99%) tps: 79,013 tflops: 272.69 mfu: 27.57% global_avg_ntp_loss: 2.8712 global_avg_mtp_loss: 15.0271 +[titan] 2025-07-09 14:39:31,868 - root - INFO - lr: 2.9902e-04 gnorm: 1.10 [ 1:04:36<21:19:52] +[titan] 2025-07-09 14:39:35,804 - root - INFO - step: 4810 loss: 17.9357 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.8958 global_avg_mtp_loss: 15.0399 +[titan] 2025-07-09 14:39:35,804 - root - INFO - lr: 2.9901e-04 gnorm: 1.00 [ 1:04:40<21:19:46] +[titan] 2025-07-09 14:39:39,709 - root - INFO - step: 4815 loss: 18.2065 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.9265 global_avg_mtp_loss: 15.2800 +[titan] 2025-07-09 14:39:39,709 - root - INFO - lr: 2.9901e-04 gnorm: 0.91 [ 1:04:43<21:19:39] +[titan] 2025-07-09 14:39:43,663 - root - INFO - step: 4820 loss: 17.9584 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.8787 global_avg_mtp_loss: 15.0797 +[titan] 2025-07-09 14:39:43,663 - root - INFO - lr: 2.9901e-04 gnorm: 0.94 [ 1:04:47<21:19:34] +[titan] 2025-07-09 14:39:47,633 - root - INFO - step: 4825 loss: 18.0074 memory: 44.58GiB(31.99%) tps: 82,561 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.8712 global_avg_mtp_loss: 15.1362 +[titan] 2025-07-09 14:39:47,633 - root - INFO - lr: 2.9901e-04 gnorm: 0.94 [ 1:04:51<21:19:28] +[titan] 2025-07-09 14:39:51,558 - root - INFO - step: 4830 loss: 17.9649 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.8785 global_avg_mtp_loss: 15.0863 +[titan] 2025-07-09 14:39:51,558 - root - INFO - lr: 2.9900e-04 gnorm: 0.93 [ 1:04:55<21:19:22] +[titan] 2025-07-09 14:39:55,454 - root - INFO - step: 4835 loss: 17.8148 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.8438 global_avg_mtp_loss: 14.9710 +[titan] 2025-07-09 14:39:55,454 - root - INFO - lr: 2.9900e-04 gnorm: 1.00 [ 1:04:59<21:19:15] +[titan] 2025-07-09 14:39:59,358 - root - INFO - step: 4840 loss: 17.8292 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.8475 global_avg_mtp_loss: 14.9817 +[titan] 2025-07-09 14:39:59,358 - root - INFO - lr: 2.9900e-04 gnorm: 0.90 [ 1:05:03<21:19:09] +[titan] 2025-07-09 14:40:03,242 - root - INFO - step: 4845 loss: 17.8632 memory: 44.58GiB(31.99%) tps: 84,367 tflops: 291.16 mfu: 29.44% global_avg_ntp_loss: 2.8481 global_avg_mtp_loss: 15.0151 +[titan] 2025-07-09 14:40:03,242 - root - INFO - lr: 2.9900e-04 gnorm: 0.87 [ 1:05:07<21:19:02] +[titan] 2025-07-09 14:40:06,384 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:40:07,176 - root - INFO - step: 4850 loss: 17.9362 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.8730 global_avg_mtp_loss: 15.0633 +[titan] 2025-07-09 14:40:07,176 - root - INFO - lr: 2.9899e-04 gnorm: 0.96 [ 1:05:11<21:18:56] +[titan] 2025-07-09 14:40:11,088 - root - INFO - step: 4855 loss: 18.0972 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.9086 global_avg_mtp_loss: 15.1886 +[titan] 2025-07-09 14:40:11,088 - root - INFO - lr: 2.9899e-04 gnorm: 0.94 [ 1:05:15<21:18:49] +[titan] 2025-07-09 14:40:15,014 - root - INFO - step: 4860 loss: 18.1470 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.9376 global_avg_mtp_loss: 15.2094 +[titan] 2025-07-09 14:40:15,014 - root - INFO - lr: 2.9899e-04 gnorm: 1.03 [ 1:05:19<21:18:43] +[titan] 2025-07-09 14:40:18,947 - root - INFO - step: 4865 loss: 18.1555 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.9131 global_avg_mtp_loss: 15.2424 +[titan] 2025-07-09 14:40:18,947 - root - INFO - lr: 2.9899e-04 gnorm: 1.05 [ 1:05:23<21:18:37] +[titan] 2025-07-09 14:40:22,883 - root - INFO - step: 4870 loss: 18.2980 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.9458 global_avg_mtp_loss: 15.3522 +[titan] 2025-07-09 14:40:22,883 - root - INFO - lr: 2.9898e-04 gnorm: 0.98 [ 1:05:27<21:18:31] +[titan] 2025-07-09 14:40:26,815 - root - INFO - step: 4875 loss: 18.0232 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.8962 global_avg_mtp_loss: 15.1270 +[titan] 2025-07-09 14:40:26,815 - root - INFO - lr: 2.9898e-04 gnorm: 0.94 [ 1:05:31<21:18:25] +[titan] 2025-07-09 14:40:30,728 - root - INFO - step: 4880 loss: 18.1458 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.9180 global_avg_mtp_loss: 15.2278 +[titan] 2025-07-09 14:40:30,728 - root - INFO - lr: 2.9898e-04 gnorm: 1.08 [ 1:05:34<21:18:19] +[titan] 2025-07-09 14:40:34,638 - root - INFO - step: 4885 loss: 17.8830 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.8566 global_avg_mtp_loss: 15.0264 +[titan] 2025-07-09 14:40:34,639 - root - INFO - lr: 2.9897e-04 gnorm: 1.03 [ 1:05:38<21:18:13] +[titan] 2025-07-09 14:40:38,594 - root - INFO - step: 4890 loss: 17.9527 memory: 44.58GiB(31.99%) tps: 82,847 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.8708 global_avg_mtp_loss: 15.0820 +[titan] 2025-07-09 14:40:38,594 - root - INFO - lr: 2.9897e-04 gnorm: 0.95 [ 1:05:42<21:18:07] +[titan] 2025-07-09 14:40:42,495 - root - INFO - step: 4895 loss: 18.1810 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.9327 global_avg_mtp_loss: 15.2483 +[titan] 2025-07-09 14:40:42,495 - root - INFO - lr: 2.9897e-04 gnorm: 1.07 [ 1:05:46<21:18:01] +[titan] 2025-07-09 14:40:45,621 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:40:46,417 - root - INFO - step: 4900 loss: 17.9330 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.8786 global_avg_mtp_loss: 15.0544 +[titan] 2025-07-09 14:40:46,418 - root - INFO - lr: 2.9897e-04 gnorm: 0.91 [ 1:05:50<21:17:54] +[titan] 2025-07-09 14:40:50,364 - root - INFO - step: 4905 loss: 18.1041 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.9268 global_avg_mtp_loss: 15.1774 +[titan] 2025-07-09 14:40:50,364 - root - INFO - lr: 2.9896e-04 gnorm: 0.97 [ 1:05:54<21:17:49] +[titan] 2025-07-09 14:40:54,281 - root - INFO - step: 4910 loss: 18.0165 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.8903 global_avg_mtp_loss: 15.1262 +[titan] 2025-07-09 14:40:54,281 - root - INFO - lr: 2.9896e-04 gnorm: 1.02 [ 1:05:58<21:17:42] +[titan] 2025-07-09 14:40:58,193 - root - INFO - step: 4915 loss: 18.0311 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.9330 global_avg_mtp_loss: 15.0980 +[titan] 2025-07-09 14:40:58,194 - root - INFO - lr: 2.9896e-04 gnorm: 1.08 [ 1:06:02<21:17:36] +[titan] 2025-07-09 14:41:02,100 - root - INFO - step: 4920 loss: 18.0734 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.9064 global_avg_mtp_loss: 15.1670 +[titan] 2025-07-09 14:41:02,100 - root - INFO - lr: 2.9896e-04 gnorm: 0.95 [ 1:06:06<21:17:30] +[titan] 2025-07-09 14:41:06,002 - root - INFO - step: 4925 loss: 17.9826 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.8694 global_avg_mtp_loss: 15.1132 +[titan] 2025-07-09 14:41:06,003 - root - INFO - lr: 2.9895e-04 gnorm: 1.10 [ 1:06:10<21:17:23] +[titan] 2025-07-09 14:41:09,907 - root - INFO - step: 4930 loss: 17.7018 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.8349 global_avg_mtp_loss: 14.8669 +[titan] 2025-07-09 14:41:09,907 - root - INFO - lr: 2.9895e-04 gnorm: 1.10 [ 1:06:14<21:17:17] +[titan] 2025-07-09 14:41:13,830 - root - INFO - step: 4935 loss: 18.0366 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.8882 global_avg_mtp_loss: 15.1484 +[titan] 2025-07-09 14:41:13,831 - root - INFO - lr: 2.9895e-04 gnorm: 1.02 [ 1:06:18<21:17:11] +[titan] 2025-07-09 14:41:17,751 - root - INFO - step: 4940 loss: 17.9967 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.8942 global_avg_mtp_loss: 15.1025 +[titan] 2025-07-09 14:41:17,752 - root - INFO - lr: 2.9895e-04 gnorm: 0.99 [ 1:06:21<21:17:04] +[titan] 2025-07-09 14:41:21,686 - root - INFO - step: 4945 loss: 17.9811 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.8842 global_avg_mtp_loss: 15.0969 +[titan] 2025-07-09 14:41:21,687 - root - INFO - lr: 2.9894e-04 gnorm: 1.05 [ 1:06:25<21:16:59] +[titan] 2025-07-09 14:41:24,819 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:41:25,623 - root - INFO - step: 4950 loss: 17.7747 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.8550 global_avg_mtp_loss: 14.9197 +[titan] 2025-07-09 14:41:25,623 - root - INFO - lr: 2.9894e-04 gnorm: 1.04 [ 1:06:29<21:16:53] +[titan] 2025-07-09 14:41:29,557 - root - INFO - step: 4955 loss: 17.9185 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.8633 global_avg_mtp_loss: 15.0552 +[titan] 2025-07-09 14:41:29,557 - root - INFO - lr: 2.9894e-04 gnorm: 1.00 [ 1:06:33<21:16:47] +[titan] 2025-07-09 14:41:33,469 - root - INFO - step: 4960 loss: 18.0011 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.8771 global_avg_mtp_loss: 15.1240 +[titan] 2025-07-09 14:41:33,470 - root - INFO - lr: 2.9893e-04 gnorm: 0.90 [ 1:06:37<21:16:41] +[titan] 2025-07-09 14:41:37,373 - root - INFO - step: 4965 loss: 17.9235 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.8687 global_avg_mtp_loss: 15.0548 +[titan] 2025-07-09 14:41:37,374 - root - INFO - lr: 2.9893e-04 gnorm: 1.03 [ 1:06:41<21:16:34] +[titan] 2025-07-09 14:41:41,294 - root - INFO - step: 4970 loss: 17.9923 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.8735 global_avg_mtp_loss: 15.1188 +[titan] 2025-07-09 14:41:41,294 - root - INFO - lr: 2.9893e-04 gnorm: 1.03 [ 1:06:45<21:16:28] +[titan] 2025-07-09 14:41:45,284 - root - INFO - step: 4975 loss: 17.9414 memory: 44.58GiB(31.99%) tps: 82,139 tflops: 283.47 mfu: 28.66% global_avg_ntp_loss: 2.8709 global_avg_mtp_loss: 15.0705 +[titan] 2025-07-09 14:41:45,284 - root - INFO - lr: 2.9893e-04 gnorm: 0.90 [ 1:06:49<21:16:23] +[titan] 2025-07-09 14:41:49,188 - root - INFO - step: 4980 loss: 18.1026 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.9035 global_avg_mtp_loss: 15.1990 +[titan] 2025-07-09 14:41:49,188 - root - INFO - lr: 2.9892e-04 gnorm: 0.91 [ 1:06:53<21:16:17] +[titan] 2025-07-09 14:41:53,083 - root - INFO - step: 4985 loss: 18.1107 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.8919 global_avg_mtp_loss: 15.2188 +[titan] 2025-07-09 14:41:53,083 - root - INFO - lr: 2.9892e-04 gnorm: 0.94 [ 1:06:57<21:16:10] +[titan] 2025-07-09 14:41:57,040 - root - INFO - step: 4990 loss: 18.1694 memory: 44.58GiB(31.99%) tps: 82,820 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.9202 global_avg_mtp_loss: 15.2493 +[titan] 2025-07-09 14:41:57,040 - root - INFO - lr: 2.9892e-04 gnorm: 1.06 [ 1:07:01<21:16:05] +[titan] 2025-07-09 14:42:00,945 - root - INFO - step: 4995 loss: 17.7961 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.8675 global_avg_mtp_loss: 14.9286 +[titan] 2025-07-09 14:42:00,945 - root - INFO - lr: 2.9892e-04 gnorm: 1.42 [ 1:07:05<21:15:58] +[titan] 2025-07-09 14:42:04,113 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:42:04,905 - root - INFO - step: 5000 loss: 17.8560 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.8495 global_avg_mtp_loss: 15.0065 +[titan] 2025-07-09 14:42:04,905 - root - INFO - lr: 2.9891e-04 gnorm: 1.05 [ 1:07:09<21:15:53] +[titan] 2025-07-09 14:42:08,807 - root - INFO - step: 5005 loss: 17.9041 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.8631 global_avg_mtp_loss: 15.0410 +[titan] 2025-07-09 14:42:08,807 - root - INFO - lr: 2.9891e-04 gnorm: 0.97 [ 1:07:13<21:15:47] +[titan] 2025-07-09 14:42:12,717 - root - INFO - step: 5010 loss: 17.5684 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.8016 global_avg_mtp_loss: 14.7668 +[titan] 2025-07-09 14:42:12,717 - root - INFO - lr: 2.9891e-04 gnorm: 1.01 [ 1:07:16<21:15:40] +[titan] 2025-07-09 14:42:16,621 - root - INFO - step: 5015 loss: 17.7661 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.8270 global_avg_mtp_loss: 14.9391 +[titan] 2025-07-09 14:42:16,621 - root - INFO - lr: 2.9891e-04 gnorm: 1.01 [ 1:07:20<21:15:34] +[titan] 2025-07-09 14:42:20,546 - root - INFO - step: 5020 loss: 18.0743 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.8962 global_avg_mtp_loss: 15.1781 +[titan] 2025-07-09 14:42:20,546 - root - INFO - lr: 2.9890e-04 gnorm: 1.03 [ 1:07:24<21:15:28] +[titan] 2025-07-09 14:42:24,508 - root - INFO - step: 5025 loss: 17.7646 memory: 44.58GiB(31.99%) tps: 82,723 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.8437 global_avg_mtp_loss: 14.9209 +[titan] 2025-07-09 14:42:24,508 - root - INFO - lr: 2.9890e-04 gnorm: 0.94 [ 1:07:28<21:15:23] +[titan] 2025-07-09 14:42:28,423 - root - INFO - step: 5030 loss: 18.2771 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.9473 global_avg_mtp_loss: 15.3298 +[titan] 2025-07-09 14:42:28,424 - root - INFO - lr: 2.9890e-04 gnorm: 1.04 [ 1:07:32<21:15:16] +[titan] 2025-07-09 14:42:32,331 - root - INFO - step: 5035 loss: 17.9813 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.8893 global_avg_mtp_loss: 15.0919 +[titan] 2025-07-09 14:42:32,331 - root - INFO - lr: 2.9889e-04 gnorm: 0.97 [ 1:07:36<21:15:10] +[titan] 2025-07-09 14:42:36,219 - root - INFO - step: 5040 loss: 17.8208 memory: 44.58GiB(31.99%) tps: 84,278 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.8442 global_avg_mtp_loss: 14.9766 +[titan] 2025-07-09 14:42:36,219 - root - INFO - lr: 2.9889e-04 gnorm: 1.04 [ 1:07:40<21:15:03] +[titan] 2025-07-09 14:42:40,142 - root - INFO - step: 5045 loss: 17.9838 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.8803 global_avg_mtp_loss: 15.1035 +[titan] 2025-07-09 14:42:40,142 - root - INFO - lr: 2.9889e-04 gnorm: 0.99 [ 1:07:44<21:14:57] +[titan] 2025-07-09 14:42:43,252 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:42:44,041 - root - INFO - step: 5050 loss: 17.7270 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.8300 global_avg_mtp_loss: 14.8971 +[titan] 2025-07-09 14:42:44,041 - root - INFO - lr: 2.9889e-04 gnorm: 1.07 [ 1:07:48<21:14:51] +[titan] 2025-07-09 14:42:47,980 - root - INFO - step: 5055 loss: 17.9422 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.8739 global_avg_mtp_loss: 15.0684 +[titan] 2025-07-09 14:42:47,980 - root - INFO - lr: 2.9888e-04 gnorm: 0.98 [ 1:07:52<21:14:45] +[titan] 2025-07-09 14:42:51,865 - root - INFO - step: 5060 loss: 17.9216 memory: 44.58GiB(31.99%) tps: 84,358 tflops: 291.13 mfu: 29.44% global_avg_ntp_loss: 2.8799 global_avg_mtp_loss: 15.0417 +[titan] 2025-07-09 14:42:51,865 - root - INFO - lr: 2.9888e-04 gnorm: 1.08 [ 1:07:56<21:14:38] +[titan] 2025-07-09 14:42:55,777 - root - INFO - step: 5065 loss: 17.9859 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.8681 global_avg_mtp_loss: 15.1179 +[titan] 2025-07-09 14:42:55,777 - root - INFO - lr: 2.9888e-04 gnorm: 0.94 [ 1:07:59<21:14:32] +[titan] 2025-07-09 14:42:59,695 - root - INFO - step: 5070 loss: 17.8365 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.8472 global_avg_mtp_loss: 14.9893 +[titan] 2025-07-09 14:42:59,695 - root - INFO - lr: 2.9888e-04 gnorm: 0.97 [ 1:08:03<21:14:26] +[titan] 2025-07-09 14:43:03,691 - root - INFO - step: 5075 loss: 18.1479 memory: 44.58GiB(31.99%) tps: 82,008 tflops: 283.02 mfu: 28.62% global_avg_ntp_loss: 2.9201 global_avg_mtp_loss: 15.2277 +[titan] 2025-07-09 14:43:03,692 - root - INFO - lr: 2.9887e-04 gnorm: 0.97 [ 1:08:07<21:14:22] +[titan] 2025-07-09 14:43:07,665 - root - INFO - step: 5080 loss: 17.9335 memory: 44.58GiB(31.99%) tps: 82,462 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.8706 global_avg_mtp_loss: 15.0629 +[titan] 2025-07-09 14:43:07,666 - root - INFO - lr: 2.9887e-04 gnorm: 1.04 [ 1:08:11<21:14:16] +[titan] 2025-07-09 14:43:11,579 - root - INFO - step: 5085 loss: 17.9902 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.8870 global_avg_mtp_loss: 15.1032 +[titan] 2025-07-09 14:43:11,579 - root - INFO - lr: 2.9887e-04 gnorm: 1.07 [ 1:08:15<21:14:10] +[titan] 2025-07-09 14:43:15,526 - root - INFO - step: 5090 loss: 18.0364 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.8928 global_avg_mtp_loss: 15.1436 +[titan] 2025-07-09 14:43:15,527 - root - INFO - lr: 2.9886e-04 gnorm: 1.01 [ 1:08:19<21:14:05] +[titan] 2025-07-09 14:43:19,473 - root - INFO - step: 5095 loss: 17.5948 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.7989 global_avg_mtp_loss: 14.7959 +[titan] 2025-07-09 14:43:19,473 - root - INFO - lr: 2.9886e-04 gnorm: 0.99 [ 1:08:23<21:13:59] +[titan] 2025-07-09 14:43:22,595 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:43:23,382 - root - INFO - step: 5100 loss: 17.7444 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.8280 global_avg_mtp_loss: 14.9163 +[titan] 2025-07-09 14:43:23,383 - root - INFO - lr: 2.9886e-04 gnorm: 0.94 [ 1:08:27<21:13:53] +[titan] 2025-07-09 14:43:27,343 - root - INFO - step: 5105 loss: 17.9624 memory: 44.58GiB(31.99%) tps: 82,742 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.8662 global_avg_mtp_loss: 15.0961 +[titan] 2025-07-09 14:43:27,343 - root - INFO - lr: 2.9886e-04 gnorm: 0.85 [ 1:08:31<21:13:48] +[titan] 2025-07-09 14:43:31,257 - root - INFO - step: 5110 loss: 17.9055 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.8595 global_avg_mtp_loss: 15.0460 +[titan] 2025-07-09 14:43:31,257 - root - INFO - lr: 2.9885e-04 gnorm: 0.97 [ 1:08:35<21:13:42] +[titan] 2025-07-09 14:43:35,168 - root - INFO - step: 5115 loss: 17.9761 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.8762 global_avg_mtp_loss: 15.0999 +[titan] 2025-07-09 14:43:35,168 - root - INFO - lr: 2.9885e-04 gnorm: 0.94 [ 1:08:39<21:13:35] +[titan] 2025-07-09 14:43:39,140 - root - INFO - step: 5120 loss: 17.8815 memory: 44.58GiB(31.99%) tps: 82,500 tflops: 284.72 mfu: 28.79% global_avg_ntp_loss: 2.8621 global_avg_mtp_loss: 15.0194 +[titan] 2025-07-09 14:43:39,140 - root - INFO - lr: 2.9885e-04 gnorm: 1.08 [ 1:08:43<21:13:30] +[titan] 2025-07-09 14:43:39,282 - root - INFO - Dumping profiler traces at step 5120 +[titan] 2025-07-09 14:43:39,314 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:43:43,216 - root - INFO - step: 5125 loss: 17.8113 memory: 44.58GiB(31.99%) tps: 80,389 tflops: 277.44 mfu: 28.05% global_avg_ntp_loss: 2.8409 global_avg_mtp_loss: 14.9704 +[titan] 2025-07-09 14:43:43,217 - root - INFO - lr: 2.9884e-04 gnorm: 1.09 [ 1:08:47<21:13:27] +[titan] 2025-07-09 14:43:47,112 - root - INFO - step: 5130 loss: 17.9425 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.8700 global_avg_mtp_loss: 15.0725 +[titan] 2025-07-09 14:43:47,112 - root - INFO - lr: 2.9884e-04 gnorm: 1.04 [ 1:08:51<21:13:21] +[titan] 2025-07-09 14:43:51,057 - root - INFO - step: 5135 loss: 17.9901 memory: 44.58GiB(31.99%) tps: 83,078 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.8782 global_avg_mtp_loss: 15.1119 +[titan] 2025-07-09 14:43:51,057 - root - INFO - lr: 2.9884e-04 gnorm: 0.97 [ 1:08:55<21:13:15] +[titan] 2025-07-09 14:43:54,995 - root - INFO - step: 5140 loss: 18.3193 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.9559 global_avg_mtp_loss: 15.3634 +[titan] 2025-07-09 14:43:54,996 - root - INFO - lr: 2.9884e-04 gnorm: 1.09 [ 1:08:59<21:13:10] +[titan] 2025-07-09 14:43:58,916 - root - INFO - step: 5145 loss: 18.2145 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.9255 global_avg_mtp_loss: 15.2890 +[titan] 2025-07-09 14:43:58,917 - root - INFO - lr: 2.9883e-04 gnorm: 1.02 [ 1:09:03<21:13:04] +[titan] 2025-07-09 14:44:02,024 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:44:02,811 - root - INFO - step: 5150 loss: 17.8169 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.8573 global_avg_mtp_loss: 14.9596 +[titan] 2025-07-09 14:44:02,811 - root - INFO - lr: 2.9883e-04 gnorm: 0.92 [ 1:09:07<21:12:57] +[titan] 2025-07-09 14:44:06,777 - root - INFO - step: 5155 loss: 17.8196 memory: 44.58GiB(31.99%) tps: 82,618 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.8552 global_avg_mtp_loss: 14.9644 +[titan] 2025-07-09 14:44:06,778 - root - INFO - lr: 2.9883e-04 gnorm: 0.93 [ 1:09:10<21:12:52] +[titan] 2025-07-09 14:44:10,671 - root - INFO - step: 5160 loss: 18.1841 memory: 44.58GiB(31.99%) tps: 84,175 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.9324 global_avg_mtp_loss: 15.2518 +[titan] 2025-07-09 14:44:10,671 - root - INFO - lr: 2.9882e-04 gnorm: 1.05 [ 1:09:14<21:12:46] +[titan] 2025-07-09 14:44:14,575 - root - INFO - step: 5165 loss: 18.1323 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.9140 global_avg_mtp_loss: 15.2183 +[titan] 2025-07-09 14:44:14,575 - root - INFO - lr: 2.9882e-04 gnorm: 0.97 [ 1:09:18<21:12:39] +[titan] 2025-07-09 14:44:18,471 - root - INFO - step: 5170 loss: 18.0942 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.9080 global_avg_mtp_loss: 15.1862 +[titan] 2025-07-09 14:44:18,471 - root - INFO - lr: 2.9882e-04 gnorm: 1.05 [ 1:09:22<21:12:33] +[titan] 2025-07-09 14:44:22,382 - root - INFO - step: 5175 loss: 17.9183 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.8736 global_avg_mtp_loss: 15.0447 +[titan] 2025-07-09 14:44:22,382 - root - INFO - lr: 2.9882e-04 gnorm: 0.94 [ 1:09:26<21:12:27] +[titan] 2025-07-09 14:44:26,290 - root - INFO - step: 5180 loss: 17.8887 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.8474 global_avg_mtp_loss: 15.0413 +[titan] 2025-07-09 14:44:26,290 - root - INFO - lr: 2.9881e-04 gnorm: 1.01 [ 1:09:30<21:12:21] +[titan] 2025-07-09 14:44:30,186 - root - INFO - step: 5185 loss: 17.7635 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.8370 global_avg_mtp_loss: 14.9265 +[titan] 2025-07-09 14:44:30,186 - root - INFO - lr: 2.9881e-04 gnorm: 0.98 [ 1:09:34<21:12:14] +[titan] 2025-07-09 14:44:34,086 - root - INFO - step: 5190 loss: 17.9224 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.8752 global_avg_mtp_loss: 15.0471 +[titan] 2025-07-09 14:44:34,086 - root - INFO - lr: 2.9881e-04 gnorm: 1.10 [ 1:09:38<21:12:08] +[titan] 2025-07-09 14:44:37,998 - root - INFO - step: 5195 loss: 17.6200 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.8014 global_avg_mtp_loss: 14.8186 +[titan] 2025-07-09 14:44:37,999 - root - INFO - lr: 2.9881e-04 gnorm: 1.04 [ 1:09:42<21:12:02] +[titan] 2025-07-09 14:44:41,192 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:44:41,984 - root - INFO - step: 5200 loss: 17.7779 memory: 44.58GiB(31.99%) tps: 82,218 tflops: 283.75 mfu: 28.69% global_avg_ntp_loss: 2.8575 global_avg_mtp_loss: 14.9204 +[titan] 2025-07-09 14:44:41,984 - root - INFO - lr: 2.9880e-04 gnorm: 0.98 [ 1:09:46<21:11:57] +[titan] 2025-07-09 14:44:45,874 - root - INFO - step: 5205 loss: 17.8180 memory: 44.58GiB(31.99%) tps: 84,253 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.8445 global_avg_mtp_loss: 14.9735 +[titan] 2025-07-09 14:44:45,874 - root - INFO - lr: 2.9880e-04 gnorm: 0.97 [ 1:09:50<21:11:50] +[titan] 2025-07-09 14:44:49,759 - root - INFO - step: 5210 loss: 17.8921 memory: 44.58GiB(31.99%) tps: 84,348 tflops: 291.10 mfu: 29.43% global_avg_ntp_loss: 2.8663 global_avg_mtp_loss: 15.0258 +[titan] 2025-07-09 14:44:49,759 - root - INFO - lr: 2.9880e-04 gnorm: 0.96 [ 1:09:53<21:11:44] +[titan] 2025-07-09 14:44:53,708 - root - INFO - step: 5215 loss: 17.8733 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.8600 global_avg_mtp_loss: 15.0133 +[titan] 2025-07-09 14:44:53,708 - root - INFO - lr: 2.9879e-04 gnorm: 1.01 [ 1:09:57<21:11:38] +[titan] 2025-07-09 14:44:57,625 - root - INFO - step: 5220 loss: 18.0494 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.8912 global_avg_mtp_loss: 15.1582 +[titan] 2025-07-09 14:44:57,626 - root - INFO - lr: 2.9879e-04 gnorm: 1.29 [ 1:10:01<21:11:32] +[titan] 2025-07-09 14:45:01,552 - root - INFO - step: 5225 loss: 17.7178 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.8493 global_avg_mtp_loss: 14.8686 +[titan] 2025-07-09 14:45:01,553 - root - INFO - lr: 2.9879e-04 gnorm: 0.97 [ 1:10:05<21:11:27] +[titan] 2025-07-09 14:45:05,450 - root - INFO - step: 5230 loss: 18.1271 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.9115 global_avg_mtp_loss: 15.2156 +[titan] 2025-07-09 14:45:05,451 - root - INFO - lr: 2.9879e-04 gnorm: 0.97 [ 1:10:09<21:11:20] +[titan] 2025-07-09 14:45:09,392 - root - INFO - step: 5235 loss: 17.7734 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.8378 global_avg_mtp_loss: 14.9356 +[titan] 2025-07-09 14:45:09,392 - root - INFO - lr: 2.9878e-04 gnorm: 0.99 [ 1:10:13<21:11:15] +[titan] 2025-07-09 14:45:13,288 - root - INFO - step: 5240 loss: 18.1976 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.9153 global_avg_mtp_loss: 15.2823 +[titan] 2025-07-09 14:45:13,288 - root - INFO - lr: 2.9878e-04 gnorm: 0.94 [ 1:10:17<21:11:08] +[titan] 2025-07-09 14:45:17,189 - root - INFO - step: 5245 loss: 18.0230 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.8790 global_avg_mtp_loss: 15.1440 +[titan] 2025-07-09 14:45:17,189 - root - INFO - lr: 2.9878e-04 gnorm: 1.04 [ 1:10:21<21:11:02] +[titan] 2025-07-09 14:45:20,296 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:45:21,086 - root - INFO - step: 5250 loss: 17.7930 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.8451 global_avg_mtp_loss: 14.9479 +[titan] 2025-07-09 14:45:21,086 - root - INFO - lr: 2.9877e-04 gnorm: 1.04 [ 1:10:25<21:10:56] +[titan] 2025-07-09 14:45:25,045 - root - INFO - step: 5255 loss: 17.6236 memory: 44.58GiB(31.99%) tps: 82,771 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.7988 global_avg_mtp_loss: 14.8248 +[titan] 2025-07-09 14:45:25,045 - root - INFO - lr: 2.9877e-04 gnorm: 1.02 [ 1:10:29<21:10:51] +[titan] 2025-07-09 14:45:28,989 - root - INFO - step: 5260 loss: 17.7564 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.8340 global_avg_mtp_loss: 14.9224 +[titan] 2025-07-09 14:45:28,989 - root - INFO - lr: 2.9877e-04 gnorm: 0.99 [ 1:10:33<21:10:45] +[titan] 2025-07-09 14:45:32,891 - root - INFO - step: 5265 loss: 18.1192 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.9025 global_avg_mtp_loss: 15.2167 +[titan] 2025-07-09 14:45:32,891 - root - INFO - lr: 2.9876e-04 gnorm: 1.10 [ 1:10:37<21:10:39] +[titan] 2025-07-09 14:45:36,794 - root - INFO - step: 5270 loss: 17.9964 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.8850 global_avg_mtp_loss: 15.1114 +[titan] 2025-07-09 14:45:36,794 - root - INFO - lr: 2.9876e-04 gnorm: 1.02 [ 1:10:40<21:10:33] +[titan] 2025-07-09 14:45:40,699 - root - INFO - step: 5275 loss: 17.8818 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.8512 global_avg_mtp_loss: 15.0306 +[titan] 2025-07-09 14:45:40,699 - root - INFO - lr: 2.9876e-04 gnorm: 1.08 [ 1:10:44<21:10:27] +[titan] 2025-07-09 14:45:44,600 - root - INFO - step: 5280 loss: 17.6613 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.8065 global_avg_mtp_loss: 14.8549 +[titan] 2025-07-09 14:45:44,600 - root - INFO - lr: 2.9876e-04 gnorm: 0.96 [ 1:10:48<21:10:20] +[titan] 2025-07-09 14:45:48,516 - root - INFO - step: 5285 loss: 17.7881 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.8460 global_avg_mtp_loss: 14.9420 +[titan] 2025-07-09 14:45:48,516 - root - INFO - lr: 2.9875e-04 gnorm: 0.88 [ 1:10:52<21:10:14] +[titan] 2025-07-09 14:45:52,447 - root - INFO - step: 5290 loss: 18.0134 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.8824 global_avg_mtp_loss: 15.1311 +[titan] 2025-07-09 14:45:52,447 - root - INFO - lr: 2.9875e-04 gnorm: 0.98 [ 1:10:56<21:10:09] +[titan] 2025-07-09 14:45:56,395 - root - INFO - step: 5295 loss: 17.9603 memory: 44.58GiB(31.99%) tps: 82,996 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.8754 global_avg_mtp_loss: 15.0849 +[titan] 2025-07-09 14:45:56,396 - root - INFO - lr: 2.9875e-04 gnorm: 1.16 [ 1:11:00<21:10:03] +[titan] 2025-07-09 14:45:59,531 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:46:00,324 - root - INFO - step: 5300 loss: 17.7617 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.8323 global_avg_mtp_loss: 14.9294 +[titan] 2025-07-09 14:46:00,324 - root - INFO - lr: 2.9874e-04 gnorm: 0.91 [ 1:11:04<21:09:58] +[titan] 2025-07-09 14:46:04,228 - root - INFO - step: 5305 loss: 17.9001 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.8545 global_avg_mtp_loss: 15.0456 +[titan] 2025-07-09 14:46:04,228 - root - INFO - lr: 2.9874e-04 gnorm: 0.97 [ 1:11:08<21:09:51] +[titan] 2025-07-09 14:46:08,139 - root - INFO - step: 5310 loss: 17.7766 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.8328 global_avg_mtp_loss: 14.9438 +[titan] 2025-07-09 14:46:08,139 - root - INFO - lr: 2.9874e-04 gnorm: 0.94 [ 1:11:12<21:09:45] +[titan] 2025-07-09 14:46:12,056 - root - INFO - step: 5315 loss: 17.7316 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.8091 global_avg_mtp_loss: 14.9225 +[titan] 2025-07-09 14:46:12,057 - root - INFO - lr: 2.9874e-04 gnorm: 0.97 [ 1:11:16<21:09:40] +[titan] 2025-07-09 14:46:15,973 - root - INFO - step: 5320 loss: 17.6316 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.8173 global_avg_mtp_loss: 14.8143 +[titan] 2025-07-09 14:46:15,973 - root - INFO - lr: 2.9873e-04 gnorm: 1.00 [ 1:11:20<21:09:34] +[titan] 2025-07-09 14:46:19,874 - root - INFO - step: 5325 loss: 17.7637 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.8313 global_avg_mtp_loss: 14.9324 +[titan] 2025-07-09 14:46:19,874 - root - INFO - lr: 2.9873e-04 gnorm: 0.96 [ 1:11:24<21:09:27] +[titan] 2025-07-09 14:46:23,754 - root - INFO - step: 5330 loss: 17.8061 memory: 44.58GiB(31.99%) tps: 84,467 tflops: 291.51 mfu: 29.48% global_avg_ntp_loss: 2.8367 global_avg_mtp_loss: 14.9694 +[titan] 2025-07-09 14:46:23,754 - root - INFO - lr: 2.9873e-04 gnorm: 0.90 [ 1:11:27<21:09:21] +[titan] 2025-07-09 14:46:27,683 - root - INFO - step: 5335 loss: 17.7882 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.8418 global_avg_mtp_loss: 14.9464 +[titan] 2025-07-09 14:46:27,684 - root - INFO - lr: 2.9872e-04 gnorm: 0.98 [ 1:11:31<21:09:15] +[titan] 2025-07-09 14:46:31,601 - root - INFO - step: 5340 loss: 17.7169 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.8279 global_avg_mtp_loss: 14.8891 +[titan] 2025-07-09 14:46:31,602 - root - INFO - lr: 2.9872e-04 gnorm: 0.99 [ 1:11:35<21:09:09] +[titan] 2025-07-09 14:46:35,492 - root - INFO - step: 5345 loss: 17.9848 memory: 44.58GiB(31.99%) tps: 84,241 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.8768 global_avg_mtp_loss: 15.1080 +[titan] 2025-07-09 14:46:35,492 - root - INFO - lr: 2.9872e-04 gnorm: 1.03 [ 1:11:39<21:09:03] +[titan] 2025-07-09 14:46:38,610 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:46:39,398 - root - INFO - step: 5350 loss: 17.9169 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.8646 global_avg_mtp_loss: 15.0524 +[titan] 2025-07-09 14:46:39,398 - root - INFO - lr: 2.9872e-04 gnorm: 0.98 [ 1:11:43<21:08:57] +[titan] 2025-07-09 14:46:43,301 - root - INFO - step: 5355 loss: 18.1398 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.9053 global_avg_mtp_loss: 15.2345 +[titan] 2025-07-09 14:46:43,302 - root - INFO - lr: 2.9871e-04 gnorm: 1.03 [ 1:11:47<21:08:51] +[titan] 2025-07-09 14:46:47,209 - root - INFO - step: 5360 loss: 17.8708 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.8509 global_avg_mtp_loss: 15.0198 +[titan] 2025-07-09 14:46:47,209 - root - INFO - lr: 2.9871e-04 gnorm: 0.88 [ 1:11:51<21:08:45] +[titan] 2025-07-09 14:46:51,108 - root - INFO - step: 5365 loss: 17.7891 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.8388 global_avg_mtp_loss: 14.9503 +[titan] 2025-07-09 14:46:51,108 - root - INFO - lr: 2.9871e-04 gnorm: 1.03 [ 1:11:55<21:08:38] +[titan] 2025-07-09 14:46:55,007 - root - INFO - step: 5370 loss: 18.0101 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.8674 global_avg_mtp_loss: 15.1427 +[titan] 2025-07-09 14:46:55,007 - root - INFO - lr: 2.9870e-04 gnorm: 0.97 [ 1:11:59<21:08:32] +[titan] 2025-07-09 14:46:58,936 - root - INFO - step: 5375 loss: 17.8641 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.8543 global_avg_mtp_loss: 15.0098 +[titan] 2025-07-09 14:46:58,936 - root - INFO - lr: 2.9870e-04 gnorm: 1.01 [ 1:12:03<21:08:27] +[titan] 2025-07-09 14:47:02,849 - root - INFO - step: 5380 loss: 17.7204 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.8268 global_avg_mtp_loss: 14.8936 +[titan] 2025-07-09 14:47:02,849 - root - INFO - lr: 2.9870e-04 gnorm: 0.94 [ 1:12:07<21:08:21] +[titan] 2025-07-09 14:47:06,782 - root - INFO - step: 5385 loss: 17.8929 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.8611 global_avg_mtp_loss: 15.0318 +[titan] 2025-07-09 14:47:06,782 - root - INFO - lr: 2.9869e-04 gnorm: 0.95 [ 1:12:10<21:08:15] +[titan] 2025-07-09 14:47:10,696 - root - INFO - step: 5390 loss: 17.7480 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.8264 global_avg_mtp_loss: 14.9216 +[titan] 2025-07-09 14:47:10,697 - root - INFO - lr: 2.9869e-04 gnorm: 0.91 [ 1:12:14<21:08:09] +[titan] 2025-07-09 14:47:14,596 - root - INFO - step: 5395 loss: 17.8842 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.8701 global_avg_mtp_loss: 15.0141 +[titan] 2025-07-09 14:47:14,596 - root - INFO - lr: 2.9869e-04 gnorm: 0.93 [ 1:12:18<21:08:03] +[titan] 2025-07-09 14:47:17,707 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:47:18,499 - root - INFO - step: 5400 loss: 17.9496 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.8634 global_avg_mtp_loss: 15.0862 +[titan] 2025-07-09 14:47:18,499 - root - INFO - lr: 2.9869e-04 gnorm: 0.97 [ 1:12:22<21:07:57] +[titan] 2025-07-09 14:47:22,435 - root - INFO - step: 5405 loss: 18.1005 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.9049 global_avg_mtp_loss: 15.1956 +[titan] 2025-07-09 14:47:22,435 - root - INFO - lr: 2.9868e-04 gnorm: 0.99 [ 1:12:26<21:07:51] +[titan] 2025-07-09 14:47:26,326 - root - INFO - step: 5410 loss: 18.0255 memory: 44.58GiB(31.99%) tps: 84,218 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.8873 global_avg_mtp_loss: 15.1382 +[titan] 2025-07-09 14:47:26,326 - root - INFO - lr: 2.9868e-04 gnorm: 1.04 [ 1:12:30<21:07:45] +[titan] 2025-07-09 14:47:30,222 - root - INFO - step: 5415 loss: 17.6779 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.8245 global_avg_mtp_loss: 14.8535 +[titan] 2025-07-09 14:47:30,222 - root - INFO - lr: 2.9868e-04 gnorm: 0.87 [ 1:12:34<21:07:39] +[titan] 2025-07-09 14:47:34,124 - root - INFO - step: 5420 loss: 17.8728 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.8584 global_avg_mtp_loss: 15.0145 +[titan] 2025-07-09 14:47:34,124 - root - INFO - lr: 2.9867e-04 gnorm: 0.90 [ 1:12:38<21:07:33] +[titan] 2025-07-09 14:47:38,026 - root - INFO - step: 5425 loss: 17.9347 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.8694 global_avg_mtp_loss: 15.0654 +[titan] 2025-07-09 14:47:38,026 - root - INFO - lr: 2.9867e-04 gnorm: 1.04 [ 1:12:42<21:07:27] +[titan] 2025-07-09 14:47:41,940 - root - INFO - step: 5430 loss: 17.8495 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.8631 global_avg_mtp_loss: 14.9864 +[titan] 2025-07-09 14:47:41,941 - root - INFO - lr: 2.9867e-04 gnorm: 1.02 [ 1:12:46<21:07:21] +[titan] 2025-07-09 14:47:45,852 - root - INFO - step: 5435 loss: 17.7415 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.8363 global_avg_mtp_loss: 14.9051 +[titan] 2025-07-09 14:47:45,852 - root - INFO - lr: 2.9866e-04 gnorm: 1.00 [ 1:12:50<21:07:15] +[titan] 2025-07-09 14:47:49,742 - root - INFO - step: 5440 loss: 17.8065 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.8763 global_avg_mtp_loss: 14.9301 +[titan] 2025-07-09 14:47:49,742 - root - INFO - lr: 2.9866e-04 gnorm: 0.96 [ 1:12:53<21:07:09] +[titan] 2025-07-09 14:47:53,633 - root - INFO - step: 5445 loss: 17.9935 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.8863 global_avg_mtp_loss: 15.1072 +[titan] 2025-07-09 14:47:53,633 - root - INFO - lr: 2.9866e-04 gnorm: 1.01 [ 1:12:57<21:07:02] +[titan] 2025-07-09 14:47:56,745 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:47:57,535 - root - INFO - step: 5450 loss: 17.7199 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.8294 global_avg_mtp_loss: 14.8906 +[titan] 2025-07-09 14:47:57,535 - root - INFO - lr: 2.9866e-04 gnorm: 1.08 [ 1:13:01<21:06:56] +[titan] 2025-07-09 14:48:01,423 - root - INFO - step: 5455 loss: 17.5853 memory: 44.58GiB(31.99%) tps: 84,292 tflops: 290.91 mfu: 29.41% global_avg_ntp_loss: 2.7944 global_avg_mtp_loss: 14.7908 +[titan] 2025-07-09 14:48:01,423 - root - INFO - lr: 2.9865e-04 gnorm: 0.88 [ 1:13:05<21:06:50] +[titan] 2025-07-09 14:48:05,334 - root - INFO - step: 5460 loss: 17.9478 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.8686 global_avg_mtp_loss: 15.0792 +[titan] 2025-07-09 14:48:05,334 - root - INFO - lr: 2.9865e-04 gnorm: 0.91 [ 1:13:09<21:06:44] +[titan] 2025-07-09 14:48:09,233 - root - INFO - step: 5465 loss: 17.9872 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.8758 global_avg_mtp_loss: 15.1114 +[titan] 2025-07-09 14:48:09,233 - root - INFO - lr: 2.9865e-04 gnorm: 1.01 [ 1:13:13<21:06:38] +[titan] 2025-07-09 14:48:13,157 - root - INFO - step: 5470 loss: 17.9982 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.8808 global_avg_mtp_loss: 15.1174 +[titan] 2025-07-09 14:48:13,157 - root - INFO - lr: 2.9864e-04 gnorm: 0.95 [ 1:13:17<21:06:32] +[titan] 2025-07-09 14:48:17,064 - root - INFO - step: 5475 loss: 17.7936 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.8325 global_avg_mtp_loss: 14.9611 +[titan] 2025-07-09 14:48:17,064 - root - INFO - lr: 2.9864e-04 gnorm: 1.00 [ 1:13:21<21:06:26] +[titan] 2025-07-09 14:48:20,959 - root - INFO - step: 5480 loss: 17.7797 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.8274 global_avg_mtp_loss: 14.9523 +[titan] 2025-07-09 14:48:20,959 - root - INFO - lr: 2.9864e-04 gnorm: 0.93 [ 1:13:25<21:06:20] +[titan] 2025-07-09 14:48:24,849 - root - INFO - step: 5485 loss: 17.7155 memory: 44.58GiB(31.99%) tps: 84,237 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.8212 global_avg_mtp_loss: 14.8943 +[titan] 2025-07-09 14:48:24,849 - root - INFO - lr: 2.9863e-04 gnorm: 0.97 [ 1:13:29<21:06:14] +[titan] 2025-07-09 14:48:28,741 - root - INFO - step: 5490 loss: 17.8128 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.8328 global_avg_mtp_loss: 14.9801 +[titan] 2025-07-09 14:48:28,742 - root - INFO - lr: 2.9863e-04 gnorm: 1.00 [ 1:13:32<21:06:08] +[titan] 2025-07-09 14:48:32,646 - root - INFO - step: 5495 loss: 17.8840 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.8559 global_avg_mtp_loss: 15.0281 +[titan] 2025-07-09 14:48:32,647 - root - INFO - lr: 2.9863e-04 gnorm: 1.01 [ 1:13:36<21:06:02] +[titan] 2025-07-09 14:48:35,773 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:48:36,557 - root - INFO - step: 5500 loss: 17.8240 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.8463 global_avg_mtp_loss: 14.9777 +[titan] 2025-07-09 14:48:36,557 - root - INFO - lr: 2.9863e-04 gnorm: 1.07 [ 1:13:40<21:05:56] +[titan] 2025-07-09 14:48:40,444 - root - INFO - step: 5505 loss: 17.7894 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.8385 global_avg_mtp_loss: 14.9509 +[titan] 2025-07-09 14:48:40,444 - root - INFO - lr: 2.9862e-04 gnorm: 0.91 [ 1:13:44<21:05:49] +[titan] 2025-07-09 14:48:44,350 - root - INFO - step: 5510 loss: 17.9798 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8710 global_avg_mtp_loss: 15.1088 +[titan] 2025-07-09 14:48:44,350 - root - INFO - lr: 2.9862e-04 gnorm: 1.09 [ 1:13:48<21:05:44] +[titan] 2025-07-09 14:48:48,240 - root - INFO - step: 5515 loss: 17.9076 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.8508 global_avg_mtp_loss: 15.0568 +[titan] 2025-07-09 14:48:48,240 - root - INFO - lr: 2.9862e-04 gnorm: 1.08 [ 1:13:52<21:05:37] +[titan] 2025-07-09 14:48:52,156 - root - INFO - step: 5520 loss: 17.8161 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.8445 global_avg_mtp_loss: 14.9716 +[titan] 2025-07-09 14:48:52,157 - root - INFO - lr: 2.9861e-04 gnorm: 1.00 [ 1:13:56<21:05:31] +[titan] 2025-07-09 14:48:56,058 - root - INFO - step: 5525 loss: 18.0788 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.8879 global_avg_mtp_loss: 15.1909 +[titan] 2025-07-09 14:48:56,059 - root - INFO - lr: 2.9861e-04 gnorm: 0.93 [ 1:14:00<21:05:25] +[titan] 2025-07-09 14:48:59,978 - root - INFO - step: 5530 loss: 17.9823 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.8785 global_avg_mtp_loss: 15.1038 +[titan] 2025-07-09 14:48:59,978 - root - INFO - lr: 2.9861e-04 gnorm: 0.95 [ 1:14:04<21:05:20] +[titan] 2025-07-09 14:49:03,930 - root - INFO - step: 5535 loss: 17.8147 memory: 44.58GiB(31.99%) tps: 82,927 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.8495 global_avg_mtp_loss: 14.9652 +[titan] 2025-07-09 14:49:03,930 - root - INFO - lr: 2.9860e-04 gnorm: 0.96 [ 1:14:08<21:05:15] +[titan] 2025-07-09 14:49:07,829 - root - INFO - step: 5540 loss: 17.6922 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.8112 global_avg_mtp_loss: 14.8810 +[titan] 2025-07-09 14:49:07,829 - root - INFO - lr: 2.9860e-04 gnorm: 0.95 [ 1:14:12<21:05:09] +[titan] 2025-07-09 14:49:11,733 - root - INFO - step: 5545 loss: 17.8254 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.8226 global_avg_mtp_loss: 15.0028 +[titan] 2025-07-09 14:49:11,734 - root - INFO - lr: 2.9860e-04 gnorm: 0.98 [ 1:14:15<21:05:03] +[titan] 2025-07-09 14:49:14,846 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:49:15,639 - root - INFO - step: 5550 loss: 17.9328 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.8617 global_avg_mtp_loss: 15.0711 +[titan] 2025-07-09 14:49:15,639 - root - INFO - lr: 2.9859e-04 gnorm: 0.94 [ 1:14:19<21:04:57] +[titan] 2025-07-09 14:49:19,528 - root - INFO - step: 5555 loss: 17.7387 memory: 44.58GiB(31.99%) tps: 84,271 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.8300 global_avg_mtp_loss: 14.9087 +[titan] 2025-07-09 14:49:19,528 - root - INFO - lr: 2.9859e-04 gnorm: 0.98 [ 1:14:23<21:04:50] +[titan] 2025-07-09 14:49:23,445 - root - INFO - step: 5560 loss: 17.9683 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.8646 global_avg_mtp_loss: 15.1037 +[titan] 2025-07-09 14:49:23,446 - root - INFO - lr: 2.9859e-04 gnorm: 0.99 [ 1:14:27<21:04:45] +[titan] 2025-07-09 14:49:27,383 - root - INFO - step: 5565 loss: 17.6372 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.8021 global_avg_mtp_loss: 14.8351 +[titan] 2025-07-09 14:49:27,383 - root - INFO - lr: 2.9859e-04 gnorm: 0.91 [ 1:14:31<21:04:39] +[titan] 2025-07-09 14:49:31,288 - root - INFO - step: 5570 loss: 17.8748 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.8587 global_avg_mtp_loss: 15.0160 +[titan] 2025-07-09 14:49:31,289 - root - INFO - lr: 2.9858e-04 gnorm: 0.96 [ 1:14:35<21:04:33] +[titan] 2025-07-09 14:49:35,205 - root - INFO - step: 5575 loss: 17.9794 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.8916 global_avg_mtp_loss: 15.0878 +[titan] 2025-07-09 14:49:35,206 - root - INFO - lr: 2.9858e-04 gnorm: 0.96 [ 1:14:39<21:04:28] +[titan] 2025-07-09 14:49:39,135 - root - INFO - step: 5580 loss: 17.7378 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.8150 global_avg_mtp_loss: 14.9228 +[titan] 2025-07-09 14:49:39,135 - root - INFO - lr: 2.9858e-04 gnorm: 1.06 [ 1:14:43<21:04:22] +[titan] 2025-07-09 14:49:43,035 - root - INFO - step: 5585 loss: 17.8077 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.8342 global_avg_mtp_loss: 14.9734 +[titan] 2025-07-09 14:49:43,035 - root - INFO - lr: 2.9857e-04 gnorm: 0.99 [ 1:14:47<21:04:16] +[titan] 2025-07-09 14:49:46,932 - root - INFO - step: 5590 loss: 18.0150 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.9005 global_avg_mtp_loss: 15.1145 +[titan] 2025-07-09 14:49:46,932 - root - INFO - lr: 2.9857e-04 gnorm: 0.95 [ 1:14:51<21:04:10] +[titan] 2025-07-09 14:49:50,843 - root - INFO - step: 5595 loss: 17.5613 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.8001 global_avg_mtp_loss: 14.7612 +[titan] 2025-07-09 14:49:50,843 - root - INFO - lr: 2.9857e-04 gnorm: 0.97 [ 1:14:55<21:04:04] +[titan] 2025-07-09 14:49:53,996 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:49:54,785 - root - INFO - step: 5600 loss: 17.7523 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.8242 global_avg_mtp_loss: 14.9281 +[titan] 2025-07-09 14:49:54,785 - root - INFO - lr: 2.9856e-04 gnorm: 0.98 [ 1:14:58<21:03:59] +[titan] 2025-07-09 14:49:58,713 - root - INFO - step: 5605 loss: 17.5211 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.8025 global_avg_mtp_loss: 14.7186 +[titan] 2025-07-09 14:49:58,713 - root - INFO - lr: 2.9856e-04 gnorm: 1.14 [ 1:15:02<21:03:53] +[titan] 2025-07-09 14:50:02,633 - root - INFO - step: 5610 loss: 17.8494 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.8652 global_avg_mtp_loss: 14.9842 +[titan] 2025-07-09 14:50:02,633 - root - INFO - lr: 2.9856e-04 gnorm: 1.11 [ 1:15:06<21:03:48] +[titan] 2025-07-09 14:50:06,535 - root - INFO - step: 5615 loss: 17.9217 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.8676 global_avg_mtp_loss: 15.0541 +[titan] 2025-07-09 14:50:06,535 - root - INFO - lr: 2.9855e-04 gnorm: 1.02 [ 1:15:10<21:03:42] +[titan] 2025-07-09 14:50:10,455 - root - INFO - step: 5620 loss: 17.7803 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.8456 global_avg_mtp_loss: 14.9347 +[titan] 2025-07-09 14:50:10,456 - root - INFO - lr: 2.9855e-04 gnorm: 1.12 [ 1:15:14<21:03:36] +[titan] 2025-07-09 14:50:14,383 - root - INFO - step: 5625 loss: 17.8220 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.8514 global_avg_mtp_loss: 14.9706 +[titan] 2025-07-09 14:50:14,383 - root - INFO - lr: 2.9855e-04 gnorm: 1.05 [ 1:15:18<21:03:31] +[titan] 2025-07-09 14:50:18,372 - root - INFO - step: 5630 loss: 17.7311 memory: 44.58GiB(31.99%) tps: 82,150 tflops: 283.51 mfu: 28.67% global_avg_ntp_loss: 2.8452 global_avg_mtp_loss: 14.8860 +[titan] 2025-07-09 14:50:18,372 - root - INFO - lr: 2.9854e-04 gnorm: 0.93 [ 1:15:22<21:03:26] +[titan] 2025-07-09 14:50:20,087 - root - INFO - Dumping profiler traces at step 5632 +[titan] 2025-07-09 14:50:20,118 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:50:22,501 - root - INFO - step: 5635 loss: 17.7181 memory: 44.58GiB(31.99%) tps: 79,361 tflops: 273.89 mfu: 27.69% global_avg_ntp_loss: 2.8132 global_avg_mtp_loss: 14.9050 +[titan] 2025-07-09 14:50:22,501 - root - INFO - lr: 2.9854e-04 gnorm: 1.07 [ 1:15:26<21:03:24] +[titan] 2025-07-09 14:50:26,457 - root - INFO - step: 5640 loss: 17.9972 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.8831 global_avg_mtp_loss: 15.1141 +[titan] 2025-07-09 14:50:26,458 - root - INFO - lr: 2.9854e-04 gnorm: 0.98 [ 1:15:30<21:03:19] +[titan] 2025-07-09 14:50:30,416 - root - INFO - step: 5645 loss: 17.7406 memory: 44.58GiB(31.99%) tps: 82,784 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.8198 global_avg_mtp_loss: 14.9208 +[titan] 2025-07-09 14:50:30,416 - root - INFO - lr: 2.9854e-04 gnorm: 1.00 [ 1:15:34<21:03:14] +[titan] 2025-07-09 14:50:33,563 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:50:34,361 - root - INFO - step: 5650 loss: 17.7693 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.8280 global_avg_mtp_loss: 14.9413 +[titan] 2025-07-09 14:50:34,361 - root - INFO - lr: 2.9853e-04 gnorm: 1.07 [ 1:15:38<21:03:09] +[titan] 2025-07-09 14:50:38,283 - root - INFO - step: 5655 loss: 17.9477 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.8703 global_avg_mtp_loss: 15.0774 +[titan] 2025-07-09 14:50:38,284 - root - INFO - lr: 2.9853e-04 gnorm: 0.99 [ 1:15:42<21:03:03] +[titan] 2025-07-09 14:50:42,190 - root - INFO - step: 5660 loss: 17.8744 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8444 global_avg_mtp_loss: 15.0301 +[titan] 2025-07-09 14:50:42,190 - root - INFO - lr: 2.9853e-04 gnorm: 1.16 [ 1:15:46<21:02:57] +[titan] 2025-07-09 14:50:46,096 - root - INFO - step: 5665 loss: 17.8685 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.8544 global_avg_mtp_loss: 15.0141 +[titan] 2025-07-09 14:50:46,096 - root - INFO - lr: 2.9852e-04 gnorm: 0.98 [ 1:15:50<21:02:52] +[titan] 2025-07-09 14:50:50,019 - root - INFO - step: 5670 loss: 17.8950 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.8547 global_avg_mtp_loss: 15.0403 +[titan] 2025-07-09 14:50:50,019 - root - INFO - lr: 2.9852e-04 gnorm: 1.01 [ 1:15:54<21:02:46] +[titan] 2025-07-09 14:50:53,937 - root - INFO - step: 5675 loss: 17.6899 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.8139 global_avg_mtp_loss: 14.8760 +[titan] 2025-07-09 14:50:53,937 - root - INFO - lr: 2.9852e-04 gnorm: 0.84 [ 1:15:58<21:02:40] +[titan] 2025-07-09 14:50:57,888 - root - INFO - step: 5680 loss: 17.8644 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.8494 global_avg_mtp_loss: 15.0151 +[titan] 2025-07-09 14:50:57,888 - root - INFO - lr: 2.9851e-04 gnorm: 0.94 [ 1:16:02<21:02:35] +[titan] 2025-07-09 14:51:01,811 - root - INFO - step: 5685 loss: 17.6809 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.8183 global_avg_mtp_loss: 14.8626 +[titan] 2025-07-09 14:51:01,812 - root - INFO - lr: 2.9851e-04 gnorm: 0.91 [ 1:16:05<21:02:30] +[titan] 2025-07-09 14:51:05,756 - root - INFO - step: 5690 loss: 17.7017 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.8210 global_avg_mtp_loss: 14.8807 +[titan] 2025-07-09 14:51:05,756 - root - INFO - lr: 2.9851e-04 gnorm: 1.03 [ 1:16:09<21:02:24] +[titan] 2025-07-09 14:51:09,680 - root - INFO - step: 5695 loss: 17.8790 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.8591 global_avg_mtp_loss: 15.0199 +[titan] 2025-07-09 14:51:09,681 - root - INFO - lr: 2.9850e-04 gnorm: 0.91 [ 1:16:13<21:02:19] +[titan] 2025-07-09 14:51:12,812 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:51:13,614 - root - INFO - step: 5700 loss: 18.0433 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.9021 global_avg_mtp_loss: 15.1412 +[titan] 2025-07-09 14:51:13,614 - root - INFO - lr: 2.9850e-04 gnorm: 1.08 [ 1:16:17<21:02:14] +[titan] 2025-07-09 14:51:17,539 - root - INFO - step: 5705 loss: 17.6766 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.8090 global_avg_mtp_loss: 14.8675 +[titan] 2025-07-09 14:51:17,539 - root - INFO - lr: 2.9850e-04 gnorm: 1.05 [ 1:16:21<21:02:08] +[titan] 2025-07-09 14:51:21,501 - root - INFO - step: 5710 loss: 17.8875 memory: 44.58GiB(31.99%) tps: 82,700 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.8530 global_avg_mtp_loss: 15.0345 +[titan] 2025-07-09 14:51:21,502 - root - INFO - lr: 2.9849e-04 gnorm: 1.00 [ 1:16:25<21:02:03] +[titan] 2025-07-09 14:51:25,459 - root - INFO - step: 5715 loss: 17.8288 memory: 44.58GiB(31.99%) tps: 82,812 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.8479 global_avg_mtp_loss: 14.9809 +[titan] 2025-07-09 14:51:25,459 - root - INFO - lr: 2.9849e-04 gnorm: 0.96 [ 1:16:29<21:01:58] +[titan] 2025-07-09 14:51:29,363 - root - INFO - step: 5720 loss: 17.7776 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.8297 global_avg_mtp_loss: 14.9480 +[titan] 2025-07-09 14:51:29,364 - root - INFO - lr: 2.9849e-04 gnorm: 0.91 [ 1:16:33<21:01:52] +[titan] 2025-07-09 14:51:33,266 - root - INFO - step: 5725 loss: 17.7086 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.8240 global_avg_mtp_loss: 14.8846 +[titan] 2025-07-09 14:51:33,266 - root - INFO - lr: 2.9848e-04 gnorm: 0.93 [ 1:16:37<21:01:46] +[titan] 2025-07-09 14:51:37,162 - root - INFO - step: 5730 loss: 17.7665 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.8343 global_avg_mtp_loss: 14.9322 +[titan] 2025-07-09 14:51:37,162 - root - INFO - lr: 2.9848e-04 gnorm: 0.92 [ 1:16:41<21:01:40] +[titan] 2025-07-09 14:51:41,093 - root - INFO - step: 5735 loss: 18.1458 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.9126 global_avg_mtp_loss: 15.2332 +[titan] 2025-07-09 14:51:41,094 - root - INFO - lr: 2.9848e-04 gnorm: 1.02 [ 1:16:45<21:01:35] +[titan] 2025-07-09 14:51:44,990 - root - INFO - step: 5740 loss: 17.6755 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.8216 global_avg_mtp_loss: 14.8539 +[titan] 2025-07-09 14:51:44,990 - root - INFO - lr: 2.9848e-04 gnorm: 1.02 [ 1:16:49<21:01:29] +[titan] 2025-07-09 14:51:48,892 - root - INFO - step: 5745 loss: 17.7485 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.8154 global_avg_mtp_loss: 14.9331 +[titan] 2025-07-09 14:51:48,893 - root - INFO - lr: 2.9847e-04 gnorm: 0.98 [ 1:16:53<21:01:23] +[titan] 2025-07-09 14:51:52,010 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:51:52,798 - root - INFO - step: 5750 loss: 17.6878 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.8203 global_avg_mtp_loss: 14.8675 +[titan] 2025-07-09 14:51:52,798 - root - INFO - lr: 2.9847e-04 gnorm: 0.96 [ 1:16:56<21:01:17] +[titan] 2025-07-09 14:51:56,700 - root - INFO - step: 5755 loss: 17.7419 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.8381 global_avg_mtp_loss: 14.9038 +[titan] 2025-07-09 14:51:56,700 - root - INFO - lr: 2.9847e-04 gnorm: 1.06 [ 1:17:00<21:01:12] +[titan] 2025-07-09 14:52:00,644 - root - INFO - step: 5760 loss: 17.7511 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.8369 global_avg_mtp_loss: 14.9142 +[titan] 2025-07-09 14:52:00,644 - root - INFO - lr: 2.9846e-04 gnorm: 1.01 [ 1:17:04<21:01:06] +[titan] 2025-07-09 14:52:04,547 - root - INFO - step: 5765 loss: 17.8924 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.8554 global_avg_mtp_loss: 15.0369 +[titan] 2025-07-09 14:52:04,547 - root - INFO - lr: 2.9846e-04 gnorm: 1.00 [ 1:17:08<21:01:00] +[titan] 2025-07-09 14:52:08,452 - root - INFO - step: 5770 loss: 17.6744 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.8138 global_avg_mtp_loss: 14.8607 +[titan] 2025-07-09 14:52:08,453 - root - INFO - lr: 2.9846e-04 gnorm: 0.98 [ 1:17:12<21:00:55] +[titan] 2025-07-09 14:52:12,374 - root - INFO - step: 5775 loss: 17.7052 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.8194 global_avg_mtp_loss: 14.8859 +[titan] 2025-07-09 14:52:12,375 - root - INFO - lr: 2.9845e-04 gnorm: 0.93 [ 1:17:16<21:00:49] +[titan] 2025-07-09 14:52:16,276 - root - INFO - step: 5780 loss: 17.7028 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.8176 global_avg_mtp_loss: 14.8851 +[titan] 2025-07-09 14:52:16,277 - root - INFO - lr: 2.9845e-04 gnorm: 1.01 [ 1:17:20<21:00:43] +[titan] 2025-07-09 14:52:20,241 - root - INFO - step: 5785 loss: 17.8452 memory: 44.58GiB(31.99%) tps: 82,668 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 2.8484 global_avg_mtp_loss: 14.9968 +[titan] 2025-07-09 14:52:20,241 - root - INFO - lr: 2.9845e-04 gnorm: 1.04 [ 1:17:24<21:00:38] +[titan] 2025-07-09 14:52:24,154 - root - INFO - step: 5790 loss: 17.8710 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.8475 global_avg_mtp_loss: 15.0235 +[titan] 2025-07-09 14:52:24,154 - root - INFO - lr: 2.9844e-04 gnorm: 1.04 [ 1:17:28<21:00:33] +[titan] 2025-07-09 14:52:28,076 - root - INFO - step: 5795 loss: 18.1044 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.8930 global_avg_mtp_loss: 15.2114 +[titan] 2025-07-09 14:52:28,076 - root - INFO - lr: 2.9844e-04 gnorm: 1.05 [ 1:17:32<21:00:27] +[titan] 2025-07-09 14:52:31,202 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:52:32,002 - root - INFO - step: 5800 loss: 17.8438 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.8573 global_avg_mtp_loss: 14.9865 +[titan] 2025-07-09 14:52:32,002 - root - INFO - lr: 2.9844e-04 gnorm: 0.89 [ 1:17:36<21:00:22] +[titan] 2025-07-09 14:52:35,954 - root - INFO - step: 5805 loss: 17.7089 memory: 44.58GiB(31.99%) tps: 82,920 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.8173 global_avg_mtp_loss: 14.8916 +[titan] 2025-07-09 14:52:35,955 - root - INFO - lr: 2.9843e-04 gnorm: 0.98 [ 1:17:40<21:00:17] +[titan] 2025-07-09 14:52:39,864 - root - INFO - step: 5810 loss: 18.0449 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.8808 global_avg_mtp_loss: 15.1641 +[titan] 2025-07-09 14:52:39,864 - root - INFO - lr: 2.9843e-04 gnorm: 0.99 [ 1:17:44<21:00:11] +[titan] 2025-07-09 14:52:43,744 - root - INFO - step: 5815 loss: 17.8730 memory: 44.58GiB(31.99%) tps: 84,454 tflops: 291.47 mfu: 29.47% global_avg_ntp_loss: 2.8524 global_avg_mtp_loss: 15.0205 +[titan] 2025-07-09 14:52:43,745 - root - INFO - lr: 2.9843e-04 gnorm: 0.95 [ 1:17:47<21:00:05] +[titan] 2025-07-09 14:52:47,648 - root - INFO - step: 5820 loss: 17.6035 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.8087 global_avg_mtp_loss: 14.7948 +[titan] 2025-07-09 14:52:47,649 - root - INFO - lr: 2.9842e-04 gnorm: 0.98 [ 1:17:51<20:59:59] +[titan] 2025-07-09 14:52:51,555 - root - INFO - step: 5825 loss: 17.5908 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.7996 global_avg_mtp_loss: 14.7912 +[titan] 2025-07-09 14:52:51,555 - root - INFO - lr: 2.9842e-04 gnorm: 0.93 [ 1:17:55<20:59:53] +[titan] 2025-07-09 14:52:55,494 - root - INFO - step: 5830 loss: 17.7328 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.8293 global_avg_mtp_loss: 14.9035 +[titan] 2025-07-09 14:52:55,494 - root - INFO - lr: 2.9842e-04 gnorm: 1.07 [ 1:17:59<20:59:48] +[titan] 2025-07-09 14:52:59,409 - root - INFO - step: 5835 loss: 17.6620 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.8135 global_avg_mtp_loss: 14.8485 +[titan] 2025-07-09 14:52:59,410 - root - INFO - lr: 2.9841e-04 gnorm: 1.00 [ 1:18:03<20:59:43] +[titan] 2025-07-09 14:53:03,306 - root - INFO - step: 5840 loss: 17.8851 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.8615 global_avg_mtp_loss: 15.0236 +[titan] 2025-07-09 14:53:03,306 - root - INFO - lr: 2.9841e-04 gnorm: 0.94 [ 1:18:07<20:59:37] +[titan] 2025-07-09 14:53:07,216 - root - INFO - step: 5845 loss: 17.7614 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.8299 global_avg_mtp_loss: 14.9315 +[titan] 2025-07-09 14:53:07,216 - root - INFO - lr: 2.9841e-04 gnorm: 0.99 [ 1:18:11<20:59:31] +[titan] 2025-07-09 14:53:10,318 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:53:11,101 - root - INFO - step: 5850 loss: 17.9099 memory: 44.58GiB(31.99%) tps: 84,342 tflops: 291.08 mfu: 29.43% global_avg_ntp_loss: 2.8538 global_avg_mtp_loss: 15.0561 +[titan] 2025-07-09 14:53:11,102 - root - INFO - lr: 2.9840e-04 gnorm: 0.92 [ 1:18:15<20:59:25] +[titan] 2025-07-09 14:53:15,007 - root - INFO - step: 5855 loss: 18.0042 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.9008 global_avg_mtp_loss: 15.1035 +[titan] 2025-07-09 14:53:15,007 - root - INFO - lr: 2.9840e-04 gnorm: 1.02 [ 1:18:19<20:59:19] +[titan] 2025-07-09 14:53:18,899 - root - INFO - step: 5860 loss: 17.8436 memory: 44.58GiB(31.99%) tps: 84,199 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.8583 global_avg_mtp_loss: 14.9853 +[titan] 2025-07-09 14:53:18,899 - root - INFO - lr: 2.9840e-04 gnorm: 1.02 [ 1:18:23<20:59:13] +[titan] 2025-07-09 14:53:22,791 - root - INFO - step: 5865 loss: 17.9294 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.8555 global_avg_mtp_loss: 15.0740 +[titan] 2025-07-09 14:53:22,792 - root - INFO - lr: 2.9839e-04 gnorm: 1.15 [ 1:18:26<20:59:07] +[titan] 2025-07-09 14:53:26,750 - root - INFO - step: 5870 loss: 17.7110 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.67 mfu: 28.89% global_avg_ntp_loss: 2.8133 global_avg_mtp_loss: 14.8976 +[titan] 2025-07-09 14:53:26,751 - root - INFO - lr: 2.9839e-04 gnorm: 1.00 [ 1:18:30<20:59:02] +[titan] 2025-07-09 14:53:30,660 - root - INFO - step: 5875 loss: 18.0274 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.8889 global_avg_mtp_loss: 15.1384 +[titan] 2025-07-09 14:53:30,660 - root - INFO - lr: 2.9839e-04 gnorm: 0.92 [ 1:18:34<20:58:57] +[titan] 2025-07-09 14:53:34,561 - root - INFO - step: 5880 loss: 17.7370 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.8197 global_avg_mtp_loss: 14.9173 +[titan] 2025-07-09 14:53:34,562 - root - INFO - lr: 2.9838e-04 gnorm: 0.96 [ 1:18:38<20:58:51] +[titan] 2025-07-09 14:53:38,486 - root - INFO - step: 5885 loss: 17.7837 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.8324 global_avg_mtp_loss: 14.9514 +[titan] 2025-07-09 14:53:38,486 - root - INFO - lr: 2.9838e-04 gnorm: 0.94 [ 1:18:42<20:58:45] +[titan] 2025-07-09 14:53:42,438 - root - INFO - step: 5890 loss: 17.9649 memory: 44.58GiB(31.99%) tps: 82,923 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.8655 global_avg_mtp_loss: 15.0994 +[titan] 2025-07-09 14:53:42,438 - root - INFO - lr: 2.9838e-04 gnorm: 0.94 [ 1:18:46<20:58:40] +[titan] 2025-07-09 14:53:46,349 - root - INFO - step: 5895 loss: 17.8704 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.8519 global_avg_mtp_loss: 15.0185 +[titan] 2025-07-09 14:53:46,349 - root - INFO - lr: 2.9837e-04 gnorm: 0.99 [ 1:18:50<20:58:35] +[titan] 2025-07-09 14:53:49,450 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:53:50,235 - root - INFO - step: 5900 loss: 17.6858 memory: 44.58GiB(31.99%) tps: 84,316 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.8091 global_avg_mtp_loss: 14.8767 +[titan] 2025-07-09 14:53:50,236 - root - INFO - lr: 2.9837e-04 gnorm: 0.97 [ 1:18:54<20:58:29] +[titan] 2025-07-09 14:53:54,142 - root - INFO - step: 5905 loss: 17.8199 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.8437 global_avg_mtp_loss: 14.9762 +[titan] 2025-07-09 14:53:54,142 - root - INFO - lr: 2.9837e-04 gnorm: 0.96 [ 1:18:58<20:58:23] +[titan] 2025-07-09 14:53:58,067 - root - INFO - step: 5910 loss: 17.3928 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.7624 global_avg_mtp_loss: 14.6305 +[titan] 2025-07-09 14:53:58,067 - root - INFO - lr: 2.9836e-04 gnorm: 1.00 [ 1:19:02<20:58:18] +[titan] 2025-07-09 14:54:01,972 - root - INFO - step: 5915 loss: 17.8258 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.8479 global_avg_mtp_loss: 14.9779 +[titan] 2025-07-09 14:54:01,972 - root - INFO - lr: 2.9836e-04 gnorm: 0.95 [ 1:19:06<20:58:12] +[titan] 2025-07-09 14:54:05,886 - root - INFO - step: 5920 loss: 17.9255 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.8615 global_avg_mtp_loss: 15.0640 +[titan] 2025-07-09 14:54:05,887 - root - INFO - lr: 2.9836e-04 gnorm: 1.04 [ 1:19:10<20:58:06] +[titan] 2025-07-09 14:54:09,820 - root - INFO - step: 5925 loss: 17.6575 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.8160 global_avg_mtp_loss: 14.8414 +[titan] 2025-07-09 14:54:09,820 - root - INFO - lr: 2.9835e-04 gnorm: 0.91 [ 1:19:13<20:58:01] +[titan] 2025-07-09 14:54:13,730 - root - INFO - step: 5930 loss: 17.7714 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.8358 global_avg_mtp_loss: 14.9355 +[titan] 2025-07-09 14:54:13,730 - root - INFO - lr: 2.9835e-04 gnorm: 0.98 [ 1:19:17<20:57:55] +[titan] 2025-07-09 14:54:17,708 - root - INFO - step: 5935 loss: 17.8702 memory: 44.58GiB(31.99%) tps: 82,373 tflops: 284.28 mfu: 28.74% global_avg_ntp_loss: 2.8445 global_avg_mtp_loss: 15.0257 +[titan] 2025-07-09 14:54:17,708 - root - INFO - lr: 2.9835e-04 gnorm: 0.98 [ 1:19:21<20:57:51] +[titan] 2025-07-09 14:54:21,621 - root - INFO - step: 5940 loss: 17.8176 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.8400 global_avg_mtp_loss: 14.9776 +[titan] 2025-07-09 14:54:21,621 - root - INFO - lr: 2.9834e-04 gnorm: 1.00 [ 1:19:25<20:57:45] +[titan] 2025-07-09 14:54:25,513 - root - INFO - step: 5945 loss: 17.8643 memory: 44.58GiB(31.99%) tps: 84,193 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.8569 global_avg_mtp_loss: 15.0074 +[titan] 2025-07-09 14:54:25,514 - root - INFO - lr: 2.9834e-04 gnorm: 0.92 [ 1:19:29<20:57:39] +[titan] 2025-07-09 14:54:28,648 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:54:29,445 - root - INFO - step: 5950 loss: 17.5204 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.7765 global_avg_mtp_loss: 14.7439 +[titan] 2025-07-09 14:54:29,445 - root - INFO - lr: 2.9834e-04 gnorm: 1.02 [ 1:19:33<20:57:34] +[titan] 2025-07-09 14:54:33,361 - root - INFO - step: 5955 loss: 17.9916 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.8744 global_avg_mtp_loss: 15.1172 +[titan] 2025-07-09 14:54:33,361 - root - INFO - lr: 2.9833e-04 gnorm: 0.98 [ 1:19:37<20:57:29] +[titan] 2025-07-09 14:54:37,267 - root - INFO - step: 5960 loss: 17.7629 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8437 global_avg_mtp_loss: 14.9192 +[titan] 2025-07-09 14:54:37,267 - root - INFO - lr: 2.9833e-04 gnorm: 1.02 [ 1:19:41<20:57:23] +[titan] 2025-07-09 14:54:41,219 - root - INFO - step: 5965 loss: 17.6411 memory: 44.58GiB(31.99%) tps: 82,921 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.7942 global_avg_mtp_loss: 14.8470 +[titan] 2025-07-09 14:54:41,219 - root - INFO - lr: 2.9833e-04 gnorm: 0.99 [ 1:19:45<20:57:18] +[titan] 2025-07-09 14:54:45,128 - root - INFO - step: 5970 loss: 17.7932 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.8379 global_avg_mtp_loss: 14.9553 +[titan] 2025-07-09 14:54:45,128 - root - INFO - lr: 2.9832e-04 gnorm: 1.07 [ 1:19:49<20:57:12] +[titan] 2025-07-09 14:54:49,044 - root - INFO - step: 5975 loss: 17.8785 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.8467 global_avg_mtp_loss: 15.0318 +[titan] 2025-07-09 14:54:49,044 - root - INFO - lr: 2.9832e-04 gnorm: 0.95 [ 1:19:53<20:57:07] +[titan] 2025-07-09 14:54:52,956 - root - INFO - step: 5980 loss: 17.6526 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.7946 global_avg_mtp_loss: 14.8580 +[titan] 2025-07-09 14:54:52,957 - root - INFO - lr: 2.9832e-04 gnorm: 0.94 [ 1:19:57<20:57:01] +[titan] 2025-07-09 14:54:56,896 - root - INFO - step: 5985 loss: 17.6076 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.8179 global_avg_mtp_loss: 14.7898 +[titan] 2025-07-09 14:54:56,896 - root - INFO - lr: 2.9831e-04 gnorm: 1.01 [ 1:20:01<20:56:56] +[titan] 2025-07-09 14:55:00,811 - root - INFO - step: 5990 loss: 17.4957 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7670 global_avg_mtp_loss: 14.7287 +[titan] 2025-07-09 14:55:00,811 - root - INFO - lr: 2.9831e-04 gnorm: 1.01 [ 1:20:04<20:56:51] +[titan] 2025-07-09 14:55:04,717 - root - INFO - step: 5995 loss: 17.9892 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8675 global_avg_mtp_loss: 15.1217 +[titan] 2025-07-09 14:55:04,717 - root - INFO - lr: 2.9831e-04 gnorm: 0.96 [ 1:20:08<20:56:45] +[titan] 2025-07-09 14:55:07,847 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:55:08,634 - root - INFO - step: 6000 loss: 17.9210 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.8548 global_avg_mtp_loss: 15.0662 +[titan] 2025-07-09 14:55:08,635 - root - INFO - lr: 2.9830e-04 gnorm: 0.89 [ 1:20:12<20:56:39] +[titan] 2025-07-09 14:55:12,612 - root - INFO - step: 6005 loss: 17.7002 memory: 44.58GiB(31.99%) tps: 82,383 tflops: 284.32 mfu: 28.75% global_avg_ntp_loss: 2.8108 global_avg_mtp_loss: 14.8894 +[titan] 2025-07-09 14:55:12,613 - root - INFO - lr: 2.9830e-04 gnorm: 0.99 [ 1:20:16<20:56:35] +[titan] 2025-07-09 14:55:16,587 - root - INFO - step: 6010 loss: 17.8421 memory: 44.58GiB(31.99%) tps: 82,450 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.8455 global_avg_mtp_loss: 14.9966 +[titan] 2025-07-09 14:55:16,587 - root - INFO - lr: 2.9830e-04 gnorm: 0.93 [ 1:20:20<20:56:30] +[titan] 2025-07-09 14:55:20,496 - root - INFO - step: 6015 loss: 17.6126 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.8144 global_avg_mtp_loss: 14.7982 +[titan] 2025-07-09 14:55:20,496 - root - INFO - lr: 2.9829e-04 gnorm: 0.96 [ 1:20:24<20:56:25] +[titan] 2025-07-09 14:55:24,412 - root - INFO - step: 6020 loss: 17.8107 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.8392 global_avg_mtp_loss: 14.9715 +[titan] 2025-07-09 14:55:24,412 - root - INFO - lr: 2.9829e-04 gnorm: 0.91 [ 1:20:28<20:56:19] +[titan] 2025-07-09 14:55:28,325 - root - INFO - step: 6025 loss: 17.8015 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.8434 global_avg_mtp_loss: 14.9581 +[titan] 2025-07-09 14:55:28,325 - root - INFO - lr: 2.9829e-04 gnorm: 0.92 [ 1:20:32<20:56:14] +[titan] 2025-07-09 14:55:32,251 - root - INFO - step: 6030 loss: 17.7329 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.8295 global_avg_mtp_loss: 14.9035 +[titan] 2025-07-09 14:55:32,251 - root - INFO - lr: 2.9828e-04 gnorm: 0.97 [ 1:20:36<20:56:08] +[titan] 2025-07-09 14:55:36,140 - root - INFO - step: 6035 loss: 17.7081 memory: 44.58GiB(31.99%) tps: 84,281 tflops: 290.87 mfu: 29.41% global_avg_ntp_loss: 2.8147 global_avg_mtp_loss: 14.8935 +[titan] 2025-07-09 14:55:36,140 - root - INFO - lr: 2.9828e-04 gnorm: 1.01 [ 1:20:40<20:56:02] +[titan] 2025-07-09 14:55:40,055 - root - INFO - step: 6040 loss: 17.7417 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.8236 global_avg_mtp_loss: 14.9181 +[titan] 2025-07-09 14:55:40,055 - root - INFO - lr: 2.9828e-04 gnorm: 0.97 [ 1:20:44<20:55:57] +[titan] 2025-07-09 14:55:43,987 - root - INFO - step: 6045 loss: 17.7222 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.8215 global_avg_mtp_loss: 14.9007 +[titan] 2025-07-09 14:55:43,988 - root - INFO - lr: 2.9827e-04 gnorm: 0.91 [ 1:20:48<20:55:52] +[titan] 2025-07-09 14:55:47,108 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:55:47,891 - root - INFO - step: 6050 loss: 17.7447 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.8350 global_avg_mtp_loss: 14.9097 +[titan] 2025-07-09 14:55:47,892 - root - INFO - lr: 2.9827e-04 gnorm: 1.05 [ 1:20:52<20:55:46] +[titan] 2025-07-09 14:55:51,798 - root - INFO - step: 6055 loss: 17.7169 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8148 global_avg_mtp_loss: 14.9021 +[titan] 2025-07-09 14:55:51,798 - root - INFO - lr: 2.9827e-04 gnorm: 0.95 [ 1:20:55<20:55:40] +[titan] 2025-07-09 14:55:55,711 - root - INFO - step: 6060 loss: 17.4644 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.7743 global_avg_mtp_loss: 14.6901 +[titan] 2025-07-09 14:55:55,711 - root - INFO - lr: 2.9826e-04 gnorm: 0.92 [ 1:20:59<20:55:35] +[titan] 2025-07-09 14:55:59,633 - root - INFO - step: 6065 loss: 18.0407 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.8764 global_avg_mtp_loss: 15.1643 +[titan] 2025-07-09 14:55:59,634 - root - INFO - lr: 2.9826e-04 gnorm: 0.96 [ 1:21:03<20:55:30] +[titan] 2025-07-09 14:56:03,553 - root - INFO - step: 6070 loss: 17.8656 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.8376 global_avg_mtp_loss: 15.0280 +[titan] 2025-07-09 14:56:03,553 - root - INFO - lr: 2.9826e-04 gnorm: 0.93 [ 1:21:07<20:55:24] +[titan] 2025-07-09 14:56:07,459 - root - INFO - step: 6075 loss: 17.8231 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.8508 global_avg_mtp_loss: 14.9723 +[titan] 2025-07-09 14:56:07,459 - root - INFO - lr: 2.9825e-04 gnorm: 0.90 [ 1:21:11<20:55:19] +[titan] 2025-07-09 14:56:11,363 - root - INFO - step: 6080 loss: 17.4743 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.7637 global_avg_mtp_loss: 14.7105 +[titan] 2025-07-09 14:56:11,363 - root - INFO - lr: 2.9825e-04 gnorm: 1.02 [ 1:21:15<20:55:13] +[titan] 2025-07-09 14:56:15,264 - root - INFO - step: 6085 loss: 17.8535 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.8459 global_avg_mtp_loss: 15.0076 +[titan] 2025-07-09 14:56:15,264 - root - INFO - lr: 2.9825e-04 gnorm: 0.97 [ 1:21:19<20:55:07] +[titan] 2025-07-09 14:56:19,183 - root - INFO - step: 6090 loss: 17.8496 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.8379 global_avg_mtp_loss: 15.0117 +[titan] 2025-07-09 14:56:19,183 - root - INFO - lr: 2.9824e-04 gnorm: 1.01 [ 1:21:23<20:55:02] +[titan] 2025-07-09 14:56:23,115 - root - INFO - step: 6095 loss: 17.5294 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.7963 global_avg_mtp_loss: 14.7331 +[titan] 2025-07-09 14:56:23,116 - root - INFO - lr: 2.9824e-04 gnorm: 0.92 [ 1:21:27<20:54:57] +[titan] 2025-07-09 14:56:26,229 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:56:27,026 - root - INFO - step: 6100 loss: 17.7278 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.8167 global_avg_mtp_loss: 14.9111 +[titan] 2025-07-09 14:56:27,026 - root - INFO - lr: 2.9824e-04 gnorm: 0.98 [ 1:21:31<20:54:51] +[titan] 2025-07-09 14:56:30,923 - root - INFO - step: 6105 loss: 17.7937 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.8387 global_avg_mtp_loss: 14.9550 +[titan] 2025-07-09 14:56:30,923 - root - INFO - lr: 2.9823e-04 gnorm: 0.93 [ 1:21:35<20:54:45] +[titan] 2025-07-09 14:56:34,825 - root - INFO - step: 6110 loss: 17.6459 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.8062 global_avg_mtp_loss: 14.8398 +[titan] 2025-07-09 14:56:34,825 - root - INFO - lr: 2.9823e-04 gnorm: 1.01 [ 1:21:38<20:54:40] +[titan] 2025-07-09 14:56:38,725 - root - INFO - step: 6115 loss: 17.7642 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.8304 global_avg_mtp_loss: 14.9338 +[titan] 2025-07-09 14:56:38,725 - root - INFO - lr: 2.9822e-04 gnorm: 1.01 [ 1:21:42<20:54:34] +[titan] 2025-07-09 14:56:42,648 - root - INFO - step: 6120 loss: 17.6331 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.8072 global_avg_mtp_loss: 14.8258 +[titan] 2025-07-09 14:56:42,648 - root - INFO - lr: 2.9822e-04 gnorm: 0.97 [ 1:21:46<20:54:29] +[titan] 2025-07-09 14:56:46,537 - root - INFO - step: 6125 loss: 17.5603 memory: 44.58GiB(31.99%) tps: 84,269 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.7962 global_avg_mtp_loss: 14.7642 +[titan] 2025-07-09 14:56:46,537 - root - INFO - lr: 2.9822e-04 gnorm: 1.15 [ 1:21:50<20:54:23] +[titan] 2025-07-09 14:56:50,457 - root - INFO - step: 6130 loss: 17.6817 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.8001 global_avg_mtp_loss: 14.8816 +[titan] 2025-07-09 14:56:50,457 - root - INFO - lr: 2.9821e-04 gnorm: 0.97 [ 1:21:54<20:54:17] +[titan] 2025-07-09 14:56:54,394 - root - INFO - step: 6135 loss: 18.0599 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.8919 global_avg_mtp_loss: 15.1680 +[titan] 2025-07-09 14:56:54,394 - root - INFO - lr: 2.9821e-04 gnorm: 1.12 [ 1:21:58<20:54:12] +[titan] 2025-07-09 14:56:58,316 - root - INFO - step: 6140 loss: 17.7018 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.8200 global_avg_mtp_loss: 14.8818 +[titan] 2025-07-09 14:56:58,316 - root - INFO - lr: 2.9821e-04 gnorm: 0.89 [ 1:22:02<20:54:07] +[titan] 2025-07-09 14:57:01,636 - root - INFO - Dumping profiler traces at step 6144 +[titan] 2025-07-09 14:57:01,668 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 14:57:02,456 - root - INFO - step: 6145 loss: 17.8616 memory: 44.58GiB(31.99%) tps: 79,161 tflops: 273.20 mfu: 27.62% global_avg_ntp_loss: 2.8475 global_avg_mtp_loss: 15.0142 +[titan] 2025-07-09 14:57:02,456 - root - INFO - lr: 2.9820e-04 gnorm: 0.96 [ 1:22:06<20:54:05] +[titan] 2025-07-09 14:57:05,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:57:06,369 - root - INFO - step: 6150 loss: 17.9782 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.8695 global_avg_mtp_loss: 15.1087 +[titan] 2025-07-09 14:57:06,369 - root - INFO - lr: 2.9820e-04 gnorm: 1.01 [ 1:22:10<20:54:00] +[titan] 2025-07-09 14:57:10,293 - root - INFO - step: 6155 loss: 17.7872 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.8291 global_avg_mtp_loss: 14.9581 +[titan] 2025-07-09 14:57:10,294 - root - INFO - lr: 2.9820e-04 gnorm: 0.98 [ 1:22:14<20:53:54] +[titan] 2025-07-09 14:57:14,212 - root - INFO - step: 6160 loss: 17.6988 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.8144 global_avg_mtp_loss: 14.8845 +[titan] 2025-07-09 14:57:14,212 - root - INFO - lr: 2.9819e-04 gnorm: 1.05 [ 1:22:18<20:53:49] +[titan] 2025-07-09 14:57:18,145 - root - INFO - step: 6165 loss: 17.8980 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.8538 global_avg_mtp_loss: 15.0441 +[titan] 2025-07-09 14:57:18,146 - root - INFO - lr: 2.9819e-04 gnorm: 0.94 [ 1:22:22<20:53:44] +[titan] 2025-07-09 14:57:22,042 - root - INFO - step: 6170 loss: 17.8653 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.8498 global_avg_mtp_loss: 15.0154 +[titan] 2025-07-09 14:57:22,042 - root - INFO - lr: 2.9819e-04 gnorm: 1.05 [ 1:22:26<20:53:38] +[titan] 2025-07-09 14:57:25,956 - root - INFO - step: 6175 loss: 17.5169 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7800 global_avg_mtp_loss: 14.7369 +[titan] 2025-07-09 14:57:25,956 - root - INFO - lr: 2.9818e-04 gnorm: 0.94 [ 1:22:30<20:53:33] +[titan] 2025-07-09 14:57:29,933 - root - INFO - step: 6180 loss: 17.7628 memory: 44.58GiB(31.99%) tps: 82,409 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.8243 global_avg_mtp_loss: 14.9385 +[titan] 2025-07-09 14:57:29,933 - root - INFO - lr: 2.9818e-04 gnorm: 1.04 [ 1:22:34<20:53:28] +[titan] 2025-07-09 14:57:33,861 - root - INFO - step: 6185 loss: 17.8271 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.8358 global_avg_mtp_loss: 14.9913 +[titan] 2025-07-09 14:57:33,861 - root - INFO - lr: 2.9818e-04 gnorm: 1.01 [ 1:22:37<20:53:23] +[titan] 2025-07-09 14:57:37,808 - root - INFO - step: 6190 loss: 17.6716 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.7966 global_avg_mtp_loss: 14.8750 +[titan] 2025-07-09 14:57:37,808 - root - INFO - lr: 2.9817e-04 gnorm: 0.89 [ 1:22:41<20:53:18] +[titan] 2025-07-09 14:57:41,716 - root - INFO - step: 6195 loss: 17.9028 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.8584 global_avg_mtp_loss: 15.0444 +[titan] 2025-07-09 14:57:41,716 - root - INFO - lr: 2.9817e-04 gnorm: 0.86 [ 1:22:45<20:53:12] +[titan] 2025-07-09 14:57:44,876 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:57:45,670 - root - INFO - step: 6200 loss: 17.5690 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.7964 global_avg_mtp_loss: 14.7726 +[titan] 2025-07-09 14:57:45,670 - root - INFO - lr: 2.9817e-04 gnorm: 0.93 [ 1:22:49<20:53:07] +[titan] 2025-07-09 14:57:49,574 - root - INFO - step: 6205 loss: 17.8966 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.8431 global_avg_mtp_loss: 15.0535 +[titan] 2025-07-09 14:57:49,575 - root - INFO - lr: 2.9816e-04 gnorm: 0.88 [ 1:22:53<20:53:02] +[titan] 2025-07-09 14:57:53,488 - root - INFO - step: 6210 loss: 17.9470 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.8641 global_avg_mtp_loss: 15.0828 +[titan] 2025-07-09 14:57:53,488 - root - INFO - lr: 2.9816e-04 gnorm: 0.95 [ 1:22:57<20:52:56] +[titan] 2025-07-09 14:57:57,414 - root - INFO - step: 6215 loss: 17.8398 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.8478 global_avg_mtp_loss: 14.9919 +[titan] 2025-07-09 14:57:57,414 - root - INFO - lr: 2.9815e-04 gnorm: 1.03 [ 1:23:01<20:52:51] +[titan] 2025-07-09 14:58:01,334 - root - INFO - step: 6220 loss: 17.9331 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.8611 global_avg_mtp_loss: 15.0720 +[titan] 2025-07-09 14:58:01,334 - root - INFO - lr: 2.9815e-04 gnorm: 0.94 [ 1:23:05<20:52:46] +[titan] 2025-07-09 14:58:05,251 - root - INFO - step: 6225 loss: 17.6616 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.8187 global_avg_mtp_loss: 14.8429 +[titan] 2025-07-09 14:58:05,252 - root - INFO - lr: 2.9815e-04 gnorm: 0.96 [ 1:23:09<20:52:41] +[titan] 2025-07-09 14:58:09,189 - root - INFO - step: 6230 loss: 17.5340 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.7744 global_avg_mtp_loss: 14.7596 +[titan] 2025-07-09 14:58:09,189 - root - INFO - lr: 2.9814e-04 gnorm: 1.05 [ 1:23:13<20:52:35] +[titan] 2025-07-09 14:58:13,122 - root - INFO - step: 6235 loss: 17.6448 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.8129 global_avg_mtp_loss: 14.8319 +[titan] 2025-07-09 14:58:13,122 - root - INFO - lr: 2.9814e-04 gnorm: 0.94 [ 1:23:17<20:52:30] +[titan] 2025-07-09 14:58:17,030 - root - INFO - step: 6240 loss: 17.6974 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.8066 global_avg_mtp_loss: 14.8908 +[titan] 2025-07-09 14:58:17,030 - root - INFO - lr: 2.9814e-04 gnorm: 1.06 [ 1:23:21<20:52:25] +[titan] 2025-07-09 14:58:20,974 - root - INFO - step: 6245 loss: 17.7105 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.8053 global_avg_mtp_loss: 14.9052 +[titan] 2025-07-09 14:58:20,974 - root - INFO - lr: 2.9813e-04 gnorm: 0.97 [ 1:23:25<20:52:20] +[titan] 2025-07-09 14:58:24,097 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:58:24,887 - root - INFO - step: 6250 loss: 17.7502 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.8207 global_avg_mtp_loss: 14.9295 +[titan] 2025-07-09 14:58:24,887 - root - INFO - lr: 2.9813e-04 gnorm: 1.09 [ 1:23:28<20:52:14] +[titan] 2025-07-09 14:58:28,791 - root - INFO - step: 6255 loss: 17.5630 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.8011 global_avg_mtp_loss: 14.7619 +[titan] 2025-07-09 14:58:28,792 - root - INFO - lr: 2.9813e-04 gnorm: 1.04 [ 1:23:32<20:52:09] +[titan] 2025-07-09 14:58:32,679 - root - INFO - step: 6260 loss: 17.6683 memory: 44.58GiB(31.99%) tps: 84,303 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.8080 global_avg_mtp_loss: 14.8603 +[titan] 2025-07-09 14:58:32,679 - root - INFO - lr: 2.9812e-04 gnorm: 1.00 [ 1:23:36<20:52:03] +[titan] 2025-07-09 14:58:36,568 - root - INFO - step: 6265 loss: 17.6445 memory: 44.58GiB(31.99%) tps: 84,257 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.8145 global_avg_mtp_loss: 14.8300 +[titan] 2025-07-09 14:58:36,568 - root - INFO - lr: 2.9812e-04 gnorm: 0.99 [ 1:23:40<20:51:57] +[titan] 2025-07-09 14:58:40,465 - root - INFO - step: 6270 loss: 17.7305 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.8131 global_avg_mtp_loss: 14.9174 +[titan] 2025-07-09 14:58:40,465 - root - INFO - lr: 2.9812e-04 gnorm: 0.96 [ 1:23:44<20:51:52] +[titan] 2025-07-09 14:58:44,379 - root - INFO - step: 6275 loss: 17.8700 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.8676 global_avg_mtp_loss: 15.0024 +[titan] 2025-07-09 14:58:44,380 - root - INFO - lr: 2.9811e-04 gnorm: 0.97 [ 1:23:48<20:51:46] +[titan] 2025-07-09 14:58:48,290 - root - INFO - step: 6280 loss: 17.6146 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.8059 global_avg_mtp_loss: 14.8087 +[titan] 2025-07-09 14:58:48,290 - root - INFO - lr: 2.9811e-04 gnorm: 1.00 [ 1:23:52<20:51:41] +[titan] 2025-07-09 14:58:52,204 - root - INFO - step: 6285 loss: 17.5759 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7870 global_avg_mtp_loss: 14.7889 +[titan] 2025-07-09 14:58:52,204 - root - INFO - lr: 2.9811e-04 gnorm: 0.97 [ 1:23:56<20:51:35] +[titan] 2025-07-09 14:58:56,111 - root - INFO - step: 6290 loss: 17.7945 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.8273 global_avg_mtp_loss: 14.9672 +[titan] 2025-07-09 14:58:56,111 - root - INFO - lr: 2.9810e-04 gnorm: 0.88 [ 1:24:00<20:51:30] +[titan] 2025-07-09 14:59:00,034 - root - INFO - step: 6295 loss: 17.8455 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.8546 global_avg_mtp_loss: 14.9909 +[titan] 2025-07-09 14:59:00,034 - root - INFO - lr: 2.9810e-04 gnorm: 0.90 [ 1:24:04<20:51:25] +[titan] 2025-07-09 14:59:03,162 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:59:03,977 - root - INFO - step: 6300 loss: 17.6956 memory: 44.58GiB(31.99%) tps: 83,110 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.8226 global_avg_mtp_loss: 14.8730 +[titan] 2025-07-09 14:59:03,977 - root - INFO - lr: 2.9809e-04 gnorm: 0.87 [ 1:24:08<20:51:20] +[titan] 2025-07-09 14:59:07,900 - root - INFO - step: 6305 loss: 17.7508 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.8224 global_avg_mtp_loss: 14.9284 +[titan] 2025-07-09 14:59:07,900 - root - INFO - lr: 2.9809e-04 gnorm: 0.96 [ 1:24:12<20:51:14] +[titan] 2025-07-09 14:59:11,829 - root - INFO - step: 6310 loss: 17.7345 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.8177 global_avg_mtp_loss: 14.9168 +[titan] 2025-07-09 14:59:11,829 - root - INFO - lr: 2.9809e-04 gnorm: 0.98 [ 1:24:15<20:51:09] +[titan] 2025-07-09 14:59:15,747 - root - INFO - step: 6315 loss: 17.5114 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.7779 global_avg_mtp_loss: 14.7335 +[titan] 2025-07-09 14:59:15,747 - root - INFO - lr: 2.9808e-04 gnorm: 1.02 [ 1:24:19<20:51:04] +[titan] 2025-07-09 14:59:19,699 - root - INFO - step: 6320 loss: 17.5967 memory: 44.58GiB(31.99%) tps: 82,933 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.7811 global_avg_mtp_loss: 14.8156 +[titan] 2025-07-09 14:59:19,699 - root - INFO - lr: 2.9808e-04 gnorm: 0.95 [ 1:24:23<20:50:59] +[titan] 2025-07-09 14:59:23,631 - root - INFO - step: 6325 loss: 17.6494 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.8116 global_avg_mtp_loss: 14.8378 +[titan] 2025-07-09 14:59:23,631 - root - INFO - lr: 2.9808e-04 gnorm: 1.38 [ 1:24:27<20:50:54] +[titan] 2025-07-09 14:59:27,538 - root - INFO - step: 6330 loss: 17.8552 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.8357 global_avg_mtp_loss: 15.0195 +[titan] 2025-07-09 14:59:27,539 - root - INFO - lr: 2.9807e-04 gnorm: 1.00 [ 1:24:31<20:50:49] +[titan] 2025-07-09 14:59:31,447 - root - INFO - step: 6335 loss: 17.7063 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.8238 global_avg_mtp_loss: 14.8825 +[titan] 2025-07-09 14:59:31,447 - root - INFO - lr: 2.9807e-04 gnorm: 1.09 [ 1:24:35<20:50:43] +[titan] 2025-07-09 14:59:35,349 - root - INFO - step: 6340 loss: 17.9589 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.8608 global_avg_mtp_loss: 15.0982 +[titan] 2025-07-09 14:59:35,350 - root - INFO - lr: 2.9807e-04 gnorm: 1.07 [ 1:24:39<20:50:38] +[titan] 2025-07-09 14:59:39,265 - root - INFO - step: 6345 loss: 17.7012 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.8154 global_avg_mtp_loss: 14.8858 +[titan] 2025-07-09 14:59:39,265 - root - INFO - lr: 2.9806e-04 gnorm: 0.96 [ 1:24:43<20:50:32] +[titan] 2025-07-09 14:59:42,386 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 14:59:43,184 - root - INFO - step: 6350 loss: 17.7769 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.8427 global_avg_mtp_loss: 14.9342 +[titan] 2025-07-09 14:59:43,184 - root - INFO - lr: 2.9806e-04 gnorm: 1.14 [ 1:24:47<20:50:27] +[titan] 2025-07-09 14:59:47,113 - root - INFO - step: 6355 loss: 17.4462 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.7704 global_avg_mtp_loss: 14.6759 +[titan] 2025-07-09 14:59:47,113 - root - INFO - lr: 2.9805e-04 gnorm: 0.93 [ 1:24:51<20:50:22] +[titan] 2025-07-09 14:59:51,010 - root - INFO - step: 6360 loss: 18.2264 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.9462 global_avg_mtp_loss: 15.2803 +[titan] 2025-07-09 14:59:51,011 - root - INFO - lr: 2.9805e-04 gnorm: 1.34 [ 1:24:55<20:50:16] +[titan] 2025-07-09 14:59:54,913 - root - INFO - step: 6365 loss: 17.6527 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.7909 global_avg_mtp_loss: 14.8618 +[titan] 2025-07-09 14:59:54,914 - root - INFO - lr: 2.9805e-04 gnorm: 0.91 [ 1:24:59<20:50:11] +[titan] 2025-07-09 14:59:58,833 - root - INFO - step: 6370 loss: 17.6470 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.8253 global_avg_mtp_loss: 14.8217 +[titan] 2025-07-09 14:59:58,833 - root - INFO - lr: 2.9804e-04 gnorm: 0.94 [ 1:25:02<20:50:05] +[titan] 2025-07-09 15:00:02,767 - root - INFO - step: 6375 loss: 17.6463 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.7984 global_avg_mtp_loss: 14.8479 +[titan] 2025-07-09 15:00:02,767 - root - INFO - lr: 2.9804e-04 gnorm: 0.93 [ 1:25:06<20:50:00] +[titan] 2025-07-09 15:00:06,658 - root - INFO - step: 6380 loss: 17.7856 memory: 44.58GiB(31.99%) tps: 84,214 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.8311 global_avg_mtp_loss: 14.9545 +[titan] 2025-07-09 15:00:06,659 - root - INFO - lr: 2.9804e-04 gnorm: 1.07 [ 1:25:10<20:49:55] +[titan] 2025-07-09 15:00:10,564 - root - INFO - step: 6385 loss: 17.4882 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7812 global_avg_mtp_loss: 14.7070 +[titan] 2025-07-09 15:00:10,564 - root - INFO - lr: 2.9803e-04 gnorm: 1.05 [ 1:25:14<20:49:49] +[titan] 2025-07-09 15:00:14,482 - root - INFO - step: 6390 loss: 17.8412 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.8342 global_avg_mtp_loss: 15.0070 +[titan] 2025-07-09 15:00:14,482 - root - INFO - lr: 2.9803e-04 gnorm: 0.92 [ 1:25:18<20:49:44] +[titan] 2025-07-09 15:00:18,377 - root - INFO - step: 6395 loss: 17.5870 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7870 global_avg_mtp_loss: 14.8000 +[titan] 2025-07-09 15:00:18,377 - root - INFO - lr: 2.9803e-04 gnorm: 0.97 [ 1:25:22<20:49:38] +[titan] 2025-07-09 15:00:21,585 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:00:22,393 - root - INFO - step: 6400 loss: 17.7092 memory: 44.58GiB(31.99%) tps: 81,606 tflops: 281.64 mfu: 28.48% global_avg_ntp_loss: 2.8264 global_avg_mtp_loss: 14.8827 +[titan] 2025-07-09 15:00:22,393 - root - INFO - lr: 2.9802e-04 gnorm: 0.93 [ 1:25:26<20:49:34] +[titan] 2025-07-09 15:00:26,305 - root - INFO - step: 6405 loss: 17.5670 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.7761 global_avg_mtp_loss: 14.7909 +[titan] 2025-07-09 15:00:26,305 - root - INFO - lr: 2.9802e-04 gnorm: 0.94 [ 1:25:30<20:49:29] +[titan] 2025-07-09 15:00:30,201 - root - INFO - step: 6410 loss: 17.7607 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.8334 global_avg_mtp_loss: 14.9273 +[titan] 2025-07-09 15:00:30,201 - root - INFO - lr: 2.9801e-04 gnorm: 0.94 [ 1:25:34<20:49:23] +[titan] 2025-07-09 15:00:34,092 - root - INFO - step: 6415 loss: 17.7477 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.8274 global_avg_mtp_loss: 14.9202 +[titan] 2025-07-09 15:00:34,092 - root - INFO - lr: 2.9801e-04 gnorm: 1.04 [ 1:25:38<20:49:18] +[titan] 2025-07-09 15:00:37,994 - root - INFO - step: 6420 loss: 17.6830 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.7989 global_avg_mtp_loss: 14.8841 +[titan] 2025-07-09 15:00:37,994 - root - INFO - lr: 2.9801e-04 gnorm: 0.93 [ 1:25:42<20:49:12] +[titan] 2025-07-09 15:00:41,931 - root - INFO - step: 6425 loss: 17.7528 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.8148 global_avg_mtp_loss: 14.9381 +[titan] 2025-07-09 15:00:41,931 - root - INFO - lr: 2.9800e-04 gnorm: 1.10 [ 1:25:46<20:49:07] +[titan] 2025-07-09 15:00:45,831 - root - INFO - step: 6430 loss: 17.8344 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.8566 global_avg_mtp_loss: 14.9778 +[titan] 2025-07-09 15:00:45,831 - root - INFO - lr: 2.9800e-04 gnorm: 0.93 [ 1:25:49<20:49:02] +[titan] 2025-07-09 15:00:49,734 - root - INFO - step: 6435 loss: 17.8374 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.8339 global_avg_mtp_loss: 15.0035 +[titan] 2025-07-09 15:00:49,734 - root - INFO - lr: 2.9800e-04 gnorm: 0.94 [ 1:25:53<20:48:56] +[titan] 2025-07-09 15:00:53,686 - root - INFO - step: 6440 loss: 17.7128 memory: 44.58GiB(31.99%) tps: 82,916 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.8114 global_avg_mtp_loss: 14.9013 +[titan] 2025-07-09 15:00:53,686 - root - INFO - lr: 2.9799e-04 gnorm: 1.00 [ 1:25:57<20:48:51] +[titan] 2025-07-09 15:00:57,611 - root - INFO - step: 6445 loss: 18.0490 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.8882 global_avg_mtp_loss: 15.1608 +[titan] 2025-07-09 15:00:57,611 - root - INFO - lr: 2.9799e-04 gnorm: 0.98 [ 1:26:01<20:48:46] +[titan] 2025-07-09 15:01:00,737 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:01:01,532 - root - INFO - step: 6450 loss: 17.7613 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.8457 global_avg_mtp_loss: 14.9156 +[titan] 2025-07-09 15:01:01,532 - root - INFO - lr: 2.9799e-04 gnorm: 0.96 [ 1:26:05<20:48:41] +[titan] 2025-07-09 15:01:05,464 - root - INFO - step: 6455 loss: 17.7653 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.8157 global_avg_mtp_loss: 14.9496 +[titan] 2025-07-09 15:01:05,464 - root - INFO - lr: 2.9798e-04 gnorm: 0.95 [ 1:26:09<20:48:36] +[titan] 2025-07-09 15:01:09,401 - root - INFO - step: 6460 loss: 17.5986 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7910 global_avg_mtp_loss: 14.8076 +[titan] 2025-07-09 15:01:09,401 - root - INFO - lr: 2.9798e-04 gnorm: 0.98 [ 1:26:13<20:48:31] +[titan] 2025-07-09 15:01:13,308 - root - INFO - step: 6465 loss: 17.7352 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.8226 global_avg_mtp_loss: 14.9125 +[titan] 2025-07-09 15:01:13,308 - root - INFO - lr: 2.9797e-04 gnorm: 1.08 [ 1:26:17<20:48:26] +[titan] 2025-07-09 15:01:17,234 - root - INFO - step: 6470 loss: 17.5954 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.7935 global_avg_mtp_loss: 14.8019 +[titan] 2025-07-09 15:01:17,234 - root - INFO - lr: 2.9797e-04 gnorm: 1.04 [ 1:26:21<20:48:20] +[titan] 2025-07-09 15:01:21,174 - root - INFO - step: 6475 loss: 17.9398 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.8644 global_avg_mtp_loss: 15.0754 +[titan] 2025-07-09 15:01:21,175 - root - INFO - lr: 2.9797e-04 gnorm: 0.93 [ 1:26:25<20:48:15] +[titan] 2025-07-09 15:01:25,107 - root - INFO - step: 6480 loss: 17.8340 memory: 44.58GiB(31.99%) tps: 83,339 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.8407 global_avg_mtp_loss: 14.9933 +[titan] 2025-07-09 15:01:25,107 - root - INFO - lr: 2.9796e-04 gnorm: 0.99 [ 1:26:29<20:48:10] +[titan] 2025-07-09 15:01:29,017 - root - INFO - step: 6485 loss: 17.9237 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.8512 global_avg_mtp_loss: 15.0725 +[titan] 2025-07-09 15:01:29,017 - root - INFO - lr: 2.9796e-04 gnorm: 1.08 [ 1:26:33<20:48:05] +[titan] 2025-07-09 15:01:32,945 - root - INFO - step: 6490 loss: 17.9875 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.8773 global_avg_mtp_loss: 15.1102 +[titan] 2025-07-09 15:01:32,946 - root - INFO - lr: 2.9796e-04 gnorm: 0.96 [ 1:26:37<20:48:00] +[titan] 2025-07-09 15:01:36,869 - root - INFO - step: 6495 loss: 17.8174 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.8494 global_avg_mtp_loss: 14.9680 +[titan] 2025-07-09 15:01:36,870 - root - INFO - lr: 2.9795e-04 gnorm: 0.91 [ 1:26:40<20:47:55] +[titan] 2025-07-09 15:01:39,995 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:01:40,787 - root - INFO - step: 6500 loss: 17.6719 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.8097 global_avg_mtp_loss: 14.8622 +[titan] 2025-07-09 15:01:40,787 - root - INFO - lr: 2.9795e-04 gnorm: 0.95 [ 1:26:44<20:47:50] +[titan] 2025-07-09 15:01:44,683 - root - INFO - step: 6505 loss: 17.7897 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.8352 global_avg_mtp_loss: 14.9545 +[titan] 2025-07-09 15:01:44,684 - root - INFO - lr: 2.9794e-04 gnorm: 1.09 [ 1:26:48<20:47:44] +[titan] 2025-07-09 15:01:48,596 - root - INFO - step: 6510 loss: 17.9335 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.8526 global_avg_mtp_loss: 15.0809 +[titan] 2025-07-09 15:01:48,597 - root - INFO - lr: 2.9794e-04 gnorm: 0.99 [ 1:26:52<20:47:39] +[titan] 2025-07-09 15:01:52,506 - root - INFO - step: 6515 loss: 17.8552 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.8314 global_avg_mtp_loss: 15.0238 +[titan] 2025-07-09 15:01:52,507 - root - INFO - lr: 2.9794e-04 gnorm: 0.97 [ 1:26:56<20:47:33] +[titan] 2025-07-09 15:01:56,439 - root - INFO - step: 6520 loss: 17.5211 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.7847 global_avg_mtp_loss: 14.7363 +[titan] 2025-07-09 15:01:56,439 - root - INFO - lr: 2.9793e-04 gnorm: 0.98 [ 1:27:00<20:47:28] +[titan] 2025-07-09 15:02:00,328 - root - INFO - step: 6525 loss: 17.8853 memory: 44.58GiB(31.99%) tps: 84,261 tflops: 290.80 mfu: 29.40% global_avg_ntp_loss: 2.8547 global_avg_mtp_loss: 15.0306 +[titan] 2025-07-09 15:02:00,328 - root - INFO - lr: 2.9793e-04 gnorm: 0.90 [ 1:27:04<20:47:23] +[titan] 2025-07-09 15:02:04,260 - root - INFO - step: 6530 loss: 17.4693 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.7834 global_avg_mtp_loss: 14.6859 +[titan] 2025-07-09 15:02:04,261 - root - INFO - lr: 2.9793e-04 gnorm: 0.96 [ 1:27:08<20:47:18] +[titan] 2025-07-09 15:02:08,158 - root - INFO - step: 6535 loss: 17.6877 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.8086 global_avg_mtp_loss: 14.8791 +[titan] 2025-07-09 15:02:08,158 - root - INFO - lr: 2.9792e-04 gnorm: 1.08 [ 1:27:12<20:47:12] +[titan] 2025-07-09 15:02:12,068 - root - INFO - step: 6540 loss: 17.4569 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.7598 global_avg_mtp_loss: 14.6971 +[titan] 2025-07-09 15:02:12,068 - root - INFO - lr: 2.9792e-04 gnorm: 0.93 [ 1:27:16<20:47:07] +[titan] 2025-07-09 15:02:15,979 - root - INFO - step: 6545 loss: 17.9365 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.8575 global_avg_mtp_loss: 15.0791 +[titan] 2025-07-09 15:02:15,979 - root - INFO - lr: 2.9791e-04 gnorm: 0.90 [ 1:27:20<20:47:01] +[titan] 2025-07-09 15:02:19,098 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:02:19,887 - root - INFO - step: 6550 loss: 17.7911 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.8443 global_avg_mtp_loss: 14.9467 +[titan] 2025-07-09 15:02:19,887 - root - INFO - lr: 2.9791e-04 gnorm: 0.97 [ 1:27:23<20:46:56] +[titan] 2025-07-09 15:02:23,814 - root - INFO - step: 6555 loss: 18.0607 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.8787 global_avg_mtp_loss: 15.1820 +[titan] 2025-07-09 15:02:23,814 - root - INFO - lr: 2.9791e-04 gnorm: 0.94 [ 1:27:27<20:46:51] +[titan] 2025-07-09 15:02:27,759 - root - INFO - step: 6560 loss: 17.6907 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.8179 global_avg_mtp_loss: 14.8728 +[titan] 2025-07-09 15:02:27,759 - root - INFO - lr: 2.9790e-04 gnorm: 0.97 [ 1:27:31<20:46:46] +[titan] 2025-07-09 15:02:31,656 - root - INFO - step: 6565 loss: 17.8831 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.8469 global_avg_mtp_loss: 15.0362 +[titan] 2025-07-09 15:02:31,656 - root - INFO - lr: 2.9790e-04 gnorm: 1.07 [ 1:27:35<20:46:41] +[titan] 2025-07-09 15:02:35,591 - root - INFO - step: 6570 loss: 17.8805 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.8665 global_avg_mtp_loss: 15.0140 +[titan] 2025-07-09 15:02:35,591 - root - INFO - lr: 2.9790e-04 gnorm: 1.07 [ 1:27:39<20:46:36] +[titan] 2025-07-09 15:02:39,483 - root - INFO - step: 6575 loss: 17.5825 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.7854 global_avg_mtp_loss: 14.7971 +[titan] 2025-07-09 15:02:39,483 - root - INFO - lr: 2.9789e-04 gnorm: 0.93 [ 1:27:43<20:46:30] +[titan] 2025-07-09 15:02:43,381 - root - INFO - step: 6580 loss: 17.6551 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.8028 global_avg_mtp_loss: 14.8522 +[titan] 2025-07-09 15:02:43,381 - root - INFO - lr: 2.9789e-04 gnorm: 0.88 [ 1:27:47<20:46:25] +[titan] 2025-07-09 15:02:47,279 - root - INFO - step: 6585 loss: 17.7441 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.8197 global_avg_mtp_loss: 14.9245 +[titan] 2025-07-09 15:02:47,280 - root - INFO - lr: 2.9788e-04 gnorm: 1.04 [ 1:27:51<20:46:19] +[titan] 2025-07-09 15:02:51,168 - root - INFO - step: 6590 loss: 17.8045 memory: 44.58GiB(31.99%) tps: 84,271 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.8373 global_avg_mtp_loss: 14.9673 +[titan] 2025-07-09 15:02:51,168 - root - INFO - lr: 2.9788e-04 gnorm: 1.01 [ 1:27:55<20:46:13] +[titan] 2025-07-09 15:02:55,076 - root - INFO - step: 6595 loss: 17.6799 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.8140 global_avg_mtp_loss: 14.8660 +[titan] 2025-07-09 15:02:55,076 - root - INFO - lr: 2.9788e-04 gnorm: 1.03 [ 1:27:59<20:46:08] +[titan] 2025-07-09 15:02:58,228 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:02:59,029 - root - INFO - step: 6600 loss: 17.3589 memory: 44.58GiB(31.99%) tps: 82,898 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.7308 global_avg_mtp_loss: 14.6281 +[titan] 2025-07-09 15:02:59,030 - root - INFO - lr: 2.9787e-04 gnorm: 0.98 [ 1:28:03<20:46:03] +[titan] 2025-07-09 15:03:02,943 - root - INFO - step: 6605 loss: 17.7436 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.8054 global_avg_mtp_loss: 14.9381 +[titan] 2025-07-09 15:03:02,943 - root - INFO - lr: 2.9787e-04 gnorm: 0.94 [ 1:28:07<20:45:58] +[titan] 2025-07-09 15:03:06,863 - root - INFO - step: 6610 loss: 17.6383 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.7945 global_avg_mtp_loss: 14.8438 +[titan] 2025-07-09 15:03:06,863 - root - INFO - lr: 2.9787e-04 gnorm: 1.04 [ 1:28:10<20:45:53] +[titan] 2025-07-09 15:03:10,763 - root - INFO - step: 6615 loss: 17.5044 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.7778 global_avg_mtp_loss: 14.7266 +[titan] 2025-07-09 15:03:10,763 - root - INFO - lr: 2.9786e-04 gnorm: 0.94 [ 1:28:14<20:45:48] +[titan] 2025-07-09 15:03:14,661 - root - INFO - step: 6620 loss: 17.9556 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.8587 global_avg_mtp_loss: 15.0969 +[titan] 2025-07-09 15:03:14,661 - root - INFO - lr: 2.9786e-04 gnorm: 0.98 [ 1:28:18<20:45:42] +[titan] 2025-07-09 15:03:18,594 - root - INFO - step: 6625 loss: 17.7163 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.8139 global_avg_mtp_loss: 14.9024 +[titan] 2025-07-09 15:03:18,595 - root - INFO - lr: 2.9785e-04 gnorm: 0.86 [ 1:28:22<20:45:37] +[titan] 2025-07-09 15:03:22,490 - root - INFO - step: 6630 loss: 17.6374 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.8020 global_avg_mtp_loss: 14.8353 +[titan] 2025-07-09 15:03:22,490 - root - INFO - lr: 2.9785e-04 gnorm: 0.94 [ 1:28:26<20:45:32] +[titan] 2025-07-09 15:03:26,430 - root - INFO - step: 6635 loss: 17.4999 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.7684 global_avg_mtp_loss: 14.7315 +[titan] 2025-07-09 15:03:26,431 - root - INFO - lr: 2.9785e-04 gnorm: 1.02 [ 1:28:30<20:45:27] +[titan] 2025-07-09 15:03:30,369 - root - INFO - step: 6640 loss: 17.6522 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.7976 global_avg_mtp_loss: 14.8545 +[titan] 2025-07-09 15:03:30,369 - root - INFO - lr: 2.9784e-04 gnorm: 1.16 [ 1:28:34<20:45:22] +[titan] 2025-07-09 15:03:34,288 - root - INFO - step: 6645 loss: 17.4543 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.7604 global_avg_mtp_loss: 14.6938 +[titan] 2025-07-09 15:03:34,288 - root - INFO - lr: 2.9784e-04 gnorm: 0.96 [ 1:28:38<20:45:17] +[titan] 2025-07-09 15:03:37,406 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:03:38,194 - root - INFO - step: 6650 loss: 17.6805 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8119 global_avg_mtp_loss: 14.8686 +[titan] 2025-07-09 15:03:38,194 - root - INFO - lr: 2.9784e-04 gnorm: 0.93 [ 1:28:42<20:45:11] +[titan] 2025-07-09 15:03:42,172 - root - INFO - step: 6655 loss: 17.6720 memory: 44.58GiB(31.99%) tps: 82,388 tflops: 284.34 mfu: 28.75% global_avg_ntp_loss: 2.7997 global_avg_mtp_loss: 14.8723 +[titan] 2025-07-09 15:03:42,172 - root - INFO - lr: 2.9783e-04 gnorm: 0.95 [ 1:28:46<20:45:07] +[titan] 2025-07-09 15:03:43,097 - root - INFO - Dumping profiler traces at step 6656 +[titan] 2025-07-09 15:03:43,128 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:03:46,256 - root - INFO - step: 6660 loss: 17.7061 memory: 44.58GiB(31.99%) tps: 80,243 tflops: 276.93 mfu: 28.00% global_avg_ntp_loss: 2.8022 global_avg_mtp_loss: 14.9039 +[titan] 2025-07-09 15:03:46,256 - root - INFO - lr: 2.9783e-04 gnorm: 0.95 [ 1:28:50<20:45:04] +[titan] 2025-07-09 15:03:50,154 - root - INFO - step: 6665 loss: 17.5815 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.7882 global_avg_mtp_loss: 14.7933 +[titan] 2025-07-09 15:03:50,155 - root - INFO - lr: 2.9782e-04 gnorm: 1.13 [ 1:28:54<20:44:59] +[titan] 2025-07-09 15:03:54,056 - root - INFO - step: 6670 loss: 17.4739 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.7719 global_avg_mtp_loss: 14.7019 +[titan] 2025-07-09 15:03:54,056 - root - INFO - lr: 2.9782e-04 gnorm: 1.01 [ 1:28:58<20:44:53] +[titan] 2025-07-09 15:03:57,951 - root - INFO - step: 6675 loss: 17.5913 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.7761 global_avg_mtp_loss: 14.8152 +[titan] 2025-07-09 15:03:57,951 - root - INFO - lr: 2.9782e-04 gnorm: 1.08 [ 1:29:02<20:44:48] +[titan] 2025-07-09 15:04:01,846 - root - INFO - step: 6680 loss: 17.7844 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.8329 global_avg_mtp_loss: 14.9515 +[titan] 2025-07-09 15:04:01,846 - root - INFO - lr: 2.9781e-04 gnorm: 1.08 [ 1:29:05<20:44:42] +[titan] 2025-07-09 15:04:05,795 - root - INFO - step: 6685 loss: 17.5794 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.8011 global_avg_mtp_loss: 14.7783 +[titan] 2025-07-09 15:04:05,795 - root - INFO - lr: 2.9781e-04 gnorm: 0.89 [ 1:29:09<20:44:37] +[titan] 2025-07-09 15:04:09,703 - root - INFO - step: 6690 loss: 17.7183 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.8081 global_avg_mtp_loss: 14.9102 +[titan] 2025-07-09 15:04:09,704 - root - INFO - lr: 2.9780e-04 gnorm: 1.11 [ 1:29:13<20:44:32] +[titan] 2025-07-09 15:04:13,610 - root - INFO - step: 6695 loss: 17.8532 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8621 global_avg_mtp_loss: 14.9911 +[titan] 2025-07-09 15:04:13,610 - root - INFO - lr: 2.9780e-04 gnorm: 1.00 [ 1:29:17<20:44:27] +[titan] 2025-07-09 15:04:16,719 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:04:17,524 - root - INFO - step: 6700 loss: 17.6665 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.8176 global_avg_mtp_loss: 14.8489 +[titan] 2025-07-09 15:04:17,524 - root - INFO - lr: 2.9780e-04 gnorm: 0.92 [ 1:29:21<20:44:22] +[titan] 2025-07-09 15:04:21,451 - root - INFO - step: 6705 loss: 17.4861 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.7666 global_avg_mtp_loss: 14.7195 +[titan] 2025-07-09 15:04:21,451 - root - INFO - lr: 2.9779e-04 gnorm: 0.93 [ 1:29:25<20:44:17] +[titan] 2025-07-09 15:04:25,362 - root - INFO - step: 6710 loss: 17.8321 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.8449 global_avg_mtp_loss: 14.9872 +[titan] 2025-07-09 15:04:25,362 - root - INFO - lr: 2.9779e-04 gnorm: 0.90 [ 1:29:29<20:44:11] +[titan] 2025-07-09 15:04:29,278 - root - INFO - step: 6715 loss: 17.7866 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.8290 global_avg_mtp_loss: 14.9575 +[titan] 2025-07-09 15:04:29,278 - root - INFO - lr: 2.9779e-04 gnorm: 0.94 [ 1:29:33<20:44:06] +[titan] 2025-07-09 15:04:33,186 - root - INFO - step: 6720 loss: 17.6298 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.7941 global_avg_mtp_loss: 14.8357 +[titan] 2025-07-09 15:04:33,186 - root - INFO - lr: 2.9778e-04 gnorm: 0.93 [ 1:29:37<20:44:01] +[titan] 2025-07-09 15:04:37,082 - root - INFO - step: 6725 loss: 17.4570 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.7678 global_avg_mtp_loss: 14.6892 +[titan] 2025-07-09 15:04:37,082 - root - INFO - lr: 2.9778e-04 gnorm: 1.00 [ 1:29:41<20:43:55] +[titan] 2025-07-09 15:04:40,989 - root - INFO - step: 6730 loss: 17.5596 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.7795 global_avg_mtp_loss: 14.7802 +[titan] 2025-07-09 15:04:40,989 - root - INFO - lr: 2.9777e-04 gnorm: 1.00 [ 1:29:45<20:43:50] +[titan] 2025-07-09 15:04:44,905 - root - INFO - step: 6735 loss: 17.4446 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.7566 global_avg_mtp_loss: 14.6879 +[titan] 2025-07-09 15:04:44,906 - root - INFO - lr: 2.9777e-04 gnorm: 0.98 [ 1:29:48<20:43:45] +[titan] 2025-07-09 15:04:48,824 - root - INFO - step: 6740 loss: 17.5932 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.7924 global_avg_mtp_loss: 14.8008 +[titan] 2025-07-09 15:04:48,825 - root - INFO - lr: 2.9777e-04 gnorm: 0.93 [ 1:29:52<20:43:40] +[titan] 2025-07-09 15:04:52,736 - root - INFO - step: 6745 loss: 17.9242 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.8474 global_avg_mtp_loss: 15.0768 +[titan] 2025-07-09 15:04:52,737 - root - INFO - lr: 2.9776e-04 gnorm: 0.93 [ 1:29:56<20:43:35] +[titan] 2025-07-09 15:04:55,870 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:04:56,658 - root - INFO - step: 6750 loss: 17.8988 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.8549 global_avg_mtp_loss: 15.0440 +[titan] 2025-07-09 15:04:56,658 - root - INFO - lr: 2.9776e-04 gnorm: 0.95 [ 1:30:00<20:43:29] +[titan] 2025-07-09 15:05:00,590 - root - INFO - step: 6755 loss: 17.6184 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.7992 global_avg_mtp_loss: 14.8192 +[titan] 2025-07-09 15:05:00,591 - root - INFO - lr: 2.9775e-04 gnorm: 1.10 [ 1:30:04<20:43:24] +[titan] 2025-07-09 15:05:04,511 - root - INFO - step: 6760 loss: 17.7881 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.8294 global_avg_mtp_loss: 14.9587 +[titan] 2025-07-09 15:05:04,512 - root - INFO - lr: 2.9775e-04 gnorm: 0.93 [ 1:30:08<20:43:19] +[titan] 2025-07-09 15:05:08,421 - root - INFO - step: 6765 loss: 17.8835 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.8466 global_avg_mtp_loss: 15.0370 +[titan] 2025-07-09 15:05:08,421 - root - INFO - lr: 2.9775e-04 gnorm: 0.91 [ 1:30:12<20:43:14] +[titan] 2025-07-09 15:05:12,332 - root - INFO - step: 6770 loss: 17.6498 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.8063 global_avg_mtp_loss: 14.8435 +[titan] 2025-07-09 15:05:12,332 - root - INFO - lr: 2.9774e-04 gnorm: 1.10 [ 1:30:16<20:43:09] +[titan] 2025-07-09 15:05:16,242 - root - INFO - step: 6775 loss: 17.9446 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.8701 global_avg_mtp_loss: 15.0745 +[titan] 2025-07-09 15:05:16,242 - root - INFO - lr: 2.9774e-04 gnorm: 0.98 [ 1:30:20<20:43:04] +[titan] 2025-07-09 15:05:20,169 - root - INFO - step: 6780 loss: 17.7234 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.8287 global_avg_mtp_loss: 14.8948 +[titan] 2025-07-09 15:05:20,169 - root - INFO - lr: 2.9773e-04 gnorm: 1.05 [ 1:30:24<20:42:59] +[titan] 2025-07-09 15:05:24,083 - root - INFO - step: 6785 loss: 17.5642 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.7706 global_avg_mtp_loss: 14.7937 +[titan] 2025-07-09 15:05:24,084 - root - INFO - lr: 2.9773e-04 gnorm: 1.05 [ 1:30:28<20:42:53] +[titan] 2025-07-09 15:05:28,024 - root - INFO - step: 6790 loss: 17.7507 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.8176 global_avg_mtp_loss: 14.9331 +[titan] 2025-07-09 15:05:28,024 - root - INFO - lr: 2.9773e-04 gnorm: 1.14 [ 1:30:32<20:42:49] +[titan] 2025-07-09 15:05:31,950 - root - INFO - step: 6795 loss: 17.6619 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.7955 global_avg_mtp_loss: 14.8664 +[titan] 2025-07-09 15:05:31,950 - root - INFO - lr: 2.9772e-04 gnorm: 0.99 [ 1:30:36<20:42:44] +[titan] 2025-07-09 15:05:35,080 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:05:35,875 - root - INFO - step: 6800 loss: 17.3463 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.7457 global_avg_mtp_loss: 14.6007 +[titan] 2025-07-09 15:05:35,875 - root - INFO - lr: 2.9772e-04 gnorm: 0.99 [ 1:30:39<20:42:39] +[titan] 2025-07-09 15:05:39,855 - root - INFO - step: 6805 loss: 17.7134 memory: 44.58GiB(31.99%) tps: 82,341 tflops: 284.17 mfu: 28.73% global_avg_ntp_loss: 2.8409 global_avg_mtp_loss: 14.8725 +[titan] 2025-07-09 15:05:39,855 - root - INFO - lr: 2.9772e-04 gnorm: 1.15 [ 1:30:43<20:42:34] +[titan] 2025-07-09 15:05:43,799 - root - INFO - step: 6810 loss: 17.6851 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.7952 global_avg_mtp_loss: 14.8898 +[titan] 2025-07-09 15:05:43,799 - root - INFO - lr: 2.9771e-04 gnorm: 0.88 [ 1:30:47<20:42:29] +[titan] 2025-07-09 15:05:47,721 - root - INFO - step: 6815 loss: 17.7430 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.8227 global_avg_mtp_loss: 14.9203 +[titan] 2025-07-09 15:05:47,721 - root - INFO - lr: 2.9771e-04 gnorm: 1.05 [ 1:30:51<20:42:24] +[titan] 2025-07-09 15:05:51,651 - root - INFO - step: 6820 loss: 17.6631 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.7993 global_avg_mtp_loss: 14.8638 +[titan] 2025-07-09 15:05:51,651 - root - INFO - lr: 2.9770e-04 gnorm: 0.93 [ 1:30:55<20:42:19] +[titan] 2025-07-09 15:05:55,563 - root - INFO - step: 6825 loss: 17.7986 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.8324 global_avg_mtp_loss: 14.9662 +[titan] 2025-07-09 15:05:55,563 - root - INFO - lr: 2.9770e-04 gnorm: 0.94 [ 1:30:59<20:42:14] +[titan] 2025-07-09 15:05:59,497 - root - INFO - step: 6830 loss: 17.4431 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.7773 global_avg_mtp_loss: 14.6658 +[titan] 2025-07-09 15:05:59,497 - root - INFO - lr: 2.9770e-04 gnorm: 1.01 [ 1:31:03<20:42:09] +[titan] 2025-07-09 15:06:03,415 - root - INFO - step: 6835 loss: 17.5760 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.7861 global_avg_mtp_loss: 14.7899 +[titan] 2025-07-09 15:06:03,416 - root - INFO - lr: 2.9769e-04 gnorm: 1.01 [ 1:31:07<20:42:04] +[titan] 2025-07-09 15:06:07,334 - root - INFO - step: 6840 loss: 17.6180 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.7993 global_avg_mtp_loss: 14.8187 +[titan] 2025-07-09 15:06:07,334 - root - INFO - lr: 2.9769e-04 gnorm: 0.95 [ 1:31:11<20:41:59] +[titan] 2025-07-09 15:06:11,274 - root - INFO - step: 6845 loss: 17.7146 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.7967 global_avg_mtp_loss: 14.9179 +[titan] 2025-07-09 15:06:11,274 - root - INFO - lr: 2.9768e-04 gnorm: 1.00 [ 1:31:15<20:41:54] +[titan] 2025-07-09 15:06:14,444 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:06:15,229 - root - INFO - step: 6850 loss: 17.5432 memory: 44.58GiB(31.99%) tps: 82,852 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.7792 global_avg_mtp_loss: 14.7640 +[titan] 2025-07-09 15:06:15,229 - root - INFO - lr: 2.9768e-04 gnorm: 0.90 [ 1:31:19<20:41:50] +[titan] 2025-07-09 15:06:19,144 - root - INFO - step: 6855 loss: 17.5252 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7765 global_avg_mtp_loss: 14.7487 +[titan] 2025-07-09 15:06:19,144 - root - INFO - lr: 2.9768e-04 gnorm: 1.01 [ 1:31:23<20:41:45] +[titan] 2025-07-09 15:06:23,071 - root - INFO - step: 6860 loss: 17.7221 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.8243 global_avg_mtp_loss: 14.8978 +[titan] 2025-07-09 15:06:23,071 - root - INFO - lr: 2.9767e-04 gnorm: 1.12 [ 1:31:27<20:41:40] +[titan] 2025-07-09 15:06:27,013 - root - INFO - step: 6865 loss: 17.7944 memory: 44.58GiB(31.99%) tps: 83,134 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.8267 global_avg_mtp_loss: 14.9676 +[titan] 2025-07-09 15:06:27,013 - root - INFO - lr: 2.9767e-04 gnorm: 1.04 [ 1:31:31<20:41:35] +[titan] 2025-07-09 15:06:30,929 - root - INFO - step: 6870 loss: 17.5938 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.7809 global_avg_mtp_loss: 14.8129 +[titan] 2025-07-09 15:06:30,930 - root - INFO - lr: 2.9766e-04 gnorm: 1.05 [ 1:31:34<20:41:30] +[titan] 2025-07-09 15:06:34,823 - root - INFO - step: 6875 loss: 17.6114 memory: 44.58GiB(31.99%) tps: 84,176 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.7945 global_avg_mtp_loss: 14.8168 +[titan] 2025-07-09 15:06:34,823 - root - INFO - lr: 2.9766e-04 gnorm: 0.95 [ 1:31:38<20:41:24] +[titan] 2025-07-09 15:06:38,748 - root - INFO - step: 6880 loss: 17.6996 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.7990 global_avg_mtp_loss: 14.9005 +[titan] 2025-07-09 15:06:38,749 - root - INFO - lr: 2.9766e-04 gnorm: 0.93 [ 1:31:42<20:41:19] +[titan] 2025-07-09 15:06:42,643 - root - INFO - step: 6885 loss: 17.6672 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.8047 global_avg_mtp_loss: 14.8625 +[titan] 2025-07-09 15:06:42,644 - root - INFO - lr: 2.9765e-04 gnorm: 0.95 [ 1:31:46<20:41:14] +[titan] 2025-07-09 15:06:46,581 - root - INFO - step: 6890 loss: 17.6299 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.7904 global_avg_mtp_loss: 14.8395 +[titan] 2025-07-09 15:06:46,581 - root - INFO - lr: 2.9765e-04 gnorm: 0.93 [ 1:31:50<20:41:09] +[titan] 2025-07-09 15:06:50,548 - root - INFO - step: 6895 loss: 17.8675 memory: 44.58GiB(31.99%) tps: 82,620 tflops: 285.14 mfu: 28.83% global_avg_ntp_loss: 2.8517 global_avg_mtp_loss: 15.0157 +[titan] 2025-07-09 15:06:50,548 - root - INFO - lr: 2.9764e-04 gnorm: 1.01 [ 1:31:54<20:41:05] +[titan] 2025-07-09 15:06:53,684 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:06:54,466 - root - INFO - step: 6900 loss: 17.5933 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.7988 global_avg_mtp_loss: 14.7945 +[titan] 2025-07-09 15:06:54,466 - root - INFO - lr: 2.9764e-04 gnorm: 1.02 [ 1:31:58<20:40:59] +[titan] 2025-07-09 15:06:58,380 - root - INFO - step: 6905 loss: 17.6692 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.8015 global_avg_mtp_loss: 14.8677 +[titan] 2025-07-09 15:06:58,381 - root - INFO - lr: 2.9764e-04 gnorm: 0.93 [ 1:32:02<20:40:54] +[titan] 2025-07-09 15:07:02,305 - root - INFO - step: 6910 loss: 17.7293 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.8118 global_avg_mtp_loss: 14.9175 +[titan] 2025-07-09 15:07:02,306 - root - INFO - lr: 2.9763e-04 gnorm: 0.95 [ 1:32:06<20:40:49] +[titan] 2025-07-09 15:07:06,224 - root - INFO - step: 6915 loss: 17.9103 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.8561 global_avg_mtp_loss: 15.0542 +[titan] 2025-07-09 15:07:06,224 - root - INFO - lr: 2.9763e-04 gnorm: 0.99 [ 1:32:10<20:40:44] +[titan] 2025-07-09 15:07:10,125 - root - INFO - step: 6920 loss: 17.4483 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.7618 global_avg_mtp_loss: 14.6865 +[titan] 2025-07-09 15:07:10,126 - root - INFO - lr: 2.9762e-04 gnorm: 1.04 [ 1:32:14<20:40:39] +[titan] 2025-07-09 15:07:14,133 - root - INFO - step: 6925 loss: 17.9862 memory: 44.58GiB(31.99%) tps: 81,775 tflops: 282.22 mfu: 28.54% global_avg_ntp_loss: 2.8894 global_avg_mtp_loss: 15.0968 +[titan] 2025-07-09 15:07:14,133 - root - INFO - lr: 2.9762e-04 gnorm: 1.26 [ 1:32:18<20:40:35] +[titan] 2025-07-09 15:07:18,046 - root - INFO - step: 6930 loss: 17.5476 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.7689 global_avg_mtp_loss: 14.7786 +[titan] 2025-07-09 15:07:18,047 - root - INFO - lr: 2.9762e-04 gnorm: 1.18 [ 1:32:22<20:40:30] +[titan] 2025-07-09 15:07:21,939 - root - INFO - step: 6935 loss: 17.7188 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.8150 global_avg_mtp_loss: 14.9039 +[titan] 2025-07-09 15:07:21,940 - root - INFO - lr: 2.9761e-04 gnorm: 1.04 [ 1:32:25<20:40:24] +[titan] 2025-07-09 15:07:25,856 - root - INFO - step: 6940 loss: 17.5606 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7866 global_avg_mtp_loss: 14.7740 +[titan] 2025-07-09 15:07:25,856 - root - INFO - lr: 2.9761e-04 gnorm: 0.93 [ 1:32:29<20:40:19] +[titan] 2025-07-09 15:07:29,769 - root - INFO - step: 6945 loss: 17.8181 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.8167 global_avg_mtp_loss: 15.0014 +[titan] 2025-07-09 15:07:29,770 - root - INFO - lr: 2.9760e-04 gnorm: 0.94 [ 1:32:33<20:40:14] +[titan] 2025-07-09 15:07:32,911 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:07:33,706 - root - INFO - step: 6950 loss: 17.6122 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.7911 global_avg_mtp_loss: 14.8211 +[titan] 2025-07-09 15:07:33,706 - root - INFO - lr: 2.9760e-04 gnorm: 1.04 [ 1:32:37<20:40:09] +[titan] 2025-07-09 15:07:37,633 - root - INFO - step: 6955 loss: 17.7759 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.8206 global_avg_mtp_loss: 14.9554 +[titan] 2025-07-09 15:07:37,633 - root - INFO - lr: 2.9760e-04 gnorm: 0.96 [ 1:32:41<20:40:04] +[titan] 2025-07-09 15:07:41,558 - root - INFO - step: 6960 loss: 17.5398 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.13% global_avg_ntp_loss: 2.7857 global_avg_mtp_loss: 14.7541 +[titan] 2025-07-09 15:07:41,558 - root - INFO - lr: 2.9759e-04 gnorm: 0.94 [ 1:32:45<20:39:59] +[titan] 2025-07-09 15:07:45,508 - root - INFO - step: 6965 loss: 17.8650 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.8553 global_avg_mtp_loss: 15.0097 +[titan] 2025-07-09 15:07:45,508 - root - INFO - lr: 2.9759e-04 gnorm: 1.02 [ 1:32:49<20:39:55] +[titan] 2025-07-09 15:07:49,410 - root - INFO - step: 6970 loss: 17.5063 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.7878 global_avg_mtp_loss: 14.7185 +[titan] 2025-07-09 15:07:49,411 - root - INFO - lr: 2.9758e-04 gnorm: 0.86 [ 1:32:53<20:39:49] +[titan] 2025-07-09 15:07:53,354 - root - INFO - step: 6975 loss: 17.7628 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.8291 global_avg_mtp_loss: 14.9337 +[titan] 2025-07-09 15:07:53,354 - root - INFO - lr: 2.9758e-04 gnorm: 0.95 [ 1:32:57<20:39:45] +[titan] 2025-07-09 15:07:57,279 - root - INFO - step: 6980 loss: 17.9055 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.8340 global_avg_mtp_loss: 15.0715 +[titan] 2025-07-09 15:07:57,279 - root - INFO - lr: 2.9758e-04 gnorm: 0.93 [ 1:33:01<20:39:40] +[titan] 2025-07-09 15:08:01,204 - root - INFO - step: 6985 loss: 17.8022 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.8400 global_avg_mtp_loss: 14.9622 +[titan] 2025-07-09 15:08:01,204 - root - INFO - lr: 2.9757e-04 gnorm: 0.95 [ 1:33:05<20:39:35] +[titan] 2025-07-09 15:08:05,094 - root - INFO - step: 6990 loss: 17.7470 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.8315 global_avg_mtp_loss: 14.9155 +[titan] 2025-07-09 15:08:05,094 - root - INFO - lr: 2.9757e-04 gnorm: 1.11 [ 1:33:09<20:39:29] +[titan] 2025-07-09 15:08:09,000 - root - INFO - step: 6995 loss: 17.8204 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.8281 global_avg_mtp_loss: 14.9923 +[titan] 2025-07-09 15:08:09,001 - root - INFO - lr: 2.9756e-04 gnorm: 0.93 [ 1:33:13<20:39:24] +[titan] 2025-07-09 15:08:12,118 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:08:12,908 - root - INFO - step: 7000 loss: 17.7172 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.8358 global_avg_mtp_loss: 14.8813 +[titan] 2025-07-09 15:08:12,908 - root - INFO - lr: 2.9756e-04 gnorm: 0.97 [ 1:33:16<20:39:19] +[titan] 2025-07-09 15:08:16,867 - root - INFO - step: 7005 loss: 17.7195 memory: 44.58GiB(31.99%) tps: 82,771 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.8188 global_avg_mtp_loss: 14.9007 +[titan] 2025-07-09 15:08:16,867 - root - INFO - lr: 2.9756e-04 gnorm: 0.97 [ 1:33:20<20:39:14] +[titan] 2025-07-09 15:08:20,834 - root - INFO - step: 7010 loss: 17.4686 memory: 44.58GiB(31.99%) tps: 82,597 tflops: 285.06 mfu: 28.82% global_avg_ntp_loss: 2.7613 global_avg_mtp_loss: 14.7073 +[titan] 2025-07-09 15:08:20,835 - root - INFO - lr: 2.9755e-04 gnorm: 1.00 [ 1:33:24<20:39:10] +[titan] 2025-07-09 15:08:24,725 - root - INFO - step: 7015 loss: 17.8297 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.8302 global_avg_mtp_loss: 14.9995 +[titan] 2025-07-09 15:08:24,726 - root - INFO - lr: 2.9755e-04 gnorm: 0.96 [ 1:33:28<20:39:05] +[titan] 2025-07-09 15:08:28,644 - root - INFO - step: 7020 loss: 17.6952 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.8067 global_avg_mtp_loss: 14.8885 +[titan] 2025-07-09 15:08:28,645 - root - INFO - lr: 2.9754e-04 gnorm: 0.99 [ 1:33:32<20:39:00] +[titan] 2025-07-09 15:08:32,536 - root - INFO - step: 7025 loss: 17.7832 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.8270 global_avg_mtp_loss: 14.9562 +[titan] 2025-07-09 15:08:32,537 - root - INFO - lr: 2.9754e-04 gnorm: 0.99 [ 1:33:36<20:38:54] +[titan] 2025-07-09 15:08:36,475 - root - INFO - step: 7030 loss: 17.7618 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.8193 global_avg_mtp_loss: 14.9425 +[titan] 2025-07-09 15:08:36,475 - root - INFO - lr: 2.9754e-04 gnorm: 0.98 [ 1:33:40<20:38:49] +[titan] 2025-07-09 15:08:40,362 - root - INFO - step: 7035 loss: 17.5088 memory: 44.58GiB(31.99%) tps: 84,316 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.7790 global_avg_mtp_loss: 14.7298 +[titan] 2025-07-09 15:08:40,362 - root - INFO - lr: 2.9753e-04 gnorm: 0.98 [ 1:33:44<20:38:44] +[titan] 2025-07-09 15:08:44,302 - root - INFO - step: 7040 loss: 17.5249 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.7799 global_avg_mtp_loss: 14.7451 +[titan] 2025-07-09 15:08:44,302 - root - INFO - lr: 2.9753e-04 gnorm: 1.09 [ 1:33:48<20:38:39] +[titan] 2025-07-09 15:08:48,222 - root - INFO - step: 7045 loss: 17.5838 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.7847 global_avg_mtp_loss: 14.7991 +[titan] 2025-07-09 15:08:48,222 - root - INFO - lr: 2.9752e-04 gnorm: 1.00 [ 1:33:52<20:38:34] +[titan] 2025-07-09 15:08:51,393 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:08:52,183 - root - INFO - step: 7050 loss: 17.6211 memory: 44.58GiB(31.99%) tps: 82,726 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.7880 global_avg_mtp_loss: 14.8331 +[titan] 2025-07-09 15:08:52,183 - root - INFO - lr: 2.9752e-04 gnorm: 0.91 [ 1:33:56<20:38:30] +[titan] 2025-07-09 15:08:56,092 - root - INFO - step: 7055 loss: 17.5098 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.7675 global_avg_mtp_loss: 14.7423 +[titan] 2025-07-09 15:08:56,092 - root - INFO - lr: 2.9751e-04 gnorm: 0.95 [ 1:34:00<20:38:24] +[titan] 2025-07-09 15:09:00,047 - root - INFO - step: 7060 loss: 17.7384 memory: 44.58GiB(31.99%) tps: 82,866 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.8114 global_avg_mtp_loss: 14.9270 +[titan] 2025-07-09 15:09:00,047 - root - INFO - lr: 2.9751e-04 gnorm: 0.98 [ 1:34:04<20:38:20] +[titan] 2025-07-09 15:09:03,988 - root - INFO - step: 7065 loss: 17.5923 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.7718 global_avg_mtp_loss: 14.8205 +[titan] 2025-07-09 15:09:03,988 - root - INFO - lr: 2.9751e-04 gnorm: 1.03 [ 1:34:08<20:38:15] +[titan] 2025-07-09 15:09:07,915 - root - INFO - step: 7070 loss: 17.5157 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.7699 global_avg_mtp_loss: 14.7457 +[titan] 2025-07-09 15:09:07,916 - root - INFO - lr: 2.9750e-04 gnorm: 0.92 [ 1:34:11<20:38:10] +[titan] 2025-07-09 15:09:11,872 - root - INFO - step: 7075 loss: 17.7508 memory: 44.58GiB(31.99%) tps: 82,834 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.8176 global_avg_mtp_loss: 14.9332 +[titan] 2025-07-09 15:09:11,872 - root - INFO - lr: 2.9750e-04 gnorm: 0.96 [ 1:34:15<20:38:06] +[titan] 2025-07-09 15:09:15,780 - root - INFO - step: 7080 loss: 17.6660 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.7992 global_avg_mtp_loss: 14.8668 +[titan] 2025-07-09 15:09:15,781 - root - INFO - lr: 2.9749e-04 gnorm: 0.89 [ 1:34:19<20:38:00] +[titan] 2025-07-09 15:09:19,703 - root - INFO - step: 7085 loss: 17.7241 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.8154 global_avg_mtp_loss: 14.9088 +[titan] 2025-07-09 15:09:19,703 - root - INFO - lr: 2.9749e-04 gnorm: 1.09 [ 1:34:23<20:37:56] +[titan] 2025-07-09 15:09:23,639 - root - INFO - step: 7090 loss: 17.7804 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.8196 global_avg_mtp_loss: 14.9608 +[titan] 2025-07-09 15:09:23,639 - root - INFO - lr: 2.9749e-04 gnorm: 0.97 [ 1:34:27<20:37:51] +[titan] 2025-07-09 15:09:27,547 - root - INFO - step: 7095 loss: 17.4654 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.7721 global_avg_mtp_loss: 14.6933 +[titan] 2025-07-09 15:09:27,548 - root - INFO - lr: 2.9748e-04 gnorm: 1.00 [ 1:34:31<20:37:46] +[titan] 2025-07-09 15:09:30,667 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:09:31,450 - root - INFO - step: 7100 loss: 17.5794 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.7884 global_avg_mtp_loss: 14.7911 +[titan] 2025-07-09 15:09:31,451 - root - INFO - lr: 2.9748e-04 gnorm: 1.02 [ 1:34:35<20:37:40] +[titan] 2025-07-09 15:09:35,361 - root - INFO - step: 7105 loss: 17.5705 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.7736 global_avg_mtp_loss: 14.7968 +[titan] 2025-07-09 15:09:35,362 - root - INFO - lr: 2.9747e-04 gnorm: 0.98 [ 1:34:39<20:37:35] +[titan] 2025-07-09 15:09:39,253 - root - INFO - step: 7110 loss: 17.6533 memory: 44.58GiB(31.99%) tps: 84,208 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.8037 global_avg_mtp_loss: 14.8495 +[titan] 2025-07-09 15:09:39,253 - root - INFO - lr: 2.9747e-04 gnorm: 1.07 [ 1:34:43<20:37:30] +[titan] 2025-07-09 15:09:43,175 - root - INFO - step: 7115 loss: 17.7529 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.8183 global_avg_mtp_loss: 14.9347 +[titan] 2025-07-09 15:09:43,175 - root - INFO - lr: 2.9747e-04 gnorm: 1.01 [ 1:34:47<20:37:25] +[titan] 2025-07-09 15:09:47,116 - root - INFO - step: 7120 loss: 17.7126 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.8137 global_avg_mtp_loss: 14.8989 +[titan] 2025-07-09 15:09:47,116 - root - INFO - lr: 2.9746e-04 gnorm: 1.01 [ 1:34:51<20:37:20] +[titan] 2025-07-09 15:09:51,012 - root - INFO - step: 7125 loss: 17.7354 memory: 44.58GiB(31.99%) tps: 84,109 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.8110 global_avg_mtp_loss: 14.9244 +[titan] 2025-07-09 15:09:51,012 - root - INFO - lr: 2.9746e-04 gnorm: 0.98 [ 1:34:55<20:37:15] +[titan] 2025-07-09 15:09:54,934 - root - INFO - step: 7130 loss: 17.6709 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.7929 global_avg_mtp_loss: 14.8781 +[titan] 2025-07-09 15:09:54,935 - root - INFO - lr: 2.9745e-04 gnorm: 0.89 [ 1:34:58<20:37:10] +[titan] 2025-07-09 15:09:58,867 - root - INFO - step: 7135 loss: 17.6204 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.8009 global_avg_mtp_loss: 14.8194 +[titan] 2025-07-09 15:09:58,867 - root - INFO - lr: 2.9745e-04 gnorm: 1.02 [ 1:35:02<20:37:05] +[titan] 2025-07-09 15:10:02,774 - root - INFO - step: 7140 loss: 17.5987 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7967 global_avg_mtp_loss: 14.8020 +[titan] 2025-07-09 15:10:02,774 - root - INFO - lr: 2.9744e-04 gnorm: 0.89 [ 1:35:06<20:37:00] +[titan] 2025-07-09 15:10:06,672 - root - INFO - step: 7145 loss: 17.6490 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.7899 global_avg_mtp_loss: 14.8590 +[titan] 2025-07-09 15:10:06,672 - root - INFO - lr: 2.9744e-04 gnorm: 0.92 [ 1:35:10<20:36:55] +[titan] 2025-07-09 15:10:09,774 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:10:10,564 - root - INFO - step: 7150 loss: 17.4329 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.7582 global_avg_mtp_loss: 14.6747 +[titan] 2025-07-09 15:10:10,564 - root - INFO - lr: 2.9744e-04 gnorm: 0.93 [ 1:35:14<20:36:49] +[titan] 2025-07-09 15:10:14,475 - root - INFO - step: 7155 loss: 17.6623 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.8102 global_avg_mtp_loss: 14.8521 +[titan] 2025-07-09 15:10:14,475 - root - INFO - lr: 2.9743e-04 gnorm: 0.87 [ 1:35:18<20:36:44] +[titan] 2025-07-09 15:10:18,402 - root - INFO - step: 7160 loss: 17.5888 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.7743 global_avg_mtp_loss: 14.8145 +[titan] 2025-07-09 15:10:18,402 - root - INFO - lr: 2.9743e-04 gnorm: 0.85 [ 1:35:22<20:36:39] +[titan] 2025-07-09 15:10:22,383 - root - INFO - step: 7165 loss: 17.7597 memory: 44.58GiB(31.99%) tps: 82,330 tflops: 284.13 mfu: 28.73% global_avg_ntp_loss: 2.8166 global_avg_mtp_loss: 14.9431 +[titan] 2025-07-09 15:10:22,383 - root - INFO - lr: 2.9742e-04 gnorm: 1.01 [ 1:35:26<20:36:35] +[titan] 2025-07-09 15:10:24,889 - root - INFO - Dumping profiler traces at step 7168 +[titan] 2025-07-09 15:10:24,921 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:10:26,503 - root - INFO - step: 7170 loss: 17.6451 memory: 44.58GiB(31.99%) tps: 79,539 tflops: 274.50 mfu: 27.76% global_avg_ntp_loss: 2.7966 global_avg_mtp_loss: 14.8485 +[titan] 2025-07-09 15:10:26,503 - root - INFO - lr: 2.9742e-04 gnorm: 0.89 [ 1:35:30<20:36:33] +[titan] 2025-07-09 15:10:30,412 - root - INFO - step: 7175 loss: 17.4985 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7637 global_avg_mtp_loss: 14.7348 +[titan] 2025-07-09 15:10:30,412 - root - INFO - lr: 2.9742e-04 gnorm: 0.92 [ 1:35:34<20:36:27] +[titan] 2025-07-09 15:10:34,319 - root - INFO - step: 7180 loss: 17.5904 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7890 global_avg_mtp_loss: 14.8014 +[titan] 2025-07-09 15:10:34,319 - root - INFO - lr: 2.9741e-04 gnorm: 0.90 [ 1:35:38<20:36:22] +[titan] 2025-07-09 15:10:38,212 - root - INFO - step: 7185 loss: 17.5552 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 2.7822 global_avg_mtp_loss: 14.7730 +[titan] 2025-07-09 15:10:38,212 - root - INFO - lr: 2.9741e-04 gnorm: 0.94 [ 1:35:42<20:36:17] +[titan] 2025-07-09 15:10:42,116 - root - INFO - step: 7190 loss: 17.3181 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7444 global_avg_mtp_loss: 14.5737 +[titan] 2025-07-09 15:10:42,116 - root - INFO - lr: 2.9740e-04 gnorm: 0.99 [ 1:35:46<20:36:12] +[titan] 2025-07-09 15:10:46,025 - root - INFO - step: 7195 loss: 17.5921 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7829 global_avg_mtp_loss: 14.8092 +[titan] 2025-07-09 15:10:46,025 - root - INFO - lr: 2.9740e-04 gnorm: 1.04 [ 1:35:50<20:36:07] +[titan] 2025-07-09 15:10:49,167 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:10:49,976 - root - INFO - step: 7200 loss: 17.3810 memory: 44.58GiB(31.99%) tps: 82,947 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.7535 global_avg_mtp_loss: 14.6275 +[titan] 2025-07-09 15:10:49,976 - root - INFO - lr: 2.9739e-04 gnorm: 1.07 [ 1:35:53<20:36:02] +[titan] 2025-07-09 15:10:53,900 - root - INFO - step: 7205 loss: 17.5628 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.7815 global_avg_mtp_loss: 14.7813 +[titan] 2025-07-09 15:10:53,901 - root - INFO - lr: 2.9739e-04 gnorm: 1.03 [ 1:35:57<20:35:57] +[titan] 2025-07-09 15:10:57,821 - root - INFO - step: 7210 loss: 17.6042 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.7879 global_avg_mtp_loss: 14.8163 +[titan] 2025-07-09 15:10:57,821 - root - INFO - lr: 2.9739e-04 gnorm: 1.00 [ 1:36:01<20:35:52] +[titan] 2025-07-09 15:11:01,743 - root - INFO - step: 7215 loss: 17.4755 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7711 global_avg_mtp_loss: 14.7044 +[titan] 2025-07-09 15:11:01,743 - root - INFO - lr: 2.9738e-04 gnorm: 1.20 [ 1:36:05<20:35:47] +[titan] 2025-07-09 15:11:05,636 - root - INFO - step: 7220 loss: 17.6598 memory: 44.58GiB(31.99%) tps: 84,172 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.7979 global_avg_mtp_loss: 14.8619 +[titan] 2025-07-09 15:11:05,636 - root - INFO - lr: 2.9738e-04 gnorm: 1.10 [ 1:36:09<20:35:42] +[titan] 2025-07-09 15:11:09,546 - root - INFO - step: 7225 loss: 17.7273 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.8130 global_avg_mtp_loss: 14.9143 +[titan] 2025-07-09 15:11:09,547 - root - INFO - lr: 2.9737e-04 gnorm: 0.90 [ 1:36:13<20:35:37] +[titan] 2025-07-09 15:11:13,481 - root - INFO - step: 7230 loss: 17.8875 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.8471 global_avg_mtp_loss: 15.0404 +[titan] 2025-07-09 15:11:13,482 - root - INFO - lr: 2.9737e-04 gnorm: 1.03 [ 1:36:17<20:35:32] +[titan] 2025-07-09 15:11:17,401 - root - INFO - step: 7235 loss: 17.7931 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.8246 global_avg_mtp_loss: 14.9685 +[titan] 2025-07-09 15:11:17,401 - root - INFO - lr: 2.9737e-04 gnorm: 1.04 [ 1:36:21<20:35:27] +[titan] 2025-07-09 15:11:21,343 - root - INFO - step: 7240 loss: 17.2924 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.7434 global_avg_mtp_loss: 14.5490 +[titan] 2025-07-09 15:11:21,344 - root - INFO - lr: 2.9736e-04 gnorm: 0.98 [ 1:36:25<20:35:22] +[titan] 2025-07-09 15:11:25,243 - root - INFO - step: 7245 loss: 17.7294 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.8040 global_avg_mtp_loss: 14.9254 +[titan] 2025-07-09 15:11:25,243 - root - INFO - lr: 2.9736e-04 gnorm: 0.95 [ 1:36:29<20:35:17] +[titan] 2025-07-09 15:11:28,401 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:11:29,187 - root - INFO - step: 7250 loss: 17.8787 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.8657 global_avg_mtp_loss: 15.0130 +[titan] 2025-07-09 15:11:29,188 - root - INFO - lr: 2.9735e-04 gnorm: 1.29 [ 1:36:33<20:35:13] +[titan] 2025-07-09 15:11:33,096 - root - INFO - step: 7255 loss: 17.4315 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7599 global_avg_mtp_loss: 14.6716 +[titan] 2025-07-09 15:11:33,096 - root - INFO - lr: 2.9735e-04 gnorm: 1.03 [ 1:36:37<20:35:07] +[titan] 2025-07-09 15:11:37,017 - root - INFO - step: 7260 loss: 17.6205 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.7798 global_avg_mtp_loss: 14.8407 +[titan] 2025-07-09 15:11:37,017 - root - INFO - lr: 2.9734e-04 gnorm: 0.96 [ 1:36:41<20:35:02] +[titan] 2025-07-09 15:11:40,940 - root - INFO - step: 7265 loss: 17.7229 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.8117 global_avg_mtp_loss: 14.9113 +[titan] 2025-07-09 15:11:40,940 - root - INFO - lr: 2.9734e-04 gnorm: 0.92 [ 1:36:44<20:34:58] +[titan] 2025-07-09 15:11:44,878 - root - INFO - step: 7270 loss: 17.5883 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.7950 global_avg_mtp_loss: 14.7933 +[titan] 2025-07-09 15:11:44,878 - root - INFO - lr: 2.9734e-04 gnorm: 0.95 [ 1:36:48<20:34:53] +[titan] 2025-07-09 15:11:48,786 - root - INFO - step: 7275 loss: 17.4522 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.7655 global_avg_mtp_loss: 14.6867 +[titan] 2025-07-09 15:11:48,786 - root - INFO - lr: 2.9733e-04 gnorm: 0.95 [ 1:36:52<20:34:48] +[titan] 2025-07-09 15:11:52,694 - root - INFO - step: 7280 loss: 17.6806 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.8084 global_avg_mtp_loss: 14.8722 +[titan] 2025-07-09 15:11:52,695 - root - INFO - lr: 2.9733e-04 gnorm: 0.98 [ 1:36:56<20:34:43] +[titan] 2025-07-09 15:11:56,590 - root - INFO - step: 7285 loss: 17.5494 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.7766 global_avg_mtp_loss: 14.7728 +[titan] 2025-07-09 15:11:56,591 - root - INFO - lr: 2.9732e-04 gnorm: 1.00 [ 1:37:00<20:34:37] +[titan] 2025-07-09 15:12:00,516 - root - INFO - step: 7290 loss: 17.5711 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.7757 global_avg_mtp_loss: 14.7954 +[titan] 2025-07-09 15:12:00,516 - root - INFO - lr: 2.9732e-04 gnorm: 1.00 [ 1:37:04<20:34:32] +[titan] 2025-07-09 15:12:04,429 - root - INFO - step: 7295 loss: 17.6640 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7998 global_avg_mtp_loss: 14.8642 +[titan] 2025-07-09 15:12:04,429 - root - INFO - lr: 2.9731e-04 gnorm: 0.96 [ 1:37:08<20:34:27] +[titan] 2025-07-09 15:12:07,538 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:12:08,335 - root - INFO - step: 7300 loss: 17.5520 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.7814 global_avg_mtp_loss: 14.7706 +[titan] 2025-07-09 15:12:08,335 - root - INFO - lr: 2.9731e-04 gnorm: 0.83 [ 1:37:12<20:34:22] +[titan] 2025-07-09 15:12:12,257 - root - INFO - step: 7305 loss: 17.7853 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.8362 global_avg_mtp_loss: 14.9491 +[titan] 2025-07-09 15:12:12,257 - root - INFO - lr: 2.9731e-04 gnorm: 0.95 [ 1:37:16<20:34:17] +[titan] 2025-07-09 15:12:16,163 - root - INFO - step: 7310 loss: 17.4905 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.7622 global_avg_mtp_loss: 14.7283 +[titan] 2025-07-09 15:12:16,163 - root - INFO - lr: 2.9730e-04 gnorm: 0.93 [ 1:37:20<20:34:12] +[titan] 2025-07-09 15:12:20,055 - root - INFO - step: 7315 loss: 17.8544 memory: 44.58GiB(31.99%) tps: 84,205 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.8534 global_avg_mtp_loss: 15.0010 +[titan] 2025-07-09 15:12:20,055 - root - INFO - lr: 2.9730e-04 gnorm: 0.93 [ 1:37:24<20:34:07] +[titan] 2025-07-09 15:12:23,955 - root - INFO - step: 7320 loss: 17.6392 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.7995 global_avg_mtp_loss: 14.8397 +[titan] 2025-07-09 15:12:23,956 - root - INFO - lr: 2.9729e-04 gnorm: 0.89 [ 1:37:27<20:34:02] +[titan] 2025-07-09 15:12:27,862 - root - INFO - step: 7325 loss: 17.7415 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.8419 global_avg_mtp_loss: 14.8996 +[titan] 2025-07-09 15:12:27,862 - root - INFO - lr: 2.9729e-04 gnorm: 0.90 [ 1:37:31<20:33:57] +[titan] 2025-07-09 15:12:31,775 - root - INFO - step: 7330 loss: 17.8262 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.8263 global_avg_mtp_loss: 14.9999 +[titan] 2025-07-09 15:12:31,775 - root - INFO - lr: 2.9728e-04 gnorm: 0.98 [ 1:37:35<20:33:52] +[titan] 2025-07-09 15:12:35,693 - root - INFO - step: 7335 loss: 17.7625 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.8545 global_avg_mtp_loss: 14.9080 +[titan] 2025-07-09 15:12:35,693 - root - INFO - lr: 2.9728e-04 gnorm: 1.02 [ 1:37:39<20:33:47] +[titan] 2025-07-09 15:12:39,684 - root - INFO - step: 7340 loss: 17.6706 memory: 44.58GiB(31.99%) tps: 82,120 tflops: 283.41 mfu: 28.66% global_avg_ntp_loss: 2.8083 global_avg_mtp_loss: 14.8624 +[titan] 2025-07-09 15:12:39,684 - root - INFO - lr: 2.9728e-04 gnorm: 0.88 [ 1:37:43<20:33:43] +[titan] 2025-07-09 15:12:43,580 - root - INFO - step: 7345 loss: 17.3999 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.7598 global_avg_mtp_loss: 14.6401 +[titan] 2025-07-09 15:12:43,580 - root - INFO - lr: 2.9727e-04 gnorm: 1.06 [ 1:37:47<20:33:37] +[titan] 2025-07-09 15:12:46,689 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:12:47,478 - root - INFO - step: 7350 loss: 17.7237 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.8229 global_avg_mtp_loss: 14.9008 +[titan] 2025-07-09 15:12:47,478 - root - INFO - lr: 2.9727e-04 gnorm: 1.00 [ 1:37:51<20:33:32] +[titan] 2025-07-09 15:12:51,411 - root - INFO - step: 7355 loss: 17.7846 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.8188 global_avg_mtp_loss: 14.9658 +[titan] 2025-07-09 15:12:51,411 - root - INFO - lr: 2.9726e-04 gnorm: 0.91 [ 1:37:55<20:33:27] +[titan] 2025-07-09 15:12:55,372 - root - INFO - step: 7360 loss: 17.5243 memory: 44.58GiB(31.99%) tps: 82,735 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.7835 global_avg_mtp_loss: 14.7408 +[titan] 2025-07-09 15:12:55,372 - root - INFO - lr: 2.9726e-04 gnorm: 0.97 [ 1:37:59<20:33:23] +[titan] 2025-07-09 15:12:59,266 - root - INFO - step: 7365 loss: 17.5152 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.7723 global_avg_mtp_loss: 14.7430 +[titan] 2025-07-09 15:12:59,267 - root - INFO - lr: 2.9725e-04 gnorm: 1.00 [ 1:38:03<20:33:18] +[titan] 2025-07-09 15:13:03,162 - root - INFO - step: 7370 loss: 17.5368 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.7754 global_avg_mtp_loss: 14.7614 +[titan] 2025-07-09 15:13:03,162 - root - INFO - lr: 2.9725e-04 gnorm: 1.03 [ 1:38:07<20:33:12] +[titan] 2025-07-09 15:13:07,083 - root - INFO - step: 7375 loss: 17.5226 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7766 global_avg_mtp_loss: 14.7460 +[titan] 2025-07-09 15:13:07,084 - root - INFO - lr: 2.9725e-04 gnorm: 0.88 [ 1:38:11<20:33:08] +[titan] 2025-07-09 15:13:10,984 - root - INFO - step: 7380 loss: 17.6577 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.8012 global_avg_mtp_loss: 14.8565 +[titan] 2025-07-09 15:13:10,984 - root - INFO - lr: 2.9724e-04 gnorm: 0.88 [ 1:38:14<20:33:02] +[titan] 2025-07-09 15:13:14,922 - root - INFO - step: 7385 loss: 17.4885 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.7599 global_avg_mtp_loss: 14.7286 +[titan] 2025-07-09 15:13:14,922 - root - INFO - lr: 2.9724e-04 gnorm: 0.91 [ 1:38:18<20:32:58] +[titan] 2025-07-09 15:13:18,828 - root - INFO - step: 7390 loss: 17.8280 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8338 global_avg_mtp_loss: 14.9941 +[titan] 2025-07-09 15:13:18,829 - root - INFO - lr: 2.9723e-04 gnorm: 1.02 [ 1:38:22<20:32:53] +[titan] 2025-07-09 15:13:22,735 - root - INFO - step: 7395 loss: 17.7013 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.8069 global_avg_mtp_loss: 14.8944 +[titan] 2025-07-09 15:13:22,736 - root - INFO - lr: 2.9723e-04 gnorm: 1.00 [ 1:38:26<20:32:47] +[titan] 2025-07-09 15:13:25,863 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:13:26,651 - root - INFO - step: 7400 loss: 17.5030 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7632 global_avg_mtp_loss: 14.7398 +[titan] 2025-07-09 15:13:26,651 - root - INFO - lr: 2.9722e-04 gnorm: 1.09 [ 1:38:30<20:32:42] +[titan] 2025-07-09 15:13:30,586 - root - INFO - step: 7405 loss: 17.2628 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.7225 global_avg_mtp_loss: 14.5403 +[titan] 2025-07-09 15:13:30,587 - root - INFO - lr: 2.9722e-04 gnorm: 1.01 [ 1:38:34<20:32:38] +[titan] 2025-07-09 15:13:34,495 - root - INFO - step: 7410 loss: 17.6621 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.7944 global_avg_mtp_loss: 14.8677 +[titan] 2025-07-09 15:13:34,495 - root - INFO - lr: 2.9722e-04 gnorm: 0.97 [ 1:38:38<20:32:33] +[titan] 2025-07-09 15:13:38,432 - root - INFO - step: 7415 loss: 17.7068 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.8221 global_avg_mtp_loss: 14.8847 +[titan] 2025-07-09 15:13:38,432 - root - INFO - lr: 2.9721e-04 gnorm: 1.04 [ 1:38:42<20:32:28] +[titan] 2025-07-09 15:13:42,348 - root - INFO - step: 7420 loss: 17.5658 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.7744 global_avg_mtp_loss: 14.7914 +[titan] 2025-07-09 15:13:42,349 - root - INFO - lr: 2.9721e-04 gnorm: 0.89 [ 1:38:46<20:32:23] +[titan] 2025-07-09 15:13:46,253 - root - INFO - step: 7425 loss: 17.5933 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.7954 global_avg_mtp_loss: 14.7979 +[titan] 2025-07-09 15:13:46,254 - root - INFO - lr: 2.9720e-04 gnorm: 0.87 [ 1:38:50<20:32:18] +[titan] 2025-07-09 15:13:50,150 - root - INFO - step: 7430 loss: 17.9178 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.8501 global_avg_mtp_loss: 15.0677 +[titan] 2025-07-09 15:13:50,150 - root - INFO - lr: 2.9720e-04 gnorm: 0.97 [ 1:38:54<20:32:13] +[titan] 2025-07-09 15:13:54,048 - root - INFO - step: 7435 loss: 17.8023 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.8236 global_avg_mtp_loss: 14.9786 +[titan] 2025-07-09 15:13:54,048 - root - INFO - lr: 2.9719e-04 gnorm: 0.97 [ 1:38:58<20:32:08] +[titan] 2025-07-09 15:13:57,951 - root - INFO - step: 7440 loss: 17.5196 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.7728 global_avg_mtp_loss: 14.7468 +[titan] 2025-07-09 15:13:57,951 - root - INFO - lr: 2.9719e-04 gnorm: 0.97 [ 1:39:01<20:32:02] +[titan] 2025-07-09 15:14:01,846 - root - INFO - step: 7445 loss: 17.7282 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.8160 global_avg_mtp_loss: 14.9122 +[titan] 2025-07-09 15:14:01,846 - root - INFO - lr: 2.9719e-04 gnorm: 1.02 [ 1:39:05<20:31:57] +[titan] 2025-07-09 15:14:04,955 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:14:05,740 - root - INFO - step: 7450 loss: 17.5656 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.7770 global_avg_mtp_loss: 14.7887 +[titan] 2025-07-09 15:14:05,741 - root - INFO - lr: 2.9718e-04 gnorm: 0.96 [ 1:39:09<20:31:52] +[titan] 2025-07-09 15:14:09,667 - root - INFO - step: 7455 loss: 17.4606 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.05 mfu: 29.12% global_avg_ntp_loss: 2.7723 global_avg_mtp_loss: 14.6883 +[titan] 2025-07-09 15:14:09,667 - root - INFO - lr: 2.9718e-04 gnorm: 1.01 [ 1:39:13<20:31:47] +[titan] 2025-07-09 15:14:13,570 - root - INFO - step: 7460 loss: 17.5774 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.8042 global_avg_mtp_loss: 14.7732 +[titan] 2025-07-09 15:14:13,570 - root - INFO - lr: 2.9717e-04 gnorm: 0.97 [ 1:39:17<20:31:42] +[titan] 2025-07-09 15:14:17,450 - root - INFO - step: 7465 loss: 17.6127 memory: 44.58GiB(31.99%) tps: 84,446 tflops: 291.44 mfu: 29.47% global_avg_ntp_loss: 2.7982 global_avg_mtp_loss: 14.8144 +[titan] 2025-07-09 15:14:17,451 - root - INFO - lr: 2.9717e-04 gnorm: 0.96 [ 1:39:21<20:31:37] +[titan] 2025-07-09 15:14:21,351 - root - INFO - step: 7470 loss: 17.6045 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.7948 global_avg_mtp_loss: 14.8097 +[titan] 2025-07-09 15:14:21,351 - root - INFO - lr: 2.9716e-04 gnorm: 1.03 [ 1:39:25<20:31:31] +[titan] 2025-07-09 15:14:25,267 - root - INFO - step: 7475 loss: 17.6563 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.7982 global_avg_mtp_loss: 14.8581 +[titan] 2025-07-09 15:14:25,267 - root - INFO - lr: 2.9716e-04 gnorm: 0.95 [ 1:39:29<20:31:27] +[titan] 2025-07-09 15:14:29,201 - root - INFO - step: 7480 loss: 17.8841 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.8452 global_avg_mtp_loss: 15.0389 +[titan] 2025-07-09 15:14:29,201 - root - INFO - lr: 2.9716e-04 gnorm: 0.84 [ 1:39:33<20:31:22] +[titan] 2025-07-09 15:14:33,134 - root - INFO - step: 7485 loss: 17.6013 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.7995 global_avg_mtp_loss: 14.8018 +[titan] 2025-07-09 15:14:33,134 - root - INFO - lr: 2.9715e-04 gnorm: 0.96 [ 1:39:37<20:31:17] +[titan] 2025-07-09 15:14:37,065 - root - INFO - step: 7490 loss: 17.5781 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.7818 global_avg_mtp_loss: 14.7962 +[titan] 2025-07-09 15:14:37,066 - root - INFO - lr: 2.9715e-04 gnorm: 0.94 [ 1:39:41<20:31:12] +[titan] 2025-07-09 15:14:40,988 - root - INFO - step: 7495 loss: 17.5838 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.7952 global_avg_mtp_loss: 14.7886 +[titan] 2025-07-09 15:14:40,989 - root - INFO - lr: 2.9714e-04 gnorm: 0.92 [ 1:39:44<20:31:07] +[titan] 2025-07-09 15:14:44,130 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:14:44,930 - root - INFO - step: 7500 loss: 17.6546 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.8045 global_avg_mtp_loss: 14.8501 +[titan] 2025-07-09 15:14:44,930 - root - INFO - lr: 2.9714e-04 gnorm: 1.00 [ 1:39:48<20:31:03] +[titan] 2025-07-09 15:14:48,847 - root - INFO - step: 7505 loss: 17.8555 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.8376 global_avg_mtp_loss: 15.0179 +[titan] 2025-07-09 15:14:48,847 - root - INFO - lr: 2.9713e-04 gnorm: 1.03 [ 1:39:52<20:30:58] +[titan] 2025-07-09 15:14:52,758 - root - INFO - step: 7510 loss: 17.6172 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.7948 global_avg_mtp_loss: 14.8224 +[titan] 2025-07-09 15:14:52,759 - root - INFO - lr: 2.9713e-04 gnorm: 0.95 [ 1:39:56<20:30:53] +[titan] 2025-07-09 15:14:56,657 - root - INFO - step: 7515 loss: 17.7756 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.8246 global_avg_mtp_loss: 14.9510 +[titan] 2025-07-09 15:14:56,657 - root - INFO - lr: 2.9712e-04 gnorm: 1.06 [ 1:40:00<20:30:48] +[titan] 2025-07-09 15:15:00,578 - root - INFO - step: 7520 loss: 17.3652 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.7575 global_avg_mtp_loss: 14.6077 +[titan] 2025-07-09 15:15:00,578 - root - INFO - lr: 2.9712e-04 gnorm: 0.99 [ 1:40:04<20:30:43] +[titan] 2025-07-09 15:15:04,493 - root - INFO - step: 7525 loss: 17.5666 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.7913 global_avg_mtp_loss: 14.7753 +[titan] 2025-07-09 15:15:04,493 - root - INFO - lr: 2.9712e-04 gnorm: 0.90 [ 1:40:08<20:30:38] +[titan] 2025-07-09 15:15:08,394 - root - INFO - step: 7530 loss: 17.4946 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.7637 global_avg_mtp_loss: 14.7309 +[titan] 2025-07-09 15:15:08,394 - root - INFO - lr: 2.9711e-04 gnorm: 0.95 [ 1:40:12<20:30:33] +[titan] 2025-07-09 15:15:12,282 - root - INFO - step: 7535 loss: 17.5946 memory: 44.58GiB(31.99%) tps: 84,273 tflops: 290.84 mfu: 29.41% global_avg_ntp_loss: 2.7878 global_avg_mtp_loss: 14.8069 +[titan] 2025-07-09 15:15:12,282 - root - INFO - lr: 2.9711e-04 gnorm: 0.97 [ 1:40:16<20:30:28] +[titan] 2025-07-09 15:15:16,228 - root - INFO - step: 7540 loss: 17.3699 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.7414 global_avg_mtp_loss: 14.6285 +[titan] 2025-07-09 15:15:16,228 - root - INFO - lr: 2.9710e-04 gnorm: 0.91 [ 1:40:20<20:30:23] +[titan] 2025-07-09 15:15:20,137 - root - INFO - step: 7545 loss: 17.3612 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.7320 global_avg_mtp_loss: 14.6293 +[titan] 2025-07-09 15:15:20,138 - root - INFO - lr: 2.9710e-04 gnorm: 0.95 [ 1:40:24<20:30:18] +[titan] 2025-07-09 15:15:23,242 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:15:24,030 - root - INFO - step: 7550 loss: 17.7306 memory: 44.58GiB(31.99%) tps: 84,198 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.8318 global_avg_mtp_loss: 14.8989 +[titan] 2025-07-09 15:15:24,030 - root - INFO - lr: 2.9709e-04 gnorm: 1.05 [ 1:40:28<20:30:13] +[titan] 2025-07-09 15:15:27,936 - root - INFO - step: 7555 loss: 17.7219 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.8220 global_avg_mtp_loss: 14.8999 +[titan] 2025-07-09 15:15:27,936 - root - INFO - lr: 2.9709e-04 gnorm: 0.98 [ 1:40:31<20:30:08] +[titan] 2025-07-09 15:15:31,826 - root - INFO - step: 7560 loss: 17.5428 memory: 44.58GiB(31.99%) tps: 84,248 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.7812 global_avg_mtp_loss: 14.7615 +[titan] 2025-07-09 15:15:31,826 - root - INFO - lr: 2.9708e-04 gnorm: 0.92 [ 1:40:35<20:30:02] +[titan] 2025-07-09 15:15:35,737 - root - INFO - step: 7565 loss: 17.6244 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.8274 global_avg_mtp_loss: 14.7970 +[titan] 2025-07-09 15:15:35,737 - root - INFO - lr: 2.9708e-04 gnorm: 1.01 [ 1:40:39<20:29:57] +[titan] 2025-07-09 15:15:39,644 - root - INFO - step: 7570 loss: 17.5910 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.7841 global_avg_mtp_loss: 14.8070 +[titan] 2025-07-09 15:15:39,644 - root - INFO - lr: 2.9708e-04 gnorm: 0.92 [ 1:40:43<20:29:52] +[titan] 2025-07-09 15:15:43,538 - root - INFO - step: 7575 loss: 17.5273 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.7765 global_avg_mtp_loss: 14.7508 +[titan] 2025-07-09 15:15:43,538 - root - INFO - lr: 2.9707e-04 gnorm: 0.92 [ 1:40:47<20:29:47] +[titan] 2025-07-09 15:15:47,449 - root - INFO - step: 7580 loss: 17.7315 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.8084 global_avg_mtp_loss: 14.9231 +[titan] 2025-07-09 15:15:47,450 - root - INFO - lr: 2.9707e-04 gnorm: 0.87 [ 1:40:51<20:29:42] +[titan] 2025-07-09 15:15:51,334 - root - INFO - step: 7585 loss: 17.6447 memory: 44.58GiB(31.99%) tps: 84,354 tflops: 291.12 mfu: 29.44% global_avg_ntp_loss: 2.7874 global_avg_mtp_loss: 14.8573 +[titan] 2025-07-09 15:15:51,335 - root - INFO - lr: 2.9706e-04 gnorm: 1.01 [ 1:40:55<20:29:37] +[titan] 2025-07-09 15:15:55,231 - root - INFO - step: 7590 loss: 17.6689 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.7962 global_avg_mtp_loss: 14.8727 +[titan] 2025-07-09 15:15:55,232 - root - INFO - lr: 2.9706e-04 gnorm: 0.94 [ 1:40:59<20:29:32] +[titan] 2025-07-09 15:15:59,134 - root - INFO - step: 7595 loss: 17.4663 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.7536 global_avg_mtp_loss: 14.7127 +[titan] 2025-07-09 15:15:59,134 - root - INFO - lr: 2.9705e-04 gnorm: 0.96 [ 1:41:03<20:29:27] +[titan] 2025-07-09 15:16:02,284 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:16:03,069 - root - INFO - step: 7600 loss: 17.8050 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.8306 global_avg_mtp_loss: 14.9744 +[titan] 2025-07-09 15:16:03,069 - root - INFO - lr: 2.9705e-04 gnorm: 0.90 [ 1:41:07<20:29:22] +[titan] 2025-07-09 15:16:06,986 - root - INFO - step: 7605 loss: 17.5160 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.7753 global_avg_mtp_loss: 14.7406 +[titan] 2025-07-09 15:16:06,986 - root - INFO - lr: 2.9704e-04 gnorm: 0.96 [ 1:41:10<20:29:17] +[titan] 2025-07-09 15:16:10,895 - root - INFO - step: 7610 loss: 17.7837 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.8127 global_avg_mtp_loss: 14.9710 +[titan] 2025-07-09 15:16:10,895 - root - INFO - lr: 2.9704e-04 gnorm: 0.95 [ 1:41:14<20:29:12] +[titan] 2025-07-09 15:16:14,818 - root - INFO - step: 7615 loss: 17.4530 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.7688 global_avg_mtp_loss: 14.6842 +[titan] 2025-07-09 15:16:14,819 - root - INFO - lr: 2.9704e-04 gnorm: 0.99 [ 1:41:18<20:29:07] +[titan] 2025-07-09 15:16:18,731 - root - INFO - step: 7620 loss: 17.3573 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.7494 global_avg_mtp_loss: 14.6079 +[titan] 2025-07-09 15:16:18,731 - root - INFO - lr: 2.9703e-04 gnorm: 1.01 [ 1:41:22<20:29:02] +[titan] 2025-07-09 15:16:22,650 - root - INFO - step: 7625 loss: 17.5156 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.7752 global_avg_mtp_loss: 14.7404 +[titan] 2025-07-09 15:16:22,650 - root - INFO - lr: 2.9703e-04 gnorm: 1.05 [ 1:41:26<20:28:57] +[titan] 2025-07-09 15:16:26,545 - root - INFO - step: 7630 loss: 17.4695 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7533 global_avg_mtp_loss: 14.7162 +[titan] 2025-07-09 15:16:26,545 - root - INFO - lr: 2.9702e-04 gnorm: 0.99 [ 1:41:30<20:28:52] +[titan] 2025-07-09 15:16:30,453 - root - INFO - step: 7635 loss: 17.4249 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.7650 global_avg_mtp_loss: 14.6599 +[titan] 2025-07-09 15:16:30,454 - root - INFO - lr: 2.9702e-04 gnorm: 0.91 [ 1:41:34<20:28:47] +[titan] 2025-07-09 15:16:34,362 - root - INFO - step: 7640 loss: 17.1839 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.7102 global_avg_mtp_loss: 14.4738 +[titan] 2025-07-09 15:16:34,362 - root - INFO - lr: 2.9701e-04 gnorm: 0.92 [ 1:41:38<20:28:42] +[titan] 2025-07-09 15:16:38,350 - root - INFO - step: 7645 loss: 17.3653 memory: 44.58GiB(31.99%) tps: 82,177 tflops: 283.61 mfu: 28.68% global_avg_ntp_loss: 2.7464 global_avg_mtp_loss: 14.6189 +[titan] 2025-07-09 15:16:38,350 - root - INFO - lr: 2.9701e-04 gnorm: 0.95 [ 1:41:42<20:28:38] +[titan] 2025-07-09 15:16:41,475 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:16:42,269 - root - INFO - step: 7650 loss: 17.4120 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.7518 global_avg_mtp_loss: 14.6602 +[titan] 2025-07-09 15:16:42,269 - root - INFO - lr: 2.9700e-04 gnorm: 0.96 [ 1:41:46<20:28:33] +[titan] 2025-07-09 15:16:46,168 - root - INFO - step: 7655 loss: 17.6386 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.7992 global_avg_mtp_loss: 14.8394 +[titan] 2025-07-09 15:16:46,168 - root - INFO - lr: 2.9700e-04 gnorm: 0.95 [ 1:41:50<20:28:28] +[titan] 2025-07-09 15:16:50,098 - root - INFO - step: 7660 loss: 17.7632 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.8235 global_avg_mtp_loss: 14.9397 +[titan] 2025-07-09 15:16:50,098 - root - INFO - lr: 2.9700e-04 gnorm: 0.93 [ 1:41:54<20:28:24] +[titan] 2025-07-09 15:16:53,999 - root - INFO - step: 7665 loss: 17.5848 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.7914 global_avg_mtp_loss: 14.7934 +[titan] 2025-07-09 15:16:53,999 - root - INFO - lr: 2.9699e-04 gnorm: 0.92 [ 1:41:57<20:28:19] +[titan] 2025-07-09 15:16:57,904 - root - INFO - step: 7670 loss: 17.4787 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7750 global_avg_mtp_loss: 14.7037 +[titan] 2025-07-09 15:16:57,905 - root - INFO - lr: 2.9699e-04 gnorm: 0.84 [ 1:42:01<20:28:13] +[titan] 2025-07-09 15:17:01,802 - root - INFO - step: 7675 loss: 17.8635 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.8407 global_avg_mtp_loss: 15.0228 +[titan] 2025-07-09 15:17:01,802 - root - INFO - lr: 2.9698e-04 gnorm: 1.00 [ 1:42:05<20:28:08] +[titan] 2025-07-09 15:17:05,843 - root - INFO - step: 7680 loss: 17.5345 memory: 44.58GiB(31.99%) tps: 81,096 tflops: 279.88 mfu: 28.30% global_avg_ntp_loss: 2.7785 global_avg_mtp_loss: 14.7560 +[titan] 2025-07-09 15:17:05,843 - root - INFO - lr: 2.9698e-04 gnorm: 0.91 [ 1:42:09<20:28:05] +[titan] 2025-07-09 15:17:05,991 - root - INFO - Dumping profiler traces at step 7680 +[titan] 2025-07-09 15:17:06,026 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-09 15:17:09,957 - root - INFO - step: 7685 loss: 17.5169 memory: 44.58GiB(31.99%) tps: 79,662 tflops: 274.93 mfu: 27.80% global_avg_ntp_loss: 2.7798 global_avg_mtp_loss: 14.7371 +[titan] 2025-07-09 15:17:09,957 - root - INFO - lr: 2.9697e-04 gnorm: 0.94 [ 1:42:13<20:28:02] +[titan] 2025-07-09 15:17:13,900 - root - INFO - step: 7690 loss: 17.7339 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.8318 global_avg_mtp_loss: 14.9021 +[titan] 2025-07-09 15:17:13,900 - root - INFO - lr: 2.9697e-04 gnorm: 1.00 [ 1:42:17<20:27:58] +[titan] 2025-07-09 15:17:17,855 - root - INFO - step: 7695 loss: 17.4264 memory: 44.58GiB(31.99%) tps: 82,853 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.7566 global_avg_mtp_loss: 14.6698 +[titan] 2025-07-09 15:17:17,855 - root - INFO - lr: 2.9696e-04 gnorm: 1.04 [ 1:42:21<20:27:53] +[titan] 2025-07-09 15:17:21,025 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:17:21,811 - root - INFO - step: 7700 loss: 17.7371 memory: 44.58GiB(31.99%) tps: 82,833 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.8176 global_avg_mtp_loss: 14.9195 +[titan] 2025-07-09 15:17:21,812 - root - INFO - lr: 2.9696e-04 gnorm: 0.90 [ 1:42:25<20:27:49] +[titan] 2025-07-09 15:17:25,725 - root - INFO - step: 7705 loss: 17.5930 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.7763 global_avg_mtp_loss: 14.8167 +[titan] 2025-07-09 15:17:25,726 - root - INFO - lr: 2.9695e-04 gnorm: 0.92 [ 1:42:29<20:27:44] +[titan] 2025-07-09 15:17:29,711 - root - INFO - step: 7710 loss: 17.4992 memory: 44.58GiB(31.99%) tps: 82,218 tflops: 283.75 mfu: 28.69% global_avg_ntp_loss: 2.7673 global_avg_mtp_loss: 14.7319 +[titan] 2025-07-09 15:17:29,712 - root - INFO - lr: 2.9695e-04 gnorm: 0.88 [ 1:42:33<20:27:40] +[titan] 2025-07-09 15:17:33,623 - root - INFO - step: 7715 loss: 17.5711 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.7819 global_avg_mtp_loss: 14.7892 +[titan] 2025-07-09 15:17:33,623 - root - INFO - lr: 2.9695e-04 gnorm: 0.93 [ 1:42:37<20:27:35] +[titan] 2025-07-09 15:17:37,533 - root - INFO - step: 7720 loss: 17.5057 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.7677 global_avg_mtp_loss: 14.7381 +[titan] 2025-07-09 15:17:37,533 - root - INFO - lr: 2.9694e-04 gnorm: 0.94 [ 1:42:41<20:27:30] +[titan] 2025-07-09 15:17:41,442 - root - INFO - step: 7725 loss: 17.4962 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.7645 global_avg_mtp_loss: 14.7317 +[titan] 2025-07-09 15:17:41,442 - root - INFO - lr: 2.9694e-04 gnorm: 0.90 [ 1:42:45<20:27:25] +[titan] 2025-07-09 15:17:45,378 - root - INFO - step: 7730 loss: 17.5554 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7743 global_avg_mtp_loss: 14.7811 +[titan] 2025-07-09 15:17:45,378 - root - INFO - lr: 2.9693e-04 gnorm: 0.94 [ 1:42:49<20:27:21] +[titan] 2025-07-09 15:17:49,288 - root - INFO - step: 7735 loss: 17.5538 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.7818 global_avg_mtp_loss: 14.7720 +[titan] 2025-07-09 15:17:49,288 - root - INFO - lr: 2.9693e-04 gnorm: 1.04 [ 1:42:53<20:27:16] +[titan] 2025-07-09 15:17:53,214 - root - INFO - step: 7740 loss: 17.6135 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.7979 global_avg_mtp_loss: 14.8156 +[titan] 2025-07-09 15:17:53,214 - root - INFO - lr: 2.9692e-04 gnorm: 0.88 [ 1:42:57<20:27:11] +[titan] 2025-07-09 15:17:57,129 - root - INFO - step: 7745 loss: 17.4772 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.7826 global_avg_mtp_loss: 14.6946 +[titan] 2025-07-09 15:17:57,130 - root - INFO - lr: 2.9692e-04 gnorm: 0.91 [ 1:43:01<20:27:06] +[titan] 2025-07-09 15:18:00,268 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:18:01,054 - root - INFO - step: 7750 loss: 17.4109 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.7599 global_avg_mtp_loss: 14.6510 +[titan] 2025-07-09 15:18:01,054 - root - INFO - lr: 2.9691e-04 gnorm: 1.01 [ 1:43:05<20:27:01] +[titan] 2025-07-09 15:18:04,952 - root - INFO - step: 7755 loss: 17.5743 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.7787 global_avg_mtp_loss: 14.7955 +[titan] 2025-07-09 15:18:04,953 - root - INFO - lr: 2.9691e-04 gnorm: 0.88 [ 1:43:08<20:26:56] +[titan] 2025-07-09 15:18:08,850 - root - INFO - step: 7760 loss: 17.2721 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.7118 global_avg_mtp_loss: 14.5604 +[titan] 2025-07-09 15:18:08,850 - root - INFO - lr: 2.9690e-04 gnorm: 0.86 [ 1:43:12<20:26:51] +[titan] 2025-07-09 15:18:12,743 - root - INFO - step: 7765 loss: 17.4028 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.7656 global_avg_mtp_loss: 14.6372 +[titan] 2025-07-09 15:18:12,744 - root - INFO - lr: 2.9690e-04 gnorm: 0.93 [ 1:43:16<20:26:46] +[titan] 2025-07-09 15:18:16,706 - root - INFO - step: 7770 loss: 17.3784 memory: 44.58GiB(31.99%) tps: 82,703 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.7391 global_avg_mtp_loss: 14.6393 +[titan] 2025-07-09 15:18:16,706 - root - INFO - lr: 2.9690e-04 gnorm: 0.90 [ 1:43:20<20:26:42] +[titan] 2025-07-09 15:18:20,603 - root - INFO - step: 7775 loss: 17.6624 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.7977 global_avg_mtp_loss: 14.8647 +[titan] 2025-07-09 15:18:20,603 - root - INFO - lr: 2.9689e-04 gnorm: 0.97 [ 1:43:24<20:26:36] +[titan] 2025-07-09 15:18:24,520 - root - INFO - step: 7780 loss: 17.3180 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7441 global_avg_mtp_loss: 14.5740 +[titan] 2025-07-09 15:18:24,521 - root - INFO - lr: 2.9689e-04 gnorm: 1.01 [ 1:43:28<20:26:32] +[titan] 2025-07-09 15:18:28,470 - root - INFO - step: 7785 loss: 17.6329 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.7947 global_avg_mtp_loss: 14.8381 +[titan] 2025-07-09 15:18:28,470 - root - INFO - lr: 2.9688e-04 gnorm: 0.93 [ 1:43:32<20:26:27] +[titan] 2025-07-09 15:18:32,406 - root - INFO - step: 7790 loss: 17.5450 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.7647 global_avg_mtp_loss: 14.7803 +[titan] 2025-07-09 15:18:32,407 - root - INFO - lr: 2.9688e-04 gnorm: 0.97 [ 1:43:36<20:26:22] +[titan] 2025-07-09 15:18:36,321 - root - INFO - step: 7795 loss: 17.7574 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.8178 global_avg_mtp_loss: 14.9396 +[titan] 2025-07-09 15:18:36,321 - root - INFO - lr: 2.9687e-04 gnorm: 1.06 [ 1:43:40<20:26:18] +[titan] 2025-07-09 15:18:39,480 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:18:40,271 - root - INFO - step: 7800 loss: 17.5726 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.7721 global_avg_mtp_loss: 14.8006 +[titan] 2025-07-09 15:18:40,271 - root - INFO - lr: 2.9687e-04 gnorm: 0.96 [ 1:43:44<20:26:13] +[titan] 2025-07-09 15:18:44,191 - root - INFO - step: 7805 loss: 17.6886 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.7911 global_avg_mtp_loss: 14.8975 +[titan] 2025-07-09 15:18:44,191 - root - INFO - lr: 2.9686e-04 gnorm: 1.02 [ 1:43:48<20:26:08] +[titan] 2025-07-09 15:18:48,105 - root - INFO - step: 7810 loss: 17.6709 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.8172 global_avg_mtp_loss: 14.8537 +[titan] 2025-07-09 15:18:48,105 - root - INFO - lr: 2.9686e-04 gnorm: 0.94 [ 1:43:52<20:26:03] +[titan] 2025-07-09 15:18:52,051 - root - INFO - step: 7815 loss: 17.5121 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.7661 global_avg_mtp_loss: 14.7461 +[titan] 2025-07-09 15:18:52,051 - root - INFO - lr: 2.9685e-04 gnorm: 0.98 [ 1:43:56<20:25:59] +[titan] 2025-07-09 15:18:55,965 - root - INFO - step: 7820 loss: 17.5582 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.7876 global_avg_mtp_loss: 14.7706 +[titan] 2025-07-09 15:18:55,965 - root - INFO - lr: 2.9685e-04 gnorm: 0.86 [ 1:43:59<20:25:54] +[titan] 2025-07-09 15:18:59,864 - root - INFO - step: 7825 loss: 17.6124 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.7988 global_avg_mtp_loss: 14.8136 +[titan] 2025-07-09 15:18:59,864 - root - INFO - lr: 2.9685e-04 gnorm: 1.07 [ 1:44:03<20:25:49] +[titan] 2025-07-09 15:19:03,770 - root - INFO - step: 7830 loss: 17.7786 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.8332 global_avg_mtp_loss: 14.9453 +[titan] 2025-07-09 15:19:03,770 - root - INFO - lr: 2.9684e-04 gnorm: 0.94 [ 1:44:07<20:25:44] +[titan] 2025-07-09 15:19:07,688 - root - INFO - step: 7835 loss: 17.4262 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.7514 global_avg_mtp_loss: 14.6749 +[titan] 2025-07-09 15:19:07,688 - root - INFO - lr: 2.9684e-04 gnorm: 1.08 [ 1:44:11<20:25:39] +[titan] 2025-07-09 15:19:11,625 - root - INFO - step: 7840 loss: 17.4717 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.7455 global_avg_mtp_loss: 14.7261 +[titan] 2025-07-09 15:19:11,625 - root - INFO - lr: 2.9683e-04 gnorm: 0.97 [ 1:44:15<20:25:35] +[titan] 2025-07-09 15:19:15,550 - root - INFO - step: 7845 loss: 17.3083 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.7444 global_avg_mtp_loss: 14.5639 +[titan] 2025-07-09 15:19:15,551 - root - INFO - lr: 2.9683e-04 gnorm: 0.87 [ 1:44:19<20:25:30] +[titan] 2025-07-09 15:19:18,687 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:19:19,475 - root - INFO - step: 7850 loss: 17.6651 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.7913 global_avg_mtp_loss: 14.8738 +[titan] 2025-07-09 15:19:19,475 - root - INFO - lr: 2.9682e-04 gnorm: 1.04 [ 1:44:23<20:25:25] +[titan] 2025-07-09 15:19:23,405 - root - INFO - step: 7855 loss: 17.1635 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.7128 global_avg_mtp_loss: 14.4507 +[titan] 2025-07-09 15:19:23,405 - root - INFO - lr: 2.9682e-04 gnorm: 0.95 [ 1:44:27<20:25:20] +[titan] 2025-07-09 15:19:27,347 - root - INFO - step: 7860 loss: 17.5368 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.7781 global_avg_mtp_loss: 14.7587 +[titan] 2025-07-09 15:19:27,347 - root - INFO - lr: 2.9681e-04 gnorm: 0.96 [ 1:44:31<20:25:16] +[titan] 2025-07-09 15:19:31,243 - root - INFO - step: 7865 loss: 17.4080 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.7629 global_avg_mtp_loss: 14.6451 +[titan] 2025-07-09 15:19:31,243 - root - INFO - lr: 2.9681e-04 gnorm: 0.96 [ 1:44:35<20:25:11] +[titan] 2025-07-09 15:19:35,148 - root - INFO - step: 7870 loss: 17.7071 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.8087 global_avg_mtp_loss: 14.8984 +[titan] 2025-07-09 15:19:35,148 - root - INFO - lr: 2.9680e-04 gnorm: 0.98 [ 1:44:39<20:25:06] +[titan] 2025-07-09 15:19:39,114 - root - INFO - step: 7875 loss: 17.8435 memory: 44.58GiB(31.99%) tps: 82,623 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.8497 global_avg_mtp_loss: 14.9938 +[titan] 2025-07-09 15:19:39,115 - root - INFO - lr: 2.9680e-04 gnorm: 0.90 [ 1:44:43<20:25:01] +[titan] 2025-07-09 15:19:43,006 - root - INFO - step: 7880 loss: 17.5429 memory: 44.58GiB(31.99%) tps: 84,215 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.7841 global_avg_mtp_loss: 14.7589 +[titan] 2025-07-09 15:19:43,006 - root - INFO - lr: 2.9679e-04 gnorm: 1.09 [ 1:44:46<20:24:56] +[titan] 2025-07-09 15:19:46,909 - root - INFO - step: 7885 loss: 17.4200 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7484 global_avg_mtp_loss: 14.6716 +[titan] 2025-07-09 15:19:46,909 - root - INFO - lr: 2.9679e-04 gnorm: 1.13 [ 1:44:50<20:24:51] +[titan] 2025-07-09 15:19:50,846 - root - INFO - step: 7890 loss: 17.7162 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.8126 global_avg_mtp_loss: 14.9035 +[titan] 2025-07-09 15:19:50,846 - root - INFO - lr: 2.9679e-04 gnorm: 0.89 [ 1:44:54<20:24:47] +[titan] 2025-07-09 15:19:54,731 - root - INFO - step: 7895 loss: 17.5577 memory: 44.58GiB(31.99%) tps: 84,359 tflops: 291.14 mfu: 29.44% global_avg_ntp_loss: 2.7725 global_avg_mtp_loss: 14.7852 +[titan] 2025-07-09 15:19:54,731 - root - INFO - lr: 2.9678e-04 gnorm: 0.88 [ 1:44:58<20:24:41] +[titan] 2025-07-09 15:19:57,858 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:19:58,654 - root - INFO - step: 7900 loss: 17.5378 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.7774 global_avg_mtp_loss: 14.7604 +[titan] 2025-07-09 15:19:58,655 - root - INFO - lr: 2.9678e-04 gnorm: 0.92 [ 1:45:02<20:24:37] +[titan] 2025-07-09 15:20:02,562 - root - INFO - step: 7905 loss: 17.5569 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.7660 global_avg_mtp_loss: 14.7909 +[titan] 2025-07-09 15:20:02,562 - root - INFO - lr: 2.9677e-04 gnorm: 0.95 [ 1:45:06<20:24:32] +[titan] 2025-07-09 15:20:06,458 - root - INFO - step: 7910 loss: 17.3108 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7412 global_avg_mtp_loss: 14.5697 +[titan] 2025-07-09 15:20:06,458 - root - INFO - lr: 2.9677e-04 gnorm: 0.90 [ 1:45:10<20:24:27] +[titan] 2025-07-09 15:20:10,356 - root - INFO - step: 7915 loss: 17.4753 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.7605 global_avg_mtp_loss: 14.7147 +[titan] 2025-07-09 15:20:10,357 - root - INFO - lr: 2.9676e-04 gnorm: 1.07 [ 1:45:14<20:24:22] +[titan] 2025-07-09 15:20:14,264 - root - INFO - step: 7920 loss: 17.4854 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.7711 global_avg_mtp_loss: 14.7143 +[titan] 2025-07-09 15:20:14,264 - root - INFO - lr: 2.9676e-04 gnorm: 1.05 [ 1:45:18<20:24:17] +[titan] 2025-07-09 15:20:18,176 - root - INFO - step: 7925 loss: 17.4149 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.7612 global_avg_mtp_loss: 14.6538 +[titan] 2025-07-09 15:20:18,176 - root - INFO - lr: 2.9675e-04 gnorm: 0.92 [ 1:45:22<20:24:12] +[titan] 2025-07-09 15:20:22,080 - root - INFO - step: 7930 loss: 17.6054 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7813 global_avg_mtp_loss: 14.8241 +[titan] 2025-07-09 15:20:22,080 - root - INFO - lr: 2.9675e-04 gnorm: 1.02 [ 1:45:26<20:24:07] +[titan] 2025-07-09 15:20:25,996 - root - INFO - step: 7935 loss: 17.2183 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.7062 global_avg_mtp_loss: 14.5121 +[titan] 2025-07-09 15:20:25,996 - root - INFO - lr: 2.9674e-04 gnorm: 0.96 [ 1:45:29<20:24:02] +[titan] 2025-07-09 15:20:29,955 - root - INFO - step: 7940 loss: 17.5683 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.7819 global_avg_mtp_loss: 14.7865 +[titan] 2025-07-09 15:20:29,955 - root - INFO - lr: 2.9674e-04 gnorm: 0.93 [ 1:45:33<20:23:58] +[titan] 2025-07-09 15:20:33,847 - root - INFO - step: 7945 loss: 17.5136 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.7542 global_avg_mtp_loss: 14.7594 +[titan] 2025-07-09 15:20:33,848 - root - INFO - lr: 2.9673e-04 gnorm: 0.95 [ 1:45:37<20:23:53] +[titan] 2025-07-09 15:20:36,994 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:20:37,793 - root - INFO - step: 7950 loss: 17.5952 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.7869 global_avg_mtp_loss: 14.8083 +[titan] 2025-07-09 15:20:37,793 - root - INFO - lr: 2.9673e-04 gnorm: 1.00 [ 1:45:41<20:23:48] +[titan] 2025-07-09 15:20:41,684 - root - INFO - step: 7955 loss: 17.5256 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.7577 global_avg_mtp_loss: 14.7679 +[titan] 2025-07-09 15:20:41,684 - root - INFO - lr: 2.9672e-04 gnorm: 0.90 [ 1:45:45<20:23:43] +[titan] 2025-07-09 15:20:45,624 - root - INFO - step: 7960 loss: 17.6840 memory: 44.58GiB(31.99%) tps: 83,171 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.7977 global_avg_mtp_loss: 14.8863 +[titan] 2025-07-09 15:20:45,624 - root - INFO - lr: 2.9672e-04 gnorm: 0.95 [ 1:45:49<20:23:38] +[titan] 2025-07-09 15:20:49,531 - root - INFO - step: 7965 loss: 17.5529 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7783 global_avg_mtp_loss: 14.7746 +[titan] 2025-07-09 15:20:49,532 - root - INFO - lr: 2.9672e-04 gnorm: 0.94 [ 1:45:53<20:23:33] +[titan] 2025-07-09 15:20:53,448 - root - INFO - step: 7970 loss: 17.4683 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.7606 global_avg_mtp_loss: 14.7077 +[titan] 2025-07-09 15:20:53,448 - root - INFO - lr: 2.9671e-04 gnorm: 0.97 [ 1:45:57<20:23:29] +[titan] 2025-07-09 15:20:57,360 - root - INFO - step: 7975 loss: 17.5265 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.7801 global_avg_mtp_loss: 14.7464 +[titan] 2025-07-09 15:20:57,360 - root - INFO - lr: 2.9671e-04 gnorm: 1.04 [ 1:46:01<20:23:24] +[titan] 2025-07-09 15:21:01,249 - root - INFO - step: 7980 loss: 17.3430 memory: 44.58GiB(31.99%) tps: 84,265 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.7335 global_avg_mtp_loss: 14.6095 +[titan] 2025-07-09 15:21:01,249 - root - INFO - lr: 2.9670e-04 gnorm: 1.05 [ 1:46:05<20:23:19] +[titan] 2025-07-09 15:21:05,134 - root - INFO - step: 7985 loss: 17.7576 memory: 44.58GiB(31.99%) tps: 84,344 tflops: 291.09 mfu: 29.43% global_avg_ntp_loss: 2.8173 global_avg_mtp_loss: 14.9403 +[titan] 2025-07-09 15:21:05,134 - root - INFO - lr: 2.9670e-04 gnorm: 1.02 [ 1:46:09<20:23:13] +[titan] 2025-07-09 15:21:09,020 - root - INFO - step: 7990 loss: 17.6753 memory: 44.58GiB(31.99%) tps: 84,344 tflops: 291.09 mfu: 29.43% global_avg_ntp_loss: 2.7935 global_avg_mtp_loss: 14.8818 +[titan] 2025-07-09 15:21:09,020 - root - INFO - lr: 2.9669e-04 gnorm: 1.11 [ 1:46:12<20:23:08] +[titan] 2025-07-09 15:21:12,933 - root - INFO - step: 7995 loss: 17.4343 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.7381 global_avg_mtp_loss: 14.6962 +[titan] 2025-07-09 15:21:12,933 - root - INFO - lr: 2.9669e-04 gnorm: 1.06 [ 1:46:16<20:23:03] +[titan] 2025-07-09 15:21:16,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:21:16,835 - root - INFO - step: 8000 loss: 17.7899 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.8186 global_avg_mtp_loss: 14.9713 +[titan] 2025-07-09 15:21:16,836 - root - INFO - lr: 2.9668e-04 gnorm: 0.98 [ 1:46:20<20:22:58] +[titan] 2025-07-09 15:21:20,724 - root - INFO - step: 8005 loss: 17.4645 memory: 44.58GiB(31.99%) tps: 84,279 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.7740 global_avg_mtp_loss: 14.6905 +[titan] 2025-07-09 15:21:20,724 - root - INFO - lr: 2.9668e-04 gnorm: 1.01 [ 1:46:24<20:22:53] +[titan] 2025-07-09 15:21:24,610 - root - INFO - step: 8010 loss: 17.6154 memory: 44.58GiB(31.99%) tps: 84,326 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 2.7790 global_avg_mtp_loss: 14.8364 +[titan] 2025-07-09 15:21:24,610 - root - INFO - lr: 2.9667e-04 gnorm: 0.99 [ 1:46:28<20:22:48] +[titan] 2025-07-09 15:21:28,532 - root - INFO - step: 8015 loss: 17.6184 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.7882 global_avg_mtp_loss: 14.8302 +[titan] 2025-07-09 15:21:28,532 - root - INFO - lr: 2.9667e-04 gnorm: 0.99 [ 1:46:32<20:22:43] +[titan] 2025-07-09 15:21:32,454 - root - INFO - step: 8020 loss: 17.4506 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.7628 global_avg_mtp_loss: 14.6878 +[titan] 2025-07-09 15:21:32,454 - root - INFO - lr: 2.9666e-04 gnorm: 1.13 [ 1:46:36<20:22:39] +[titan] 2025-07-09 15:21:36,356 - root - INFO - step: 8025 loss: 17.3803 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.7406 global_avg_mtp_loss: 14.6397 +[titan] 2025-07-09 15:21:36,357 - root - INFO - lr: 2.9666e-04 gnorm: 0.99 [ 1:46:40<20:22:34] +[titan] 2025-07-09 15:21:40,322 - root - INFO - step: 8030 loss: 17.6409 memory: 44.58GiB(31.99%) tps: 82,641 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.7888 global_avg_mtp_loss: 14.8521 +[titan] 2025-07-09 15:21:40,322 - root - INFO - lr: 2.9665e-04 gnorm: 1.07 [ 1:46:44<20:22:29] +[titan] 2025-07-09 15:21:44,228 - root - INFO - step: 8035 loss: 17.6079 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.7939 global_avg_mtp_loss: 14.8139 +[titan] 2025-07-09 15:21:44,228 - root - INFO - lr: 2.9665e-04 gnorm: 0.99 [ 1:46:48<20:22:24] +[titan] 2025-07-09 15:21:48,151 - root - INFO - step: 8040 loss: 17.6783 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.7915 global_avg_mtp_loss: 14.8867 +[titan] 2025-07-09 15:21:48,151 - root - INFO - lr: 2.9664e-04 gnorm: 1.00 [ 1:46:52<20:22:20] +[titan] 2025-07-09 15:21:52,049 - root - INFO - step: 8045 loss: 17.5157 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.7683 global_avg_mtp_loss: 14.7473 +[titan] 2025-07-09 15:21:52,050 - root - INFO - lr: 2.9664e-04 gnorm: 0.91 [ 1:46:55<20:22:15] +[titan] 2025-07-09 15:21:55,177 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:21:55,981 - root - INFO - step: 8050 loss: 17.5113 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.7728 global_avg_mtp_loss: 14.7385 +[titan] 2025-07-09 15:21:55,981 - root - INFO - lr: 2.9663e-04 gnorm: 0.93 [ 1:46:59<20:22:10] +[titan] 2025-07-09 15:21:59,906 - root - INFO - step: 8055 loss: 17.7286 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.8290 global_avg_mtp_loss: 14.8996 +[titan] 2025-07-09 15:21:59,906 - root - INFO - lr: 2.9663e-04 gnorm: 1.14 [ 1:47:03<20:22:05] +[titan] 2025-07-09 15:22:03,793 - root - INFO - step: 8060 loss: 17.4612 memory: 44.58GiB(31.99%) tps: 84,315 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.7597 global_avg_mtp_loss: 14.7015 +[titan] 2025-07-09 15:22:03,793 - root - INFO - lr: 2.9663e-04 gnorm: 0.96 [ 1:47:07<20:22:00] +[titan] 2025-07-09 15:22:07,710 - root - INFO - step: 8065 loss: 17.3106 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.7537 global_avg_mtp_loss: 14.5568 +[titan] 2025-07-09 15:22:07,710 - root - INFO - lr: 2.9662e-04 gnorm: 0.95 [ 1:47:11<20:21:55] +[titan] 2025-07-09 15:22:11,617 - root - INFO - step: 8070 loss: 17.6120 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.7836 global_avg_mtp_loss: 14.8284 +[titan] 2025-07-09 15:22:11,617 - root - INFO - lr: 2.9662e-04 gnorm: 0.88 [ 1:47:15<20:21:51] +[titan] 2025-07-09 15:22:15,527 - root - INFO - step: 8075 loss: 17.4378 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.7447 global_avg_mtp_loss: 14.6931 +[titan] 2025-07-09 15:22:15,528 - root - INFO - lr: 2.9661e-04 gnorm: 1.04 [ 1:47:19<20:21:46] +[titan] 2025-07-09 15:22:19,482 - root - INFO - step: 8080 loss: 17.2779 memory: 44.58GiB(31.99%) tps: 82,875 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.7177 global_avg_mtp_loss: 14.5602 +[titan] 2025-07-09 15:22:19,482 - root - INFO - lr: 2.9661e-04 gnorm: 1.00 [ 1:47:23<20:21:41] +[titan] 2025-07-09 15:22:23,375 - root - INFO - step: 8085 loss: 17.4879 memory: 44.58GiB(31.99%) tps: 84,169 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.7743 global_avg_mtp_loss: 14.7136 +[titan] 2025-07-09 15:22:23,375 - root - INFO - lr: 2.9660e-04 gnorm: 1.02 [ 1:47:27<20:21:36] +[titan] 2025-07-09 15:22:27,284 - root - INFO - step: 8090 loss: 17.5834 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7914 global_avg_mtp_loss: 14.7920 +[titan] 2025-07-09 15:22:27,285 - root - INFO - lr: 2.9660e-04 gnorm: 0.96 [ 1:47:31<20:21:31] +[titan] 2025-07-09 15:22:31,228 - root - INFO - step: 8095 loss: 17.3237 memory: 44.58GiB(31.99%) tps: 83,093 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.7303 global_avg_mtp_loss: 14.5933 +[titan] 2025-07-09 15:22:31,229 - root - INFO - lr: 2.9659e-04 gnorm: 0.98 [ 1:47:35<20:21:27] +[titan] 2025-07-09 15:22:34,333 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:22:35,123 - root - INFO - step: 8100 loss: 17.4212 memory: 44.58GiB(31.99%) tps: 84,143 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.7504 global_avg_mtp_loss: 14.6708 +[titan] 2025-07-09 15:22:35,123 - root - INFO - lr: 2.9659e-04 gnorm: 0.95 [ 1:47:39<20:21:22] +[titan] 2025-07-09 15:22:39,028 - root - INFO - step: 8105 loss: 17.4564 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7660 global_avg_mtp_loss: 14.6904 +[titan] 2025-07-09 15:22:39,028 - root - INFO - lr: 2.9658e-04 gnorm: 0.96 [ 1:47:42<20:21:17] +[titan] 2025-07-09 15:22:42,942 - root - INFO - step: 8110 loss: 17.4279 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.7544 global_avg_mtp_loss: 14.6735 +[titan] 2025-07-09 15:22:42,942 - root - INFO - lr: 2.9658e-04 gnorm: 0.90 [ 1:47:46<20:21:12] +[titan] 2025-07-09 15:22:46,852 - root - INFO - step: 8115 loss: 17.3837 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.7438 global_avg_mtp_loss: 14.6399 +[titan] 2025-07-09 15:22:46,853 - root - INFO - lr: 2.9657e-04 gnorm: 0.93 [ 1:47:50<20:21:07] +[titan] 2025-07-09 15:22:50,804 - root - INFO - step: 8120 loss: 17.5452 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.7859 global_avg_mtp_loss: 14.7592 +[titan] 2025-07-09 15:22:50,804 - root - INFO - lr: 2.9657e-04 gnorm: 0.92 [ 1:47:54<20:21:03] +[titan] 2025-07-09 15:22:54,718 - root - INFO - step: 8125 loss: 17.2708 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.7276 global_avg_mtp_loss: 14.5432 +[titan] 2025-07-09 15:22:54,719 - root - INFO - lr: 2.9656e-04 gnorm: 0.97 [ 1:47:58<20:20:58] +[titan] 2025-07-09 15:22:58,640 - root - INFO - step: 8130 loss: 17.6142 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7865 global_avg_mtp_loss: 14.8277 +[titan] 2025-07-09 15:22:58,640 - root - INFO - lr: 2.9656e-04 gnorm: 0.98 [ 1:48:02<20:20:53] +[titan] 2025-07-09 15:23:02,548 - root - INFO - step: 8135 loss: 17.3856 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7492 global_avg_mtp_loss: 14.6364 +[titan] 2025-07-09 15:23:02,548 - root - INFO - lr: 2.9655e-04 gnorm: 0.93 [ 1:48:06<20:20:48] +[titan] 2025-07-09 15:23:06,443 - root - INFO - step: 8140 loss: 17.6579 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.8031 global_avg_mtp_loss: 14.8548 +[titan] 2025-07-09 15:23:06,443 - root - INFO - lr: 2.9655e-04 gnorm: 0.92 [ 1:48:10<20:20:43] +[titan] 2025-07-09 15:23:10,365 - root - INFO - step: 8145 loss: 17.6063 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.7961 global_avg_mtp_loss: 14.8102 +[titan] 2025-07-09 15:23:10,366 - root - INFO - lr: 2.9654e-04 gnorm: 0.87 [ 1:48:14<20:20:39] +[titan] 2025-07-09 15:23:13,511 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:23:14,298 - root - INFO - step: 8150 loss: 17.4343 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.7484 global_avg_mtp_loss: 14.6859 +[titan] 2025-07-09 15:23:14,298 - root - INFO - lr: 2.9654e-04 gnorm: 1.00 [ 1:48:18<20:20:34] +[titan] 2025-07-09 15:23:18,199 - root - INFO - step: 8155 loss: 17.5550 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.7958 global_avg_mtp_loss: 14.7592 +[titan] 2025-07-09 15:23:18,199 - root - INFO - lr: 2.9653e-04 gnorm: 0.96 [ 1:48:22<20:20:29] +[titan] 2025-07-09 15:23:22,180 - root - INFO - step: 8160 loss: 17.5029 memory: 44.58GiB(31.99%) tps: 82,320 tflops: 284.10 mfu: 28.73% global_avg_ntp_loss: 2.7875 global_avg_mtp_loss: 14.7154 +[titan] 2025-07-09 15:23:22,180 - root - INFO - lr: 2.9653e-04 gnorm: 1.05 [ 1:48:26<20:20:25] +[titan] 2025-07-09 15:23:26,102 - root - INFO - step: 8165 loss: 17.4876 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.7584 global_avg_mtp_loss: 14.7292 +[titan] 2025-07-09 15:23:26,103 - root - INFO - lr: 2.9652e-04 gnorm: 1.00 [ 1:48:30<20:20:20] +[titan] 2025-07-09 15:23:30,036 - root - INFO - step: 8170 loss: 17.7655 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.8238 global_avg_mtp_loss: 14.9417 +[titan] 2025-07-09 15:23:30,036 - root - INFO - lr: 2.9652e-04 gnorm: 0.92 [ 1:48:33<20:20:16] +[titan] 2025-07-09 15:23:33,955 - root - INFO - step: 8175 loss: 17.1251 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.7271 global_avg_mtp_loss: 14.3980 +[titan] 2025-07-09 15:23:33,955 - root - INFO - lr: 2.9651e-04 gnorm: 2.12 [ 1:48:37<20:20:11] +[titan] 2025-07-09 15:23:37,880 - root - INFO - step: 8180 loss: 17.5787 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.7858 global_avg_mtp_loss: 14.7930 +[titan] 2025-07-09 15:23:37,881 - root - INFO - lr: 2.9651e-04 gnorm: 0.95 [ 1:48:41<20:20:06] +[titan] 2025-07-09 15:23:41,793 - root - INFO - step: 8185 loss: 17.5462 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.7807 global_avg_mtp_loss: 14.7655 +[titan] 2025-07-09 15:23:41,793 - root - INFO - lr: 2.9651e-04 gnorm: 1.01 [ 1:48:45<20:20:02] +[titan] 2025-07-09 15:23:45,768 - root - INFO - step: 8190 loss: 17.5505 memory: 44.58GiB(31.99%) tps: 82,443 tflops: 284.53 mfu: 28.77% global_avg_ntp_loss: 2.7867 global_avg_mtp_loss: 14.7639 +[titan] 2025-07-09 15:23:45,768 - root - INFO - lr: 2.9650e-04 gnorm: 0.92 [ 1:48:49<20:19:57] +[titan] 2025-07-09 15:23:47,492 - root - INFO - Dumping profiler traces at step 8192 +[titan] 2025-07-09 15:23:47,524 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:23:49,877 - root - INFO - step: 8195 loss: 17.4616 memory: 44.58GiB(31.99%) tps: 79,759 tflops: 275.26 mfu: 27.83% global_avg_ntp_loss: 2.7759 global_avg_mtp_loss: 14.6856 +[titan] 2025-07-09 15:23:49,877 - root - INFO - lr: 2.9650e-04 gnorm: 1.12 [ 1:48:53<20:19:55] +[titan] 2025-07-09 15:23:53,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:23:53,797 - root - INFO - step: 8200 loss: 17.5347 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.7739 global_avg_mtp_loss: 14.7609 +[titan] 2025-07-09 15:23:53,797 - root - INFO - lr: 2.9649e-04 gnorm: 0.93 [ 1:48:57<20:19:50] +[titan] 2025-07-09 15:23:57,701 - root - INFO - step: 8205 loss: 17.6438 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.7818 global_avg_mtp_loss: 14.8619 +[titan] 2025-07-09 15:23:57,702 - root - INFO - lr: 2.9649e-04 gnorm: 0.87 [ 1:49:01<20:19:45] +[titan] 2025-07-09 15:24:01,594 - root - INFO - step: 8210 loss: 17.3998 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.7545 global_avg_mtp_loss: 14.6453 +[titan] 2025-07-09 15:24:01,594 - root - INFO - lr: 2.9648e-04 gnorm: 1.06 [ 1:49:05<20:19:40] +[titan] 2025-07-09 15:24:05,496 - root - INFO - step: 8215 loss: 17.5230 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.7643 global_avg_mtp_loss: 14.7587 +[titan] 2025-07-09 15:24:05,497 - root - INFO - lr: 2.9648e-04 gnorm: 0.92 [ 1:49:09<20:19:35] +[titan] 2025-07-09 15:24:09,391 - root - INFO - step: 8220 loss: 17.6517 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.7960 global_avg_mtp_loss: 14.8557 +[titan] 2025-07-09 15:24:09,391 - root - INFO - lr: 2.9647e-04 gnorm: 0.92 [ 1:49:13<20:19:30] +[titan] 2025-07-09 15:24:13,290 - root - INFO - step: 8225 loss: 17.4378 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.7568 global_avg_mtp_loss: 14.6811 +[titan] 2025-07-09 15:24:13,290 - root - INFO - lr: 2.9647e-04 gnorm: 0.95 [ 1:49:17<20:19:25] +[titan] 2025-07-09 15:24:17,182 - root - INFO - step: 8230 loss: 17.6863 memory: 44.58GiB(31.99%) tps: 84,202 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.8059 global_avg_mtp_loss: 14.8804 +[titan] 2025-07-09 15:24:17,182 - root - INFO - lr: 2.9646e-04 gnorm: 0.92 [ 1:49:21<20:19:20] +[titan] 2025-07-09 15:24:21,080 - root - INFO - step: 8235 loss: 17.2158 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.7117 global_avg_mtp_loss: 14.5040 +[titan] 2025-07-09 15:24:21,081 - root - INFO - lr: 2.9646e-04 gnorm: 0.98 [ 1:49:24<20:19:15] +[titan] 2025-07-09 15:24:24,968 - root - INFO - step: 8240 loss: 17.6275 memory: 44.58GiB(31.99%) tps: 84,290 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.7902 global_avg_mtp_loss: 14.8373 +[titan] 2025-07-09 15:24:24,969 - root - INFO - lr: 2.9645e-04 gnorm: 0.96 [ 1:49:28<20:19:10] +[titan] 2025-07-09 15:24:28,903 - root - INFO - step: 8245 loss: 17.4047 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.7483 global_avg_mtp_loss: 14.6564 +[titan] 2025-07-09 15:24:28,903 - root - INFO - lr: 2.9645e-04 gnorm: 0.98 [ 1:49:32<20:19:06] +[titan] 2025-07-09 15:24:32,013 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:24:32,812 - root - INFO - step: 8250 loss: 17.2815 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.7311 global_avg_mtp_loss: 14.5504 +[titan] 2025-07-09 15:24:32,813 - root - INFO - lr: 2.9644e-04 gnorm: 0.97 [ 1:49:36<20:19:01] +[titan] 2025-07-09 15:24:36,744 - root - INFO - step: 8255 loss: 17.3911 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.7520 global_avg_mtp_loss: 14.6391 +[titan] 2025-07-09 15:24:36,744 - root - INFO - lr: 2.9644e-04 gnorm: 0.96 [ 1:49:40<20:18:56] +[titan] 2025-07-09 15:24:40,668 - root - INFO - step: 8260 loss: 17.9340 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.8913 global_avg_mtp_loss: 15.0427 +[titan] 2025-07-09 15:24:40,669 - root - INFO - lr: 2.9643e-04 gnorm: 1.02 [ 1:49:44<20:18:51] +[titan] 2025-07-09 15:24:44,611 - root - INFO - step: 8265 loss: 17.5172 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.7611 global_avg_mtp_loss: 14.7561 +[titan] 2025-07-09 15:24:44,612 - root - INFO - lr: 2.9643e-04 gnorm: 0.92 [ 1:49:48<20:18:47] +[titan] 2025-07-09 15:24:48,583 - root - INFO - step: 8270 loss: 17.6817 memory: 44.58GiB(31.99%) tps: 82,509 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.8046 global_avg_mtp_loss: 14.8770 +[titan] 2025-07-09 15:24:48,584 - root - INFO - lr: 2.9642e-04 gnorm: 0.93 [ 1:49:52<20:18:43] +[titan] 2025-07-09 15:24:52,467 - root - INFO - step: 8275 loss: 17.6789 memory: 44.58GiB(31.99%) tps: 84,379 tflops: 291.20 mfu: 29.44% global_avg_ntp_loss: 2.8001 global_avg_mtp_loss: 14.8788 +[titan] 2025-07-09 15:24:52,467 - root - INFO - lr: 2.9642e-04 gnorm: 0.93 [ 1:49:56<20:18:38] +[titan] 2025-07-09 15:24:56,379 - root - INFO - step: 8280 loss: 17.6029 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.7850 global_avg_mtp_loss: 14.8179 +[titan] 2025-07-09 15:24:56,379 - root - INFO - lr: 2.9641e-04 gnorm: 0.96 [ 1:50:00<20:18:33] +[titan] 2025-07-09 15:25:00,291 - root - INFO - step: 8285 loss: 17.6764 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.8119 global_avg_mtp_loss: 14.8646 +[titan] 2025-07-09 15:25:00,292 - root - INFO - lr: 2.9641e-04 gnorm: 0.90 [ 1:50:04<20:18:28] +[titan] 2025-07-09 15:25:04,183 - root - INFO - step: 8290 loss: 17.4751 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.7502 global_avg_mtp_loss: 14.7249 +[titan] 2025-07-09 15:25:04,183 - root - INFO - lr: 2.9640e-04 gnorm: 0.93 [ 1:50:08<20:18:23] +[titan] 2025-07-09 15:25:08,091 - root - INFO - step: 8295 loss: 17.2775 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7118 global_avg_mtp_loss: 14.5657 +[titan] 2025-07-09 15:25:08,091 - root - INFO - lr: 2.9640e-04 gnorm: 0.94 [ 1:50:12<20:18:18] +[titan] 2025-07-09 15:25:11,222 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:25:12,009 - root - INFO - step: 8300 loss: 17.6815 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.7923 global_avg_mtp_loss: 14.8892 +[titan] 2025-07-09 15:25:12,009 - root - INFO - lr: 2.9639e-04 gnorm: 0.95 [ 1:50:15<20:18:13] +[titan] 2025-07-09 15:25:15,902 - root - INFO - step: 8305 loss: 17.5008 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.7833 global_avg_mtp_loss: 14.7175 +[titan] 2025-07-09 15:25:15,902 - root - INFO - lr: 2.9639e-04 gnorm: 1.01 [ 1:50:19<20:18:08] +[titan] 2025-07-09 15:25:19,797 - root - INFO - step: 8310 loss: 17.4967 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.7600 global_avg_mtp_loss: 14.7367 +[titan] 2025-07-09 15:25:19,798 - root - INFO - lr: 2.9638e-04 gnorm: 0.92 [ 1:50:23<20:18:03] +[titan] 2025-07-09 15:25:23,742 - root - INFO - step: 8315 loss: 17.4809 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.7740 global_avg_mtp_loss: 14.7069 +[titan] 2025-07-09 15:25:23,742 - root - INFO - lr: 2.9638e-04 gnorm: 0.89 [ 1:50:27<20:17:59] +[titan] 2025-07-09 15:25:27,633 - root - INFO - step: 8320 loss: 17.5371 memory: 44.58GiB(31.99%) tps: 84,217 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.7800 global_avg_mtp_loss: 14.7571 +[titan] 2025-07-09 15:25:27,634 - root - INFO - lr: 2.9637e-04 gnorm: 0.84 [ 1:50:31<20:17:54] +[titan] 2025-07-09 15:25:31,551 - root - INFO - step: 8325 loss: 17.6348 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.7848 global_avg_mtp_loss: 14.8500 +[titan] 2025-07-09 15:25:31,552 - root - INFO - lr: 2.9637e-04 gnorm: 0.90 [ 1:50:35<20:17:49] +[titan] 2025-07-09 15:25:35,477 - root - INFO - step: 8330 loss: 17.5171 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.7653 global_avg_mtp_loss: 14.7518 +[titan] 2025-07-09 15:25:35,477 - root - INFO - lr: 2.9636e-04 gnorm: 0.95 [ 1:50:39<20:17:45] +[titan] 2025-07-09 15:25:39,369 - root - INFO - step: 8335 loss: 17.3536 memory: 44.58GiB(31.99%) tps: 84,186 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.7283 global_avg_mtp_loss: 14.6253 +[titan] 2025-07-09 15:25:39,370 - root - INFO - lr: 2.9636e-04 gnorm: 0.98 [ 1:50:43<20:17:40] +[titan] 2025-07-09 15:25:43,273 - root - INFO - step: 8340 loss: 17.4395 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7740 global_avg_mtp_loss: 14.6655 +[titan] 2025-07-09 15:25:43,273 - root - INFO - lr: 2.9635e-04 gnorm: 0.99 [ 1:50:47<20:17:35] +[titan] 2025-07-09 15:25:47,169 - root - INFO - step: 8345 loss: 17.5494 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.7645 global_avg_mtp_loss: 14.7849 +[titan] 2025-07-09 15:25:47,170 - root - INFO - lr: 2.9635e-04 gnorm: 0.94 [ 1:50:51<20:17:30] +[titan] 2025-07-09 15:25:50,292 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:25:51,080 - root - INFO - step: 8350 loss: 17.5610 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.7741 global_avg_mtp_loss: 14.7869 +[titan] 2025-07-09 15:25:51,080 - root - INFO - lr: 2.9634e-04 gnorm: 0.87 [ 1:50:54<20:17:25] +[titan] 2025-07-09 15:25:54,979 - root - INFO - step: 8355 loss: 17.6148 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.7905 global_avg_mtp_loss: 14.8244 +[titan] 2025-07-09 15:25:54,980 - root - INFO - lr: 2.9634e-04 gnorm: 0.91 [ 1:50:58<20:17:20] +[titan] 2025-07-09 15:25:58,927 - root - INFO - step: 8360 loss: 17.3737 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.7441 global_avg_mtp_loss: 14.6296 +[titan] 2025-07-09 15:25:58,928 - root - INFO - lr: 2.9633e-04 gnorm: 0.88 [ 1:51:02<20:17:16] +[titan] 2025-07-09 15:26:02,821 - root - INFO - step: 8365 loss: 17.3410 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.7476 global_avg_mtp_loss: 14.5934 +[titan] 2025-07-09 15:26:02,821 - root - INFO - lr: 2.9633e-04 gnorm: 0.88 [ 1:51:06<20:17:11] +[titan] 2025-07-09 15:26:06,792 - root - INFO - step: 8370 loss: 17.4493 memory: 44.58GiB(31.99%) tps: 82,528 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.7523 global_avg_mtp_loss: 14.6970 +[titan] 2025-07-09 15:26:06,792 - root - INFO - lr: 2.9632e-04 gnorm: 0.91 [ 1:51:10<20:17:07] +[titan] 2025-07-09 15:26:10,676 - root - INFO - step: 8375 loss: 17.4969 memory: 44.58GiB(31.99%) tps: 84,362 tflops: 291.15 mfu: 29.44% global_avg_ntp_loss: 2.7540 global_avg_mtp_loss: 14.7429 +[titan] 2025-07-09 15:26:10,677 - root - INFO - lr: 2.9632e-04 gnorm: 0.93 [ 1:51:14<20:17:01] +[titan] 2025-07-09 15:26:14,587 - root - INFO - step: 8380 loss: 17.3557 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.7271 global_avg_mtp_loss: 14.6287 +[titan] 2025-07-09 15:26:14,587 - root - INFO - lr: 2.9631e-04 gnorm: 1.01 [ 1:51:18<20:16:57] +[titan] 2025-07-09 15:26:18,489 - root - INFO - step: 8385 loss: 17.5519 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.7764 global_avg_mtp_loss: 14.7755 +[titan] 2025-07-09 15:26:18,489 - root - INFO - lr: 2.9631e-04 gnorm: 0.92 [ 1:51:22<20:16:52] +[titan] 2025-07-09 15:26:22,377 - root - INFO - step: 8390 loss: 17.5026 memory: 44.58GiB(31.99%) tps: 84,288 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.7644 global_avg_mtp_loss: 14.7382 +[titan] 2025-07-09 15:26:22,377 - root - INFO - lr: 2.9630e-04 gnorm: 1.07 [ 1:51:26<20:16:47] +[titan] 2025-07-09 15:26:26,277 - root - INFO - step: 8395 loss: 17.5297 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.7689 global_avg_mtp_loss: 14.7608 +[titan] 2025-07-09 15:26:26,278 - root - INFO - lr: 2.9630e-04 gnorm: 0.94 [ 1:51:30<20:16:42] +[titan] 2025-07-09 15:26:29,416 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:26:30,210 - root - INFO - step: 8400 loss: 17.6083 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.7797 global_avg_mtp_loss: 14.8286 +[titan] 2025-07-09 15:26:30,210 - root - INFO - lr: 2.9629e-04 gnorm: 0.93 [ 1:51:34<20:16:37] +[titan] 2025-07-09 15:26:34,126 - root - INFO - step: 8405 loss: 17.6348 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.7925 global_avg_mtp_loss: 14.8423 +[titan] 2025-07-09 15:26:34,126 - root - INFO - lr: 2.9629e-04 gnorm: 1.09 [ 1:51:38<20:16:32] +[titan] 2025-07-09 15:26:38,022 - root - INFO - step: 8410 loss: 17.2482 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.7275 global_avg_mtp_loss: 14.5207 +[titan] 2025-07-09 15:26:38,022 - root - INFO - lr: 2.9628e-04 gnorm: 0.90 [ 1:51:41<20:16:28] +[titan] 2025-07-09 15:26:41,964 - root - INFO - step: 8415 loss: 17.0802 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.7150 global_avg_mtp_loss: 14.3651 +[titan] 2025-07-09 15:26:41,964 - root - INFO - lr: 2.9628e-04 gnorm: 1.46 [ 1:51:45<20:16:23] +[titan] 2025-07-09 15:26:45,855 - root - INFO - step: 8420 loss: 17.4182 memory: 44.58GiB(31.99%) tps: 84,216 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.7419 global_avg_mtp_loss: 14.6763 +[titan] 2025-07-09 15:26:45,856 - root - INFO - lr: 2.9627e-04 gnorm: 0.95 [ 1:51:49<20:16:18] +[titan] 2025-07-09 15:26:49,748 - root - INFO - step: 8425 loss: 17.5450 memory: 44.58GiB(31.99%) tps: 84,181 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 2.7809 global_avg_mtp_loss: 14.7640 +[titan] 2025-07-09 15:26:49,749 - root - INFO - lr: 2.9627e-04 gnorm: 1.02 [ 1:51:53<20:16:13] +[titan] 2025-07-09 15:26:53,651 - root - INFO - step: 8430 loss: 17.7436 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.8206 global_avg_mtp_loss: 14.9230 +[titan] 2025-07-09 15:26:53,652 - root - INFO - lr: 2.9626e-04 gnorm: 1.02 [ 1:51:57<20:16:08] +[titan] 2025-07-09 15:26:57,560 - root - INFO - step: 8435 loss: 17.1921 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.7018 global_avg_mtp_loss: 14.4904 +[titan] 2025-07-09 15:26:57,560 - root - INFO - lr: 2.9626e-04 gnorm: 0.89 [ 1:52:01<20:16:03] +[titan] 2025-07-09 15:27:01,485 - root - INFO - step: 8440 loss: 17.6502 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.7940 global_avg_mtp_loss: 14.8563 +[titan] 2025-07-09 15:27:01,485 - root - INFO - lr: 2.9625e-04 gnorm: 0.91 [ 1:52:05<20:15:59] +[titan] 2025-07-09 15:27:05,403 - root - INFO - step: 8445 loss: 17.5213 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7636 global_avg_mtp_loss: 14.7577 +[titan] 2025-07-09 15:27:05,403 - root - INFO - lr: 2.9625e-04 gnorm: 0.93 [ 1:52:09<20:15:54] +[titan] 2025-07-09 15:27:08,521 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:27:09,309 - root - INFO - step: 8450 loss: 17.5959 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.7867 global_avg_mtp_loss: 14.8092 +[titan] 2025-07-09 15:27:09,309 - root - INFO - lr: 2.9624e-04 gnorm: 0.97 [ 1:52:13<20:15:49] +[titan] 2025-07-09 15:27:13,223 - root - INFO - step: 8455 loss: 17.2115 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.7050 global_avg_mtp_loss: 14.5065 +[titan] 2025-07-09 15:27:13,224 - root - INFO - lr: 2.9624e-04 gnorm: 0.99 [ 1:52:17<20:15:44] +[titan] 2025-07-09 15:27:17,133 - root - INFO - step: 8460 loss: 17.6041 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7879 global_avg_mtp_loss: 14.8162 +[titan] 2025-07-09 15:27:17,134 - root - INFO - lr: 2.9623e-04 gnorm: 0.85 [ 1:52:21<20:15:40] +[titan] 2025-07-09 15:27:21,027 - root - INFO - step: 8465 loss: 17.5357 memory: 44.58GiB(31.99%) tps: 84,172 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.7632 global_avg_mtp_loss: 14.7725 +[titan] 2025-07-09 15:27:21,027 - root - INFO - lr: 2.9623e-04 gnorm: 0.90 [ 1:52:24<20:15:35] +[titan] 2025-07-09 15:27:24,944 - root - INFO - step: 8470 loss: 17.3025 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7360 global_avg_mtp_loss: 14.5666 +[titan] 2025-07-09 15:27:24,945 - root - INFO - lr: 2.9622e-04 gnorm: 0.95 [ 1:52:28<20:15:30] +[titan] 2025-07-09 15:27:28,902 - root - INFO - step: 8475 loss: 17.5580 memory: 44.58GiB(31.99%) tps: 82,801 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.7768 global_avg_mtp_loss: 14.7812 +[titan] 2025-07-09 15:27:28,902 - root - INFO - lr: 2.9622e-04 gnorm: 0.97 [ 1:52:32<20:15:26] +[titan] 2025-07-09 15:27:32,809 - root - INFO - step: 8480 loss: 17.6670 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7993 global_avg_mtp_loss: 14.8678 +[titan] 2025-07-09 15:27:32,809 - root - INFO - lr: 2.9621e-04 gnorm: 1.04 [ 1:52:36<20:15:21] +[titan] 2025-07-09 15:27:36,719 - root - INFO - step: 8485 loss: 17.0041 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6856 global_avg_mtp_loss: 14.3186 +[titan] 2025-07-09 15:27:36,719 - root - INFO - lr: 2.9621e-04 gnorm: 0.91 [ 1:52:40<20:15:16] +[titan] 2025-07-09 15:27:40,632 - root - INFO - step: 8490 loss: 17.4376 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7490 global_avg_mtp_loss: 14.6886 +[titan] 2025-07-09 15:27:40,632 - root - INFO - lr: 2.9620e-04 gnorm: 0.93 [ 1:52:44<20:15:11] +[titan] 2025-07-09 15:27:44,538 - root - INFO - step: 8495 loss: 17.5409 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.7694 global_avg_mtp_loss: 14.7714 +[titan] 2025-07-09 15:27:44,538 - root - INFO - lr: 2.9620e-04 gnorm: 0.94 [ 1:52:48<20:15:07] +[titan] 2025-07-09 15:27:47,652 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:27:48,446 - root - INFO - step: 8500 loss: 17.2653 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7241 global_avg_mtp_loss: 14.5413 +[titan] 2025-07-09 15:27:48,447 - root - INFO - lr: 2.9619e-04 gnorm: 0.98 [ 1:52:52<20:15:02] +[titan] 2025-07-09 15:27:52,374 - root - INFO - step: 8505 loss: 17.7058 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.8070 global_avg_mtp_loss: 14.8988 +[titan] 2025-07-09 15:27:52,375 - root - INFO - lr: 2.9619e-04 gnorm: 0.92 [ 1:52:56<20:14:57] +[titan] 2025-07-09 15:27:56,289 - root - INFO - step: 8510 loss: 17.5586 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.7762 global_avg_mtp_loss: 14.7824 +[titan] 2025-07-09 15:27:56,289 - root - INFO - lr: 2.9618e-04 gnorm: 0.94 [ 1:53:00<20:14:52] +[titan] 2025-07-09 15:28:00,205 - root - INFO - step: 8515 loss: 17.1682 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6956 global_avg_mtp_loss: 14.4726 +[titan] 2025-07-09 15:28:00,205 - root - INFO - lr: 2.9618e-04 gnorm: 0.94 [ 1:53:04<20:14:48] +[titan] 2025-07-09 15:28:04,108 - root - INFO - step: 8520 loss: 17.7638 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.8331 global_avg_mtp_loss: 14.9307 +[titan] 2025-07-09 15:28:04,109 - root - INFO - lr: 2.9617e-04 gnorm: 0.95 [ 1:53:08<20:14:43] +[titan] 2025-07-09 15:28:08,017 - root - INFO - step: 8525 loss: 17.3125 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.7332 global_avg_mtp_loss: 14.5794 +[titan] 2025-07-09 15:28:08,017 - root - INFO - lr: 2.9617e-04 gnorm: 0.97 [ 1:53:11<20:14:38] +[titan] 2025-07-09 15:28:11,974 - root - INFO - step: 8530 loss: 17.7837 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.8160 global_avg_mtp_loss: 14.9678 +[titan] 2025-07-09 15:28:11,974 - root - INFO - lr: 2.9616e-04 gnorm: 0.88 [ 1:53:15<20:14:34] +[titan] 2025-07-09 15:28:15,927 - root - INFO - step: 8535 loss: 17.4877 memory: 44.58GiB(31.99%) tps: 82,893 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.7615 global_avg_mtp_loss: 14.7262 +[titan] 2025-07-09 15:28:15,927 - root - INFO - lr: 2.9616e-04 gnorm: 0.95 [ 1:53:19<20:14:29] +[titan] 2025-07-09 15:28:19,839 - root - INFO - step: 8540 loss: 17.6283 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.7828 global_avg_mtp_loss: 14.8455 +[titan] 2025-07-09 15:28:19,840 - root - INFO - lr: 2.9615e-04 gnorm: 1.01 [ 1:53:23<20:14:25] +[titan] 2025-07-09 15:28:23,751 - root - INFO - step: 8545 loss: 17.2053 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.7088 global_avg_mtp_loss: 14.4965 +[titan] 2025-07-09 15:28:23,752 - root - INFO - lr: 2.9615e-04 gnorm: 0.89 [ 1:53:27<20:14:20] +[titan] 2025-07-09 15:28:26,855 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:28:27,642 - root - INFO - step: 8550 loss: 17.6913 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.7962 global_avg_mtp_loss: 14.8951 +[titan] 2025-07-09 15:28:27,643 - root - INFO - lr: 2.9614e-04 gnorm: 0.92 [ 1:53:31<20:14:15] +[titan] 2025-07-09 15:28:31,547 - root - INFO - step: 8555 loss: 17.5415 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.7962 global_avg_mtp_loss: 14.7453 +[titan] 2025-07-09 15:28:31,548 - root - INFO - lr: 2.9614e-04 gnorm: 0.88 [ 1:53:35<20:14:10] +[titan] 2025-07-09 15:28:35,460 - root - INFO - step: 8560 loss: 17.5239 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.7732 global_avg_mtp_loss: 14.7508 +[titan] 2025-07-09 15:28:35,460 - root - INFO - lr: 2.9613e-04 gnorm: 0.94 [ 1:53:39<20:14:05] +[titan] 2025-07-09 15:28:39,374 - root - INFO - step: 8565 loss: 17.4109 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.7364 global_avg_mtp_loss: 14.6746 +[titan] 2025-07-09 15:28:39,375 - root - INFO - lr: 2.9613e-04 gnorm: 0.90 [ 1:53:43<20:14:01] +[titan] 2025-07-09 15:28:43,301 - root - INFO - step: 8570 loss: 17.3884 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.7346 global_avg_mtp_loss: 14.6538 +[titan] 2025-07-09 15:28:43,301 - root - INFO - lr: 2.9612e-04 gnorm: 0.94 [ 1:53:47<20:13:56] +[titan] 2025-07-09 15:28:47,217 - root - INFO - step: 8575 loss: 17.3692 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7370 global_avg_mtp_loss: 14.6322 +[titan] 2025-07-09 15:28:47,218 - root - INFO - lr: 2.9612e-04 gnorm: 0.92 [ 1:53:51<20:13:51] +[titan] 2025-07-09 15:28:51,146 - root - INFO - step: 8580 loss: 17.6765 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.8110 global_avg_mtp_loss: 14.8656 +[titan] 2025-07-09 15:28:51,146 - root - INFO - lr: 2.9611e-04 gnorm: 0.97 [ 1:53:55<20:13:47] +[titan] 2025-07-09 15:28:55,046 - root - INFO - step: 8585 loss: 16.9688 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6988 global_avg_mtp_loss: 14.2699 +[titan] 2025-07-09 15:28:55,046 - root - INFO - lr: 2.9611e-04 gnorm: 0.98 [ 1:53:58<20:13:42] +[titan] 2025-07-09 15:28:58,954 - root - INFO - step: 8590 loss: 17.6343 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.7909 global_avg_mtp_loss: 14.8434 +[titan] 2025-07-09 15:28:58,954 - root - INFO - lr: 2.9610e-04 gnorm: 0.98 [ 1:54:02<20:13:37] +[titan] 2025-07-09 15:29:02,876 - root - INFO - step: 8595 loss: 17.6424 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.7829 global_avg_mtp_loss: 14.8595 +[titan] 2025-07-09 15:29:02,876 - root - INFO - lr: 2.9610e-04 gnorm: 0.96 [ 1:54:06<20:13:33] +[titan] 2025-07-09 15:29:06,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:29:06,783 - root - INFO - step: 8600 loss: 17.5595 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7731 global_avg_mtp_loss: 14.7865 +[titan] 2025-07-09 15:29:06,783 - root - INFO - lr: 2.9609e-04 gnorm: 0.94 [ 1:54:10<20:13:28] +[titan] 2025-07-09 15:29:10,697 - root - INFO - step: 8605 loss: 17.4072 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.7489 global_avg_mtp_loss: 14.6583 +[titan] 2025-07-09 15:29:10,697 - root - INFO - lr: 2.9609e-04 gnorm: 0.90 [ 1:54:14<20:13:23] +[titan] 2025-07-09 15:29:14,621 - root - INFO - step: 8610 loss: 17.3086 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.7299 global_avg_mtp_loss: 14.5787 +[titan] 2025-07-09 15:29:14,621 - root - INFO - lr: 2.9608e-04 gnorm: 0.91 [ 1:54:18<20:13:18] +[titan] 2025-07-09 15:29:18,532 - root - INFO - step: 8615 loss: 17.4538 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.7589 global_avg_mtp_loss: 14.6949 +[titan] 2025-07-09 15:29:18,532 - root - INFO - lr: 2.9608e-04 gnorm: 0.90 [ 1:54:22<20:13:14] +[titan] 2025-07-09 15:29:22,460 - root - INFO - step: 8620 loss: 17.5546 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.8072 global_avg_mtp_loss: 14.7474 +[titan] 2025-07-09 15:29:22,460 - root - INFO - lr: 2.9607e-04 gnorm: 1.03 [ 1:54:26<20:13:09] +[titan] 2025-07-09 15:29:26,369 - root - INFO - step: 8625 loss: 17.1841 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.7118 global_avg_mtp_loss: 14.4723 +[titan] 2025-07-09 15:29:26,369 - root - INFO - lr: 2.9607e-04 gnorm: 1.00 [ 1:54:30<20:13:04] +[titan] 2025-07-09 15:29:30,270 - root - INFO - step: 8630 loss: 17.5673 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.7643 global_avg_mtp_loss: 14.8030 +[titan] 2025-07-09 15:29:30,270 - root - INFO - lr: 2.9606e-04 gnorm: 0.93 [ 1:54:34<20:12:59] +[titan] 2025-07-09 15:29:34,189 - root - INFO - step: 8635 loss: 17.4789 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.7610 global_avg_mtp_loss: 14.7178 +[titan] 2025-07-09 15:29:34,189 - root - INFO - lr: 2.9606e-04 gnorm: 0.97 [ 1:54:38<20:12:55] +[titan] 2025-07-09 15:29:38,129 - root - INFO - step: 8640 loss: 17.7559 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.8102 global_avg_mtp_loss: 14.9457 +[titan] 2025-07-09 15:29:38,130 - root - INFO - lr: 2.9605e-04 gnorm: 0.97 [ 1:54:42<20:12:50] +[titan] 2025-07-09 15:29:42,040 - root - INFO - step: 8645 loss: 17.6127 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.7813 global_avg_mtp_loss: 14.8314 +[titan] 2025-07-09 15:29:42,040 - root - INFO - lr: 2.9605e-04 gnorm: 0.89 [ 1:54:45<20:12:46] +[titan] 2025-07-09 15:29:45,170 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:29:45,959 - root - INFO - step: 8650 loss: 17.5803 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.7715 global_avg_mtp_loss: 14.8088 +[titan] 2025-07-09 15:29:45,959 - root - INFO - lr: 2.9604e-04 gnorm: 0.95 [ 1:54:49<20:12:41] +[titan] 2025-07-09 15:29:49,927 - root - INFO - step: 8655 loss: 17.6138 memory: 44.58GiB(31.99%) tps: 82,584 tflops: 285.01 mfu: 28.82% global_avg_ntp_loss: 2.7899 global_avg_mtp_loss: 14.8239 +[titan] 2025-07-09 15:29:49,927 - root - INFO - lr: 2.9604e-04 gnorm: 0.99 [ 1:54:53<20:12:37] +[titan] 2025-07-09 15:29:53,839 - root - INFO - step: 8660 loss: 17.4475 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.7538 global_avg_mtp_loss: 14.6938 +[titan] 2025-07-09 15:29:53,840 - root - INFO - lr: 2.9603e-04 gnorm: 1.01 [ 1:54:57<20:12:32] +[titan] 2025-07-09 15:29:57,752 - root - INFO - step: 8665 loss: 17.6551 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7898 global_avg_mtp_loss: 14.8653 +[titan] 2025-07-09 15:29:57,753 - root - INFO - lr: 2.9603e-04 gnorm: 0.92 [ 1:55:01<20:12:27] +[titan] 2025-07-09 15:30:01,657 - root - INFO - step: 8670 loss: 17.5239 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.7741 global_avg_mtp_loss: 14.7497 +[titan] 2025-07-09 15:30:01,657 - root - INFO - lr: 2.9602e-04 gnorm: 0.91 [ 1:55:05<20:12:23] +[titan] 2025-07-09 15:30:05,566 - root - INFO - step: 8675 loss: 17.5759 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.7775 global_avg_mtp_loss: 14.7984 +[titan] 2025-07-09 15:30:05,566 - root - INFO - lr: 2.9601e-04 gnorm: 0.90 [ 1:55:09<20:12:18] +[titan] 2025-07-09 15:30:09,469 - root - INFO - step: 8680 loss: 17.4303 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7612 global_avg_mtp_loss: 14.6690 +[titan] 2025-07-09 15:30:09,470 - root - INFO - lr: 2.9601e-04 gnorm: 0.98 [ 1:55:13<20:12:13] +[titan] 2025-07-09 15:30:13,377 - root - INFO - step: 8685 loss: 17.5596 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7880 global_avg_mtp_loss: 14.7716 +[titan] 2025-07-09 15:30:13,377 - root - INFO - lr: 2.9600e-04 gnorm: 0.98 [ 1:55:17<20:12:08] +[titan] 2025-07-09 15:30:17,311 - root - INFO - step: 8690 loss: 17.5627 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.7789 global_avg_mtp_loss: 14.7838 +[titan] 2025-07-09 15:30:17,312 - root - INFO - lr: 2.9600e-04 gnorm: 0.95 [ 1:55:21<20:12:04] +[titan] 2025-07-09 15:30:21,227 - root - INFO - step: 8695 loss: 17.1350 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.4436 +[titan] 2025-07-09 15:30:21,228 - root - INFO - lr: 2.9599e-04 gnorm: 0.89 [ 1:55:25<20:11:59] +[titan] 2025-07-09 15:30:24,375 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:30:25,160 - root - INFO - step: 8700 loss: 17.4251 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.7658 global_avg_mtp_loss: 14.6593 +[titan] 2025-07-09 15:30:25,160 - root - INFO - lr: 2.9599e-04 gnorm: 0.92 [ 1:55:29<20:11:55] +[titan] 2025-07-09 15:30:28,513 - root - INFO - Dumping profiler traces at step 8704 +[titan] 2025-07-09 15:30:28,545 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:30:29,330 - root - INFO - step: 8705 loss: 17.3241 memory: 44.58GiB(31.99%) tps: 78,578 tflops: 271.18 mfu: 27.42% global_avg_ntp_loss: 2.7396 global_avg_mtp_loss: 14.5845 +[titan] 2025-07-09 15:30:29,331 - root - INFO - lr: 2.9598e-04 gnorm: 1.01 [ 1:55:33<20:11:53] +[titan] 2025-07-09 15:30:33,254 - root - INFO - step: 8710 loss: 17.4542 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.7490 global_avg_mtp_loss: 14.7052 +[titan] 2025-07-09 15:30:33,254 - root - INFO - lr: 2.9598e-04 gnorm: 0.99 [ 1:55:37<20:11:48] +[titan] 2025-07-09 15:30:37,208 - root - INFO - step: 8715 loss: 17.5899 memory: 44.58GiB(31.99%) tps: 82,872 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.7787 global_avg_mtp_loss: 14.8112 +[titan] 2025-07-09 15:30:37,209 - root - INFO - lr: 2.9597e-04 gnorm: 0.92 [ 1:55:41<20:11:44] +[titan] 2025-07-09 15:30:41,100 - root - INFO - step: 8720 loss: 17.2712 memory: 44.58GiB(31.99%) tps: 84,211 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.7204 global_avg_mtp_loss: 14.5509 +[titan] 2025-07-09 15:30:41,100 - root - INFO - lr: 2.9597e-04 gnorm: 0.97 [ 1:55:44<20:11:39] +[titan] 2025-07-09 15:30:44,998 - root - INFO - step: 8725 loss: 17.4415 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.7573 global_avg_mtp_loss: 14.6843 +[titan] 2025-07-09 15:30:44,998 - root - INFO - lr: 2.9596e-04 gnorm: 0.95 [ 1:55:48<20:11:34] +[titan] 2025-07-09 15:30:48,893 - root - INFO - step: 8730 loss: 17.4166 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7522 global_avg_mtp_loss: 14.6643 +[titan] 2025-07-09 15:30:48,893 - root - INFO - lr: 2.9596e-04 gnorm: 1.09 [ 1:55:52<20:11:29] +[titan] 2025-07-09 15:30:52,791 - root - INFO - step: 8735 loss: 17.3110 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.7275 global_avg_mtp_loss: 14.5835 +[titan] 2025-07-09 15:30:52,791 - root - INFO - lr: 2.9595e-04 gnorm: 1.03 [ 1:55:56<20:11:24] +[titan] 2025-07-09 15:30:56,686 - root - INFO - step: 8740 loss: 17.7395 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.8075 global_avg_mtp_loss: 14.9321 +[titan] 2025-07-09 15:30:56,687 - root - INFO - lr: 2.9595e-04 gnorm: 1.10 [ 1:56:00<20:11:19] +[titan] 2025-07-09 15:31:00,598 - root - INFO - step: 8745 loss: 17.4057 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.7480 global_avg_mtp_loss: 14.6576 +[titan] 2025-07-09 15:31:00,599 - root - INFO - lr: 2.9594e-04 gnorm: 0.95 [ 1:56:04<20:11:14] +[titan] 2025-07-09 15:31:03,711 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:31:04,506 - root - INFO - step: 8750 loss: 17.1596 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.7038 global_avg_mtp_loss: 14.4558 +[titan] 2025-07-09 15:31:04,506 - root - INFO - lr: 2.9594e-04 gnorm: 0.97 [ 1:56:08<20:11:10] +[titan] 2025-07-09 15:31:08,426 - root - INFO - step: 8755 loss: 17.4057 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.7389 global_avg_mtp_loss: 14.6667 +[titan] 2025-07-09 15:31:08,427 - root - INFO - lr: 2.9593e-04 gnorm: 0.99 [ 1:56:12<20:11:05] +[titan] 2025-07-09 15:31:12,355 - root - INFO - step: 8760 loss: 17.3302 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.7232 global_avg_mtp_loss: 14.6070 +[titan] 2025-07-09 15:31:12,355 - root - INFO - lr: 2.9593e-04 gnorm: 0.91 [ 1:56:16<20:11:01] +[titan] 2025-07-09 15:31:16,254 - root - INFO - step: 8765 loss: 17.6123 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.7813 global_avg_mtp_loss: 14.8310 +[titan] 2025-07-09 15:31:16,255 - root - INFO - lr: 2.9592e-04 gnorm: 0.87 [ 1:56:20<20:10:56] +[titan] 2025-07-09 15:31:20,202 - root - INFO - step: 8770 loss: 17.5431 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.7744 global_avg_mtp_loss: 14.7687 +[titan] 2025-07-09 15:31:20,202 - root - INFO - lr: 2.9592e-04 gnorm: 1.03 [ 1:56:24<20:10:51] +[titan] 2025-07-09 15:31:24,123 - root - INFO - step: 8775 loss: 17.2745 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.7278 global_avg_mtp_loss: 14.5466 +[titan] 2025-07-09 15:31:24,124 - root - INFO - lr: 2.9591e-04 gnorm: 0.96 [ 1:56:27<20:10:47] +[titan] 2025-07-09 15:31:28,080 - root - INFO - step: 8780 loss: 17.4940 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.7486 global_avg_mtp_loss: 14.7454 +[titan] 2025-07-09 15:31:28,080 - root - INFO - lr: 2.9591e-04 gnorm: 0.89 [ 1:56:31<20:10:42] +[titan] 2025-07-09 15:31:32,004 - root - INFO - step: 8785 loss: 17.4863 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.7665 global_avg_mtp_loss: 14.7198 +[titan] 2025-07-09 15:31:32,005 - root - INFO - lr: 2.9590e-04 gnorm: 0.88 [ 1:56:35<20:10:38] +[titan] 2025-07-09 15:31:35,906 - root - INFO - step: 8790 loss: 17.5908 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7866 global_avg_mtp_loss: 14.8041 +[titan] 2025-07-09 15:31:35,907 - root - INFO - lr: 2.9590e-04 gnorm: 0.92 [ 1:56:39<20:10:33] +[titan] 2025-07-09 15:31:39,797 - root - INFO - step: 8795 loss: 17.5332 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.7690 global_avg_mtp_loss: 14.7642 +[titan] 2025-07-09 15:31:39,798 - root - INFO - lr: 2.9589e-04 gnorm: 0.90 [ 1:56:43<20:10:28] +[titan] 2025-07-09 15:31:42,924 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:31:43,715 - root - INFO - step: 8800 loss: 17.4342 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.7520 global_avg_mtp_loss: 14.6822 +[titan] 2025-07-09 15:31:43,715 - root - INFO - lr: 2.9588e-04 gnorm: 0.94 [ 1:56:47<20:10:24] +[titan] 2025-07-09 15:31:47,623 - root - INFO - step: 8805 loss: 17.5510 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.7848 global_avg_mtp_loss: 14.7662 +[titan] 2025-07-09 15:31:47,623 - root - INFO - lr: 2.9588e-04 gnorm: 1.02 [ 1:56:51<20:10:19] +[titan] 2025-07-09 15:31:51,533 - root - INFO - step: 8810 loss: 17.2420 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.7352 global_avg_mtp_loss: 14.5068 +[titan] 2025-07-09 15:31:51,533 - root - INFO - lr: 2.9587e-04 gnorm: 1.00 [ 1:56:55<20:10:14] +[titan] 2025-07-09 15:31:55,469 - root - INFO - step: 8815 loss: 17.5519 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7680 global_avg_mtp_loss: 14.7840 +[titan] 2025-07-09 15:31:55,469 - root - INFO - lr: 2.9587e-04 gnorm: 0.93 [ 1:56:59<20:10:10] +[titan] 2025-07-09 15:31:59,381 - root - INFO - step: 8820 loss: 17.5302 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.7679 global_avg_mtp_loss: 14.7623 +[titan] 2025-07-09 15:31:59,382 - root - INFO - lr: 2.9586e-04 gnorm: 0.92 [ 1:57:03<20:10:05] +[titan] 2025-07-09 15:32:03,271 - root - INFO - step: 8825 loss: 17.3953 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.7468 global_avg_mtp_loss: 14.6485 +[titan] 2025-07-09 15:32:03,271 - root - INFO - lr: 2.9586e-04 gnorm: 0.95 [ 1:57:07<20:10:00] +[titan] 2025-07-09 15:32:07,184 - root - INFO - step: 8830 loss: 17.4666 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.7780 global_avg_mtp_loss: 14.6886 +[titan] 2025-07-09 15:32:07,184 - root - INFO - lr: 2.9585e-04 gnorm: 0.93 [ 1:57:11<20:09:55] +[titan] 2025-07-09 15:32:11,079 - root - INFO - step: 8835 loss: 17.5790 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.7727 global_avg_mtp_loss: 14.8063 +[titan] 2025-07-09 15:32:11,079 - root - INFO - lr: 2.9585e-04 gnorm: 0.87 [ 1:57:14<20:09:50] +[titan] 2025-07-09 15:32:15,009 - root - INFO - step: 8840 loss: 17.4647 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.7508 global_avg_mtp_loss: 14.7138 +[titan] 2025-07-09 15:32:15,009 - root - INFO - lr: 2.9584e-04 gnorm: 0.94 [ 1:57:18<20:09:46] +[titan] 2025-07-09 15:32:18,923 - root - INFO - step: 8845 loss: 17.2409 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.7284 global_avg_mtp_loss: 14.5124 +[titan] 2025-07-09 15:32:18,924 - root - INFO - lr: 2.9584e-04 gnorm: 0.82 [ 1:57:22<20:09:41] +[titan] 2025-07-09 15:32:22,058 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:32:22,856 - root - INFO - step: 8850 loss: 17.6307 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.7795 global_avg_mtp_loss: 14.8512 +[titan] 2025-07-09 15:32:22,856 - root - INFO - lr: 2.9583e-04 gnorm: 1.02 [ 1:57:26<20:09:37] +[titan] 2025-07-09 15:32:26,772 - root - INFO - step: 8855 loss: 17.4338 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.7646 global_avg_mtp_loss: 14.6692 +[titan] 2025-07-09 15:32:26,772 - root - INFO - lr: 2.9583e-04 gnorm: 0.90 [ 1:57:30<20:09:32] +[titan] 2025-07-09 15:32:30,686 - root - INFO - step: 8860 loss: 17.4329 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.7477 global_avg_mtp_loss: 14.6853 +[titan] 2025-07-09 15:32:30,686 - root - INFO - lr: 2.9582e-04 gnorm: 0.92 [ 1:57:34<20:09:27] +[titan] 2025-07-09 15:32:34,575 - root - INFO - step: 8865 loss: 17.4252 memory: 44.58GiB(31.99%) tps: 84,262 tflops: 290.80 mfu: 29.40% global_avg_ntp_loss: 2.7478 global_avg_mtp_loss: 14.6775 +[titan] 2025-07-09 15:32:34,575 - root - INFO - lr: 2.9582e-04 gnorm: 0.97 [ 1:57:38<20:09:22] +[titan] 2025-07-09 15:32:38,481 - root - INFO - step: 8870 loss: 17.4218 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.7442 global_avg_mtp_loss: 14.6776 +[titan] 2025-07-09 15:32:38,481 - root - INFO - lr: 2.9581e-04 gnorm: 0.88 [ 1:57:42<20:09:18] +[titan] 2025-07-09 15:32:42,375 - root - INFO - step: 8875 loss: 17.5448 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.7580 global_avg_mtp_loss: 14.7868 +[titan] 2025-07-09 15:32:42,375 - root - INFO - lr: 2.9581e-04 gnorm: 0.92 [ 1:57:46<20:09:13] +[titan] 2025-07-09 15:32:46,259 - root - INFO - step: 8880 loss: 17.7319 memory: 44.58GiB(31.99%) tps: 84,360 tflops: 291.14 mfu: 29.44% global_avg_ntp_loss: 2.8181 global_avg_mtp_loss: 14.9137 +[titan] 2025-07-09 15:32:46,260 - root - INFO - lr: 2.9580e-04 gnorm: 1.00 [ 1:57:50<20:09:08] +[titan] 2025-07-09 15:32:50,169 - root - INFO - step: 8885 loss: 17.6380 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7971 global_avg_mtp_loss: 14.8410 +[titan] 2025-07-09 15:32:50,169 - root - INFO - lr: 2.9579e-04 gnorm: 0.90 [ 1:57:54<20:09:03] +[titan] 2025-07-09 15:32:54,107 - root - INFO - step: 8890 loss: 17.4542 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.7554 global_avg_mtp_loss: 14.6988 +[titan] 2025-07-09 15:32:54,107 - root - INFO - lr: 2.9579e-04 gnorm: 0.92 [ 1:57:57<20:08:59] +[titan] 2025-07-09 15:32:58,008 - root - INFO - step: 8895 loss: 17.3239 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.7400 global_avg_mtp_loss: 14.5839 +[titan] 2025-07-09 15:32:58,008 - root - INFO - lr: 2.9578e-04 gnorm: 0.95 [ 1:58:01<20:08:54] +[titan] 2025-07-09 15:33:01,120 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:33:01,913 - root - INFO - step: 8900 loss: 17.2461 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.7101 global_avg_mtp_loss: 14.5360 +[titan] 2025-07-09 15:33:01,913 - root - INFO - lr: 2.9578e-04 gnorm: 0.94 [ 1:58:05<20:08:49] +[titan] 2025-07-09 15:33:05,817 - root - INFO - step: 8905 loss: 17.3793 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.7373 global_avg_mtp_loss: 14.6421 +[titan] 2025-07-09 15:33:05,817 - root - INFO - lr: 2.9577e-04 gnorm: 0.87 [ 1:58:09<20:08:44] +[titan] 2025-07-09 15:33:09,740 - root - INFO - step: 8910 loss: 17.2891 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.7201 global_avg_mtp_loss: 14.5690 +[titan] 2025-07-09 15:33:09,740 - root - INFO - lr: 2.9577e-04 gnorm: 0.97 [ 1:58:13<20:08:40] +[titan] 2025-07-09 15:33:13,674 - root - INFO - step: 8915 loss: 17.2892 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.7248 global_avg_mtp_loss: 14.5645 +[titan] 2025-07-09 15:33:13,674 - root - INFO - lr: 2.9576e-04 gnorm: 0.86 [ 1:58:17<20:08:35] +[titan] 2025-07-09 15:33:17,597 - root - INFO - step: 8920 loss: 17.1876 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.7063 global_avg_mtp_loss: 14.4813 +[titan] 2025-07-09 15:33:17,598 - root - INFO - lr: 2.9576e-04 gnorm: 1.03 [ 1:58:21<20:08:31] +[titan] 2025-07-09 15:33:21,486 - root - INFO - step: 8925 loss: 17.5408 memory: 44.58GiB(31.99%) tps: 84,271 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.7789 global_avg_mtp_loss: 14.7619 +[titan] 2025-07-09 15:33:21,486 - root - INFO - lr: 2.9575e-04 gnorm: 0.96 [ 1:58:25<20:08:26] +[titan] 2025-07-09 15:33:25,436 - root - INFO - step: 8930 loss: 17.4075 memory: 44.58GiB(31.99%) tps: 82,975 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.7780 global_avg_mtp_loss: 14.6295 +[titan] 2025-07-09 15:33:25,436 - root - INFO - lr: 2.9575e-04 gnorm: 0.93 [ 1:58:29<20:08:22] +[titan] 2025-07-09 15:33:29,374 - root - INFO - step: 8935 loss: 17.5171 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.7613 global_avg_mtp_loss: 14.7558 +[titan] 2025-07-09 15:33:29,374 - root - INFO - lr: 2.9574e-04 gnorm: 0.86 [ 1:58:33<20:08:17] +[titan] 2025-07-09 15:33:33,347 - root - INFO - step: 8940 loss: 17.3058 memory: 44.58GiB(31.99%) tps: 82,489 tflops: 284.68 mfu: 28.78% global_avg_ntp_loss: 2.7341 global_avg_mtp_loss: 14.5717 +[titan] 2025-07-09 15:33:33,347 - root - INFO - lr: 2.9574e-04 gnorm: 0.89 [ 1:58:37<20:08:13] +[titan] 2025-07-09 15:33:37,252 - root - INFO - step: 8945 loss: 17.5277 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7644 global_avg_mtp_loss: 14.7632 +[titan] 2025-07-09 15:33:37,252 - root - INFO - lr: 2.9573e-04 gnorm: 0.92 [ 1:58:41<20:08:08] +[titan] 2025-07-09 15:33:40,378 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:33:41,170 - root - INFO - step: 8950 loss: 17.4651 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7664 global_avg_mtp_loss: 14.6986 +[titan] 2025-07-09 15:33:41,170 - root - INFO - lr: 2.9573e-04 gnorm: 0.97 [ 1:58:45<20:08:04] +[titan] 2025-07-09 15:33:45,094 - root - INFO - step: 8955 loss: 17.7136 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.8055 global_avg_mtp_loss: 14.9082 +[titan] 2025-07-09 15:33:45,094 - root - INFO - lr: 2.9572e-04 gnorm: 1.01 [ 1:58:48<20:07:59] +[titan] 2025-07-09 15:33:49,016 - root - INFO - step: 8960 loss: 17.4222 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.7602 global_avg_mtp_loss: 14.6620 +[titan] 2025-07-09 15:33:49,016 - root - INFO - lr: 2.9572e-04 gnorm: 0.92 [ 1:58:52<20:07:55] +[titan] 2025-07-09 15:33:52,934 - root - INFO - step: 8965 loss: 17.3836 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.7500 global_avg_mtp_loss: 14.6336 +[titan] 2025-07-09 15:33:52,934 - root - INFO - lr: 2.9571e-04 gnorm: 0.86 [ 1:58:56<20:07:50] +[titan] 2025-07-09 15:33:56,863 - root - INFO - step: 8970 loss: 17.2469 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.7062 global_avg_mtp_loss: 14.5407 +[titan] 2025-07-09 15:33:56,863 - root - INFO - lr: 2.9570e-04 gnorm: 0.92 [ 1:59:00<20:07:45] +[titan] 2025-07-09 15:34:00,760 - root - INFO - step: 8975 loss: 17.4670 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.7419 global_avg_mtp_loss: 14.7250 +[titan] 2025-07-09 15:34:00,760 - root - INFO - lr: 2.9570e-04 gnorm: 0.92 [ 1:59:04<20:07:41] +[titan] 2025-07-09 15:34:04,695 - root - INFO - step: 8980 loss: 17.4733 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.7688 global_avg_mtp_loss: 14.7045 +[titan] 2025-07-09 15:34:04,695 - root - INFO - lr: 2.9569e-04 gnorm: 0.94 [ 1:59:08<20:07:36] +[titan] 2025-07-09 15:34:08,614 - root - INFO - step: 8985 loss: 17.5220 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.7618 global_avg_mtp_loss: 14.7602 +[titan] 2025-07-09 15:34:08,615 - root - INFO - lr: 2.9569e-04 gnorm: 0.89 [ 1:59:12<20:07:32] +[titan] 2025-07-09 15:34:12,551 - root - INFO - step: 8990 loss: 17.3547 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7352 global_avg_mtp_loss: 14.6195 +[titan] 2025-07-09 15:34:12,551 - root - INFO - lr: 2.9568e-04 gnorm: 1.00 [ 1:59:16<20:07:27] +[titan] 2025-07-09 15:34:16,485 - root - INFO - step: 8995 loss: 17.1384 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.7068 global_avg_mtp_loss: 14.4317 +[titan] 2025-07-09 15:34:16,485 - root - INFO - lr: 2.9568e-04 gnorm: 0.90 [ 1:59:20<20:07:23] +[titan] 2025-07-09 15:34:19,598 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:34:20,396 - root - INFO - step: 9000 loss: 17.3392 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.7310 global_avg_mtp_loss: 14.6082 +[titan] 2025-07-09 15:34:20,396 - root - INFO - lr: 2.9567e-04 gnorm: 0.92 [ 1:59:24<20:07:18] +[titan] 2025-07-09 15:34:24,310 - root - INFO - step: 9005 loss: 17.6096 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.7881 global_avg_mtp_loss: 14.8216 +[titan] 2025-07-09 15:34:24,311 - root - INFO - lr: 2.9567e-04 gnorm: 0.87 [ 1:59:28<20:07:13] +[titan] 2025-07-09 15:34:28,223 - root - INFO - step: 9010 loss: 17.5115 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.7638 global_avg_mtp_loss: 14.7478 +[titan] 2025-07-09 15:34:28,223 - root - INFO - lr: 2.9566e-04 gnorm: 1.03 [ 1:59:32<20:07:09] +[titan] 2025-07-09 15:34:32,138 - root - INFO - step: 9015 loss: 17.4770 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.7468 global_avg_mtp_loss: 14.7302 +[titan] 2025-07-09 15:34:32,138 - root - INFO - lr: 2.9566e-04 gnorm: 0.94 [ 1:59:35<20:07:04] +[titan] 2025-07-09 15:34:36,055 - root - INFO - step: 9020 loss: 17.6308 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.7948 global_avg_mtp_loss: 14.8360 +[titan] 2025-07-09 15:34:36,055 - root - INFO - lr: 2.9565e-04 gnorm: 0.93 [ 1:59:39<20:06:59] +[titan] 2025-07-09 15:34:39,946 - root - INFO - step: 9025 loss: 17.4011 memory: 44.58GiB(31.99%) tps: 84,230 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.7505 global_avg_mtp_loss: 14.6506 +[titan] 2025-07-09 15:34:39,946 - root - INFO - lr: 2.9565e-04 gnorm: 0.92 [ 1:59:43<20:06:54] +[titan] 2025-07-09 15:34:43,866 - root - INFO - step: 9030 loss: 17.5402 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.7738 global_avg_mtp_loss: 14.7665 +[titan] 2025-07-09 15:34:43,866 - root - INFO - lr: 2.9564e-04 gnorm: 1.01 [ 1:59:47<20:06:50] +[titan] 2025-07-09 15:34:47,755 - root - INFO - step: 9035 loss: 17.4992 memory: 44.58GiB(31.99%) tps: 84,264 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.7769 global_avg_mtp_loss: 14.7224 +[titan] 2025-07-09 15:34:47,755 - root - INFO - lr: 2.9563e-04 gnorm: 0.94 [ 1:59:51<20:06:45] +[titan] 2025-07-09 15:34:51,686 - root - INFO - step: 9040 loss: 17.2729 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.7257 global_avg_mtp_loss: 14.5472 +[titan] 2025-07-09 15:34:51,687 - root - INFO - lr: 2.9563e-04 gnorm: 0.93 [ 1:59:55<20:06:41] +[titan] 2025-07-09 15:34:55,639 - root - INFO - step: 9045 loss: 17.6952 memory: 44.58GiB(31.99%) tps: 82,911 tflops: 286.14 mfu: 28.93% global_avg_ntp_loss: 2.7933 global_avg_mtp_loss: 14.9019 +[titan] 2025-07-09 15:34:55,639 - root - INFO - lr: 2.9562e-04 gnorm: 0.86 [ 1:59:59<20:06:36] +[titan] 2025-07-09 15:34:58,740 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:34:59,530 - root - INFO - step: 9050 loss: 17.6508 memory: 44.58GiB(31.99%) tps: 84,212 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.7978 global_avg_mtp_loss: 14.8530 +[titan] 2025-07-09 15:34:59,531 - root - INFO - lr: 2.9562e-04 gnorm: 0.94 [ 2:00:03<20:06:31] +[titan] 2025-07-09 15:35:03,417 - root - INFO - step: 9055 loss: 17.4121 memory: 44.58GiB(31.99%) tps: 84,320 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.7401 global_avg_mtp_loss: 14.6720 +[titan] 2025-07-09 15:35:03,417 - root - INFO - lr: 2.9561e-04 gnorm: 1.08 [ 2:00:07<20:06:26] +[titan] 2025-07-09 15:35:07,365 - root - INFO - step: 9060 loss: 17.2387 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.7101 global_avg_mtp_loss: 14.5286 +[titan] 2025-07-09 15:35:07,366 - root - INFO - lr: 2.9561e-04 gnorm: 0.94 [ 2:00:11<20:06:22] +[titan] 2025-07-09 15:35:11,286 - root - INFO - step: 9065 loss: 17.6272 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.7945 global_avg_mtp_loss: 14.8327 +[titan] 2025-07-09 15:35:11,286 - root - INFO - lr: 2.9560e-04 gnorm: 0.97 [ 2:00:15<20:06:18] +[titan] 2025-07-09 15:35:15,203 - root - INFO - step: 9070 loss: 17.6641 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.8017 global_avg_mtp_loss: 14.8625 +[titan] 2025-07-09 15:35:15,203 - root - INFO - lr: 2.9560e-04 gnorm: 0.91 [ 2:00:19<20:06:13] +[titan] 2025-07-09 15:35:19,102 - root - INFO - step: 9075 loss: 17.4377 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.7678 global_avg_mtp_loss: 14.6699 +[titan] 2025-07-09 15:35:19,102 - root - INFO - lr: 2.9559e-04 gnorm: 0.93 [ 2:00:22<20:06:08] +[titan] 2025-07-09 15:35:22,997 - root - INFO - step: 9080 loss: 17.5607 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.7672 global_avg_mtp_loss: 14.7935 +[titan] 2025-07-09 15:35:22,997 - root - INFO - lr: 2.9559e-04 gnorm: 0.91 [ 2:00:26<20:06:03] +[titan] 2025-07-09 15:35:26,910 - root - INFO - step: 9085 loss: 17.3398 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7364 global_avg_mtp_loss: 14.6034 +[titan] 2025-07-09 15:35:26,910 - root - INFO - lr: 2.9558e-04 gnorm: 0.85 [ 2:00:30<20:05:59] +[titan] 2025-07-09 15:35:30,805 - root - INFO - step: 9090 loss: 17.5570 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.7688 global_avg_mtp_loss: 14.7881 +[titan] 2025-07-09 15:35:30,805 - root - INFO - lr: 2.9557e-04 gnorm: 0.92 [ 2:00:34<20:05:54] +[titan] 2025-07-09 15:35:34,767 - root - INFO - step: 9095 loss: 17.1742 memory: 44.58GiB(31.99%) tps: 82,702 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.7063 global_avg_mtp_loss: 14.4679 +[titan] 2025-07-09 15:35:34,768 - root - INFO - lr: 2.9557e-04 gnorm: 0.92 [ 2:00:38<20:05:50] +[titan] 2025-07-09 15:35:37,905 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:35:38,708 - root - INFO - step: 9100 loss: 17.7444 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.8047 global_avg_mtp_loss: 14.9397 +[titan] 2025-07-09 15:35:38,708 - root - INFO - lr: 2.9556e-04 gnorm: 0.92 [ 2:00:42<20:05:45] +[titan] 2025-07-09 15:35:42,655 - root - INFO - step: 9105 loss: 17.5046 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.7593 global_avg_mtp_loss: 14.7453 +[titan] 2025-07-09 15:35:42,656 - root - INFO - lr: 2.9556e-04 gnorm: 0.99 [ 2:00:46<20:05:41] +[titan] 2025-07-09 15:35:46,581 - root - INFO - step: 9110 loss: 17.4161 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.7439 global_avg_mtp_loss: 14.6722 +[titan] 2025-07-09 15:35:46,582 - root - INFO - lr: 2.9555e-04 gnorm: 0.89 [ 2:00:50<20:05:37] +[titan] 2025-07-09 15:35:50,507 - root - INFO - step: 9115 loss: 17.6209 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.7773 global_avg_mtp_loss: 14.8436 +[titan] 2025-07-09 15:35:50,508 - root - INFO - lr: 2.9555e-04 gnorm: 0.96 [ 2:00:54<20:05:32] +[titan] 2025-07-09 15:35:54,429 - root - INFO - step: 9120 loss: 17.3635 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.7467 global_avg_mtp_loss: 14.6168 +[titan] 2025-07-09 15:35:54,429 - root - INFO - lr: 2.9554e-04 gnorm: 0.91 [ 2:00:58<20:05:27] +[titan] 2025-07-09 15:35:58,349 - root - INFO - step: 9125 loss: 17.3259 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.7195 global_avg_mtp_loss: 14.6064 +[titan] 2025-07-09 15:35:58,349 - root - INFO - lr: 2.9554e-04 gnorm: 0.97 [ 2:01:02<20:05:23] +[titan] 2025-07-09 15:36:02,247 - root - INFO - step: 9130 loss: 17.3752 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.7337 global_avg_mtp_loss: 14.6415 +[titan] 2025-07-09 15:36:02,247 - root - INFO - lr: 2.9553e-04 gnorm: 0.92 [ 2:01:06<20:05:18] +[titan] 2025-07-09 15:36:06,150 - root - INFO - step: 9135 loss: 17.3480 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.7298 global_avg_mtp_loss: 14.6182 +[titan] 2025-07-09 15:36:06,150 - root - INFO - lr: 2.9553e-04 gnorm: 0.87 [ 2:01:09<20:05:13] +[titan] 2025-07-09 15:36:10,095 - root - INFO - step: 9140 loss: 17.5667 memory: 44.58GiB(31.99%) tps: 83,060 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.7736 global_avg_mtp_loss: 14.7930 +[titan] 2025-07-09 15:36:10,096 - root - INFO - lr: 2.9552e-04 gnorm: 0.87 [ 2:01:13<20:05:09] +[titan] 2025-07-09 15:36:14,057 - root - INFO - step: 9145 loss: 17.4911 memory: 44.58GiB(31.99%) tps: 82,713 tflops: 285.46 mfu: 28.86% global_avg_ntp_loss: 2.7498 global_avg_mtp_loss: 14.7413 +[titan] 2025-07-09 15:36:14,058 - root - INFO - lr: 2.9551e-04 gnorm: 0.89 [ 2:01:17<20:05:05] +[titan] 2025-07-09 15:36:17,176 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:36:17,961 - root - INFO - step: 9150 loss: 17.3070 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7172 global_avg_mtp_loss: 14.5898 +[titan] 2025-07-09 15:36:17,961 - root - INFO - lr: 2.9551e-04 gnorm: 0.86 [ 2:01:21<20:05:00] +[titan] 2025-07-09 15:36:21,886 - root - INFO - step: 9155 loss: 17.4817 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.7735 global_avg_mtp_loss: 14.7082 +[titan] 2025-07-09 15:36:21,887 - root - INFO - lr: 2.9550e-04 gnorm: 0.90 [ 2:01:25<20:04:56] +[titan] 2025-07-09 15:36:25,801 - root - INFO - step: 9160 loss: 17.4371 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.7482 global_avg_mtp_loss: 14.6890 +[titan] 2025-07-09 15:36:25,802 - root - INFO - lr: 2.9550e-04 gnorm: 0.85 [ 2:01:29<20:04:51] +[titan] 2025-07-09 15:36:29,737 - root - INFO - step: 9165 loss: 17.2229 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.7155 global_avg_mtp_loss: 14.5074 +[titan] 2025-07-09 15:36:29,737 - root - INFO - lr: 2.9549e-04 gnorm: 0.97 [ 2:01:33<20:04:47] +[titan] 2025-07-09 15:36:33,654 - root - INFO - step: 9170 loss: 17.5105 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.7675 global_avg_mtp_loss: 14.7431 +[titan] 2025-07-09 15:36:33,654 - root - INFO - lr: 2.9549e-04 gnorm: 0.91 [ 2:01:37<20:04:42] +[titan] 2025-07-09 15:36:37,601 - root - INFO - step: 9175 loss: 17.1203 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.6918 global_avg_mtp_loss: 14.4285 +[titan] 2025-07-09 15:36:37,601 - root - INFO - lr: 2.9548e-04 gnorm: 1.05 [ 2:01:41<20:04:38] +[titan] 2025-07-09 15:36:41,502 - root - INFO - step: 9180 loss: 17.6007 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.7915 global_avg_mtp_loss: 14.8093 +[titan] 2025-07-09 15:36:41,502 - root - INFO - lr: 2.9548e-04 gnorm: 1.02 [ 2:01:45<20:04:33] +[titan] 2025-07-09 15:36:45,453 - root - INFO - step: 9185 loss: 17.5440 memory: 44.58GiB(31.99%) tps: 82,931 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.7684 global_avg_mtp_loss: 14.7756 +[titan] 2025-07-09 15:36:45,454 - root - INFO - lr: 2.9547e-04 gnorm: 0.97 [ 2:01:49<20:04:29] +[titan] 2025-07-09 15:36:49,380 - root - INFO - step: 9190 loss: 17.5893 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.7744 global_avg_mtp_loss: 14.8148 +[titan] 2025-07-09 15:36:49,380 - root - INFO - lr: 2.9547e-04 gnorm: 0.91 [ 2:01:53<20:04:24] +[titan] 2025-07-09 15:36:53,291 - root - INFO - step: 9195 loss: 17.2316 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.7320 global_avg_mtp_loss: 14.4997 +[titan] 2025-07-09 15:36:53,291 - root - INFO - lr: 2.9546e-04 gnorm: 1.11 [ 2:01:57<20:04:20] +[titan] 2025-07-09 15:36:56,420 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:36:57,241 - root - INFO - step: 9200 loss: 17.5435 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.7718 global_avg_mtp_loss: 14.7717 +[titan] 2025-07-09 15:36:57,241 - root - INFO - lr: 2.9545e-04 gnorm: 1.02 [ 2:02:01<20:04:15] +[titan] 2025-07-09 15:37:01,122 - root - INFO - step: 9205 loss: 17.6577 memory: 44.58GiB(31.99%) tps: 84,438 tflops: 291.41 mfu: 29.47% global_avg_ntp_loss: 2.8052 global_avg_mtp_loss: 14.8524 +[titan] 2025-07-09 15:37:01,122 - root - INFO - lr: 2.9545e-04 gnorm: 0.89 [ 2:02:04<20:04:10] +[titan] 2025-07-09 15:37:05,016 - root - INFO - step: 9210 loss: 17.3480 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.7356 global_avg_mtp_loss: 14.6124 +[titan] 2025-07-09 15:37:05,017 - root - INFO - lr: 2.9544e-04 gnorm: 1.15 [ 2:02:08<20:04:06] +[titan] 2025-07-09 15:37:08,980 - root - INFO - step: 9215 loss: 17.5162 memory: 44.58GiB(31.99%) tps: 82,687 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.7688 global_avg_mtp_loss: 14.7473 +[titan] 2025-07-09 15:37:08,980 - root - INFO - lr: 2.9544e-04 gnorm: 0.99 [ 2:02:12<20:04:01] +[titan] 2025-07-09 15:37:09,906 - root - INFO - Dumping profiler traces at step 9216 +[titan] 2025-07-09 15:37:09,938 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:37:13,075 - root - INFO - step: 9220 loss: 17.0568 memory: 44.58GiB(31.99%) tps: 80,023 tflops: 276.17 mfu: 27.92% global_avg_ntp_loss: 2.6807 global_avg_mtp_loss: 14.3761 +[titan] 2025-07-09 15:37:13,075 - root - INFO - lr: 2.9543e-04 gnorm: 1.01 [ 2:02:16<20:03:59] +[titan] 2025-07-09 15:37:16,996 - root - INFO - step: 9225 loss: 17.3686 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.7402 global_avg_mtp_loss: 14.6284 +[titan] 2025-07-09 15:37:16,996 - root - INFO - lr: 2.9543e-04 gnorm: 0.89 [ 2:02:20<20:03:54] +[titan] 2025-07-09 15:37:20,896 - root - INFO - step: 9230 loss: 17.6111 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.7913 global_avg_mtp_loss: 14.8198 +[titan] 2025-07-09 15:37:20,896 - root - INFO - lr: 2.9542e-04 gnorm: 0.91 [ 2:02:24<20:03:49] +[titan] 2025-07-09 15:37:24,831 - root - INFO - step: 9235 loss: 17.5098 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.7729 global_avg_mtp_loss: 14.7369 +[titan] 2025-07-09 15:37:24,831 - root - INFO - lr: 2.9542e-04 gnorm: 0.90 [ 2:02:28<20:03:45] +[titan] 2025-07-09 15:37:28,750 - root - INFO - step: 9240 loss: 17.3904 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.7432 global_avg_mtp_loss: 14.6471 +[titan] 2025-07-09 15:37:28,750 - root - INFO - lr: 2.9541e-04 gnorm: 0.89 [ 2:02:32<20:03:40] +[titan] 2025-07-09 15:37:32,710 - root - INFO - step: 9245 loss: 17.3032 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.7174 global_avg_mtp_loss: 14.5858 +[titan] 2025-07-09 15:37:32,710 - root - INFO - lr: 2.9540e-04 gnorm: 1.03 [ 2:02:36<20:03:36] +[titan] 2025-07-09 15:37:35,861 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:37:36,655 - root - INFO - step: 9250 loss: 17.4012 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.7404 global_avg_mtp_loss: 14.6608 +[titan] 2025-07-09 15:37:36,655 - root - INFO - lr: 2.9540e-04 gnorm: 1.01 [ 2:02:40<20:03:32] +[titan] 2025-07-09 15:37:40,584 - root - INFO - step: 9255 loss: 17.3306 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.7245 global_avg_mtp_loss: 14.6061 +[titan] 2025-07-09 15:37:40,584 - root - INFO - lr: 2.9539e-04 gnorm: 0.89 [ 2:02:44<20:03:27] +[titan] 2025-07-09 15:37:44,502 - root - INFO - step: 9260 loss: 17.3913 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7316 global_avg_mtp_loss: 14.6597 +[titan] 2025-07-09 15:37:44,502 - root - INFO - lr: 2.9539e-04 gnorm: 0.94 [ 2:02:48<20:03:23] +[titan] 2025-07-09 15:37:48,418 - root - INFO - step: 9265 loss: 17.3966 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7326 global_avg_mtp_loss: 14.6640 +[titan] 2025-07-09 15:37:48,419 - root - INFO - lr: 2.9538e-04 gnorm: 0.97 [ 2:02:52<20:03:18] +[titan] 2025-07-09 15:37:52,315 - root - INFO - step: 9270 loss: 17.4726 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.7468 global_avg_mtp_loss: 14.7258 +[titan] 2025-07-09 15:37:52,316 - root - INFO - lr: 2.9538e-04 gnorm: 0.89 [ 2:02:56<20:03:13] +[titan] 2025-07-09 15:37:56,247 - root - INFO - step: 9275 loss: 17.3115 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.7344 global_avg_mtp_loss: 14.5771 +[titan] 2025-07-09 15:37:56,247 - root - INFO - lr: 2.9537e-04 gnorm: 0.95 [ 2:03:00<20:03:09] +[titan] 2025-07-09 15:38:00,161 - root - INFO - step: 9280 loss: 17.6846 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7991 global_avg_mtp_loss: 14.8855 +[titan] 2025-07-09 15:38:00,162 - root - INFO - lr: 2.9537e-04 gnorm: 0.92 [ 2:03:03<20:03:04] +[titan] 2025-07-09 15:38:04,122 - root - INFO - step: 9285 loss: 17.5216 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.7584 global_avg_mtp_loss: 14.7632 +[titan] 2025-07-09 15:38:04,122 - root - INFO - lr: 2.9536e-04 gnorm: 0.98 [ 2:03:07<20:03:00] +[titan] 2025-07-09 15:38:08,038 - root - INFO - step: 9290 loss: 17.4520 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.7553 global_avg_mtp_loss: 14.6967 +[titan] 2025-07-09 15:38:08,039 - root - INFO - lr: 2.9535e-04 gnorm: 0.86 [ 2:03:11<20:02:56] +[titan] 2025-07-09 15:38:11,934 - root - INFO - step: 9295 loss: 17.5931 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.7848 global_avg_mtp_loss: 14.8084 +[titan] 2025-07-09 15:38:11,934 - root - INFO - lr: 2.9535e-04 gnorm: 0.98 [ 2:03:15<20:02:51] +[titan] 2025-07-09 15:38:15,044 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:38:15,830 - root - INFO - step: 9300 loss: 17.4769 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.7629 global_avg_mtp_loss: 14.7140 +[titan] 2025-07-09 15:38:15,830 - root - INFO - lr: 2.9534e-04 gnorm: 0.88 [ 2:03:19<20:02:46] +[titan] 2025-07-09 15:38:19,790 - root - INFO - step: 9305 loss: 17.4990 memory: 44.58GiB(31.99%) tps: 82,746 tflops: 285.57 mfu: 28.87% global_avg_ntp_loss: 2.7686 global_avg_mtp_loss: 14.7304 +[titan] 2025-07-09 15:38:19,790 - root - INFO - lr: 2.9534e-04 gnorm: 0.92 [ 2:03:23<20:02:42] +[titan] 2025-07-09 15:38:23,726 - root - INFO - step: 9310 loss: 17.6316 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.7851 global_avg_mtp_loss: 14.8464 +[titan] 2025-07-09 15:38:23,726 - root - INFO - lr: 2.9533e-04 gnorm: 0.82 [ 2:03:27<20:02:37] +[titan] 2025-07-09 15:38:27,667 - root - INFO - step: 9315 loss: 17.4752 memory: 44.58GiB(31.99%) tps: 83,137 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.7654 global_avg_mtp_loss: 14.7098 +[titan] 2025-07-09 15:38:27,668 - root - INFO - lr: 2.9533e-04 gnorm: 0.96 [ 2:03:31<20:02:33] +[titan] 2025-07-09 15:38:31,561 - root - INFO - step: 9320 loss: 17.4019 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.7383 global_avg_mtp_loss: 14.6637 +[titan] 2025-07-09 15:38:31,561 - root - INFO - lr: 2.9532e-04 gnorm: 0.92 [ 2:03:35<20:02:28] +[titan] 2025-07-09 15:38:35,475 - root - INFO - step: 9325 loss: 17.4655 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.7565 global_avg_mtp_loss: 14.7090 +[titan] 2025-07-09 15:38:35,476 - root - INFO - lr: 2.9532e-04 gnorm: 0.91 [ 2:03:39<20:02:24] +[titan] 2025-07-09 15:38:39,385 - root - INFO - step: 9330 loss: 17.1779 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.7064 global_avg_mtp_loss: 14.4715 +[titan] 2025-07-09 15:38:39,385 - root - INFO - lr: 2.9531e-04 gnorm: 1.05 [ 2:03:43<20:02:19] +[titan] 2025-07-09 15:38:43,286 - root - INFO - step: 9335 loss: 17.5100 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.7630 global_avg_mtp_loss: 14.7471 +[titan] 2025-07-09 15:38:43,286 - root - INFO - lr: 2.9530e-04 gnorm: 1.03 [ 2:03:47<20:02:14] +[titan] 2025-07-09 15:38:47,224 - root - INFO - step: 9340 loss: 17.2524 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.7143 global_avg_mtp_loss: 14.5381 +[titan] 2025-07-09 15:38:47,224 - root - INFO - lr: 2.9530e-04 gnorm: 0.87 [ 2:03:51<20:02:10] +[titan] 2025-07-09 15:38:51,126 - root - INFO - step: 9345 loss: 17.2311 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.7118 global_avg_mtp_loss: 14.5192 +[titan] 2025-07-09 15:38:51,127 - root - INFO - lr: 2.9529e-04 gnorm: 0.98 [ 2:03:54<20:02:05] +[titan] 2025-07-09 15:38:54,253 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:38:55,044 - root - INFO - step: 9350 loss: 17.5476 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7771 global_avg_mtp_loss: 14.7705 +[titan] 2025-07-09 15:38:55,044 - root - INFO - lr: 2.9529e-04 gnorm: 0.87 [ 2:03:58<20:02:01] +[titan] 2025-07-09 15:38:58,967 - root - INFO - step: 9355 loss: 17.6033 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.7895 global_avg_mtp_loss: 14.8138 +[titan] 2025-07-09 15:38:58,967 - root - INFO - lr: 2.9528e-04 gnorm: 0.98 [ 2:04:02<20:01:56] +[titan] 2025-07-09 15:39:02,878 - root - INFO - step: 9360 loss: 17.1739 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6957 global_avg_mtp_loss: 14.4782 +[titan] 2025-07-09 15:39:02,878 - root - INFO - lr: 2.9528e-04 gnorm: 0.97 [ 2:04:06<20:01:51] +[titan] 2025-07-09 15:39:06,788 - root - INFO - step: 9365 loss: 17.2374 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.7078 global_avg_mtp_loss: 14.5296 +[titan] 2025-07-09 15:39:06,788 - root - INFO - lr: 2.9527e-04 gnorm: 0.86 [ 2:04:10<20:01:47] +[titan] 2025-07-09 15:39:10,692 - root - INFO - step: 9370 loss: 17.5199 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7621 global_avg_mtp_loss: 14.7578 +[titan] 2025-07-09 15:39:10,692 - root - INFO - lr: 2.9526e-04 gnorm: 0.96 [ 2:04:14<20:01:42] +[titan] 2025-07-09 15:39:14,606 - root - INFO - step: 9375 loss: 17.4734 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7507 global_avg_mtp_loss: 14.7228 +[titan] 2025-07-09 15:39:14,607 - root - INFO - lr: 2.9526e-04 gnorm: 0.92 [ 2:04:18<20:01:38] +[titan] 2025-07-09 15:39:18,508 - root - INFO - step: 9380 loss: 17.3445 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.7446 global_avg_mtp_loss: 14.5999 +[titan] 2025-07-09 15:39:18,508 - root - INFO - lr: 2.9525e-04 gnorm: 1.04 [ 2:04:22<20:01:33] +[titan] 2025-07-09 15:39:22,417 - root - INFO - step: 9385 loss: 17.3568 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7439 global_avg_mtp_loss: 14.6129 +[titan] 2025-07-09 15:39:22,417 - root - INFO - lr: 2.9525e-04 gnorm: 0.97 [ 2:04:26<20:01:28] +[titan] 2025-07-09 15:39:26,318 - root - INFO - step: 9390 loss: 17.8368 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.8346 global_avg_mtp_loss: 15.0022 +[titan] 2025-07-09 15:39:26,319 - root - INFO - lr: 2.9524e-04 gnorm: 1.03 [ 2:04:30<20:01:23] +[titan] 2025-07-09 15:39:30,273 - root - INFO - step: 9395 loss: 17.4299 memory: 44.58GiB(31.99%) tps: 82,870 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.7471 global_avg_mtp_loss: 14.6828 +[titan] 2025-07-09 15:39:30,273 - root - INFO - lr: 2.9524e-04 gnorm: 1.04 [ 2:04:34<20:01:19] +[titan] 2025-07-09 15:39:33,396 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:39:34,185 - root - INFO - step: 9400 loss: 17.5887 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.7912 global_avg_mtp_loss: 14.7975 +[titan] 2025-07-09 15:39:34,186 - root - INFO - lr: 2.9523e-04 gnorm: 1.04 [ 2:04:37<20:01:15] +[titan] 2025-07-09 15:39:38,118 - root - INFO - step: 9405 loss: 17.5630 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.7806 global_avg_mtp_loss: 14.7824 +[titan] 2025-07-09 15:39:38,118 - root - INFO - lr: 2.9523e-04 gnorm: 0.94 [ 2:04:41<20:01:10] +[titan] 2025-07-09 15:39:42,071 - root - INFO - step: 9410 loss: 17.4840 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.7571 global_avg_mtp_loss: 14.7269 +[titan] 2025-07-09 15:39:42,071 - root - INFO - lr: 2.9522e-04 gnorm: 1.03 [ 2:04:45<20:01:06] +[titan] 2025-07-09 15:39:46,013 - root - INFO - step: 9415 loss: 17.7467 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.8167 global_avg_mtp_loss: 14.9300 +[titan] 2025-07-09 15:39:46,013 - root - INFO - lr: 2.9521e-04 gnorm: 0.96 [ 2:04:49<20:01:02] +[titan] 2025-07-09 15:39:49,919 - root - INFO - step: 9420 loss: 17.3799 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.7281 global_avg_mtp_loss: 14.6518 +[titan] 2025-07-09 15:39:49,920 - root - INFO - lr: 2.9521e-04 gnorm: 0.97 [ 2:04:53<20:00:57] +[titan] 2025-07-09 15:39:53,817 - root - INFO - step: 9425 loss: 17.5093 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.7652 global_avg_mtp_loss: 14.7441 +[titan] 2025-07-09 15:39:53,817 - root - INFO - lr: 2.9520e-04 gnorm: 0.87 [ 2:04:57<20:00:52] +[titan] 2025-07-09 15:39:57,761 - root - INFO - step: 9430 loss: 17.6492 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.7936 global_avg_mtp_loss: 14.8555 +[titan] 2025-07-09 15:39:57,761 - root - INFO - lr: 2.9520e-04 gnorm: 0.89 [ 2:05:01<20:00:48] +[titan] 2025-07-09 15:40:01,717 - root - INFO - step: 9435 loss: 17.0828 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.6943 global_avg_mtp_loss: 14.3885 +[titan] 2025-07-09 15:40:01,717 - root - INFO - lr: 2.9519e-04 gnorm: 0.91 [ 2:05:05<20:00:44] +[titan] 2025-07-09 15:40:05,616 - root - INFO - step: 9440 loss: 17.4237 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.7516 global_avg_mtp_loss: 14.6721 +[titan] 2025-07-09 15:40:05,616 - root - INFO - lr: 2.9519e-04 gnorm: 0.93 [ 2:05:09<20:00:39] +[titan] 2025-07-09 15:40:09,513 - root - INFO - step: 9445 loss: 17.3815 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7458 global_avg_mtp_loss: 14.6357 +[titan] 2025-07-09 15:40:09,513 - root - INFO - lr: 2.9518e-04 gnorm: 0.90 [ 2:05:13<20:00:34] +[titan] 2025-07-09 15:40:12,620 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:40:13,411 - root - INFO - step: 9450 loss: 17.5191 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.7647 global_avg_mtp_loss: 14.7544 +[titan] 2025-07-09 15:40:13,411 - root - INFO - lr: 2.9517e-04 gnorm: 1.09 [ 2:05:17<20:00:30] +[titan] 2025-07-09 15:40:17,322 - root - INFO - step: 9455 loss: 17.3490 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.7216 global_avg_mtp_loss: 14.6274 +[titan] 2025-07-09 15:40:17,322 - root - INFO - lr: 2.9517e-04 gnorm: 0.99 [ 2:05:21<20:00:25] +[titan] 2025-07-09 15:40:21,302 - root - INFO - step: 9460 loss: 17.4044 memory: 44.58GiB(31.99%) tps: 82,327 tflops: 284.13 mfu: 28.73% global_avg_ntp_loss: 2.7449 global_avg_mtp_loss: 14.6596 +[titan] 2025-07-09 15:40:21,303 - root - INFO - lr: 2.9516e-04 gnorm: 0.93 [ 2:05:25<20:00:21] +[titan] 2025-07-09 15:40:25,204 - root - INFO - step: 9465 loss: 17.3715 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7503 global_avg_mtp_loss: 14.6212 +[titan] 2025-07-09 15:40:25,205 - root - INFO - lr: 2.9516e-04 gnorm: 0.91 [ 2:05:29<20:00:16] +[titan] 2025-07-09 15:40:29,124 - root - INFO - step: 9470 loss: 17.5977 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.7880 global_avg_mtp_loss: 14.8097 +[titan] 2025-07-09 15:40:29,124 - root - INFO - lr: 2.9515e-04 gnorm: 0.94 [ 2:05:32<20:00:12] +[titan] 2025-07-09 15:40:33,024 - root - INFO - step: 9475 loss: 17.4241 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.7547 global_avg_mtp_loss: 14.6694 +[titan] 2025-07-09 15:40:33,024 - root - INFO - lr: 2.9515e-04 gnorm: 0.90 [ 2:05:36<20:00:07] +[titan] 2025-07-09 15:40:36,933 - root - INFO - step: 9480 loss: 17.3066 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7342 global_avg_mtp_loss: 14.5724 +[titan] 2025-07-09 15:40:36,933 - root - INFO - lr: 2.9514e-04 gnorm: 0.87 [ 2:05:40<20:00:02] +[titan] 2025-07-09 15:40:40,839 - root - INFO - step: 9485 loss: 17.2675 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7240 global_avg_mtp_loss: 14.5435 +[titan] 2025-07-09 15:40:40,839 - root - INFO - lr: 2.9513e-04 gnorm: 0.95 [ 2:05:44<19:59:58] +[titan] 2025-07-09 15:40:44,828 - root - INFO - step: 9490 loss: 17.5495 memory: 44.58GiB(31.99%) tps: 82,140 tflops: 283.48 mfu: 28.66% global_avg_ntp_loss: 2.7715 global_avg_mtp_loss: 14.7780 +[titan] 2025-07-09 15:40:44,828 - root - INFO - lr: 2.9513e-04 gnorm: 0.94 [ 2:05:48<19:59:54] +[titan] 2025-07-09 15:40:48,733 - root - INFO - step: 9495 loss: 17.3158 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.7230 global_avg_mtp_loss: 14.5927 +[titan] 2025-07-09 15:40:48,733 - root - INFO - lr: 2.9512e-04 gnorm: 0.94 [ 2:05:52<19:59:49] +[titan] 2025-07-09 15:40:51,857 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:40:52,658 - root - INFO - step: 9500 loss: 17.4121 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.7543 global_avg_mtp_loss: 14.6578 +[titan] 2025-07-09 15:40:52,659 - root - INFO - lr: 2.9512e-04 gnorm: 0.90 [ 2:05:56<19:59:45] +[titan] 2025-07-09 15:40:56,578 - root - INFO - step: 9505 loss: 17.3450 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.7415 global_avg_mtp_loss: 14.6035 +[titan] 2025-07-09 15:40:56,578 - root - INFO - lr: 2.9511e-04 gnorm: 1.00 [ 2:06:00<19:59:40] +[titan] 2025-07-09 15:41:00,510 - root - INFO - step: 9510 loss: 17.3451 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.7290 global_avg_mtp_loss: 14.6161 +[titan] 2025-07-09 15:41:00,511 - root - INFO - lr: 2.9511e-04 gnorm: 0.86 [ 2:06:04<19:59:36] +[titan] 2025-07-09 15:41:04,414 - root - INFO - step: 9515 loss: 17.3205 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7215 global_avg_mtp_loss: 14.5991 +[titan] 2025-07-09 15:41:04,414 - root - INFO - lr: 2.9510e-04 gnorm: 0.94 [ 2:06:08<19:59:31] +[titan] 2025-07-09 15:41:08,301 - root - INFO - step: 9520 loss: 17.3183 memory: 44.58GiB(31.99%) tps: 84,314 tflops: 290.98 mfu: 29.42% global_avg_ntp_loss: 2.7123 global_avg_mtp_loss: 14.6060 +[titan] 2025-07-09 15:41:08,301 - root - INFO - lr: 2.9509e-04 gnorm: 0.98 [ 2:06:12<19:59:26] +[titan] 2025-07-09 15:41:12,208 - root - INFO - step: 9525 loss: 17.2390 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7062 global_avg_mtp_loss: 14.5328 +[titan] 2025-07-09 15:41:12,209 - root - INFO - lr: 2.9509e-04 gnorm: 0.93 [ 2:06:16<19:59:22] +[titan] 2025-07-09 15:41:16,113 - root - INFO - step: 9530 loss: 17.4209 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7351 global_avg_mtp_loss: 14.6857 +[titan] 2025-07-09 15:41:16,113 - root - INFO - lr: 2.9508e-04 gnorm: 0.92 [ 2:06:19<19:59:17] +[titan] 2025-07-09 15:41:20,019 - root - INFO - step: 9535 loss: 17.3962 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7378 global_avg_mtp_loss: 14.6583 +[titan] 2025-07-09 15:41:20,020 - root - INFO - lr: 2.9508e-04 gnorm: 0.84 [ 2:06:23<19:59:12] +[titan] 2025-07-09 15:41:23,934 - root - INFO - step: 9540 loss: 17.3020 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7174 global_avg_mtp_loss: 14.5846 +[titan] 2025-07-09 15:41:23,934 - root - INFO - lr: 2.9507e-04 gnorm: 0.93 [ 2:06:27<19:59:08] +[titan] 2025-07-09 15:41:27,853 - root - INFO - step: 9545 loss: 17.3297 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7358 global_avg_mtp_loss: 14.5939 +[titan] 2025-07-09 15:41:27,853 - root - INFO - lr: 2.9507e-04 gnorm: 0.95 [ 2:06:31<19:59:03] +[titan] 2025-07-09 15:41:30,988 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:41:31,770 - root - INFO - step: 9550 loss: 17.3164 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.7225 global_avg_mtp_loss: 14.5938 +[titan] 2025-07-09 15:41:31,771 - root - INFO - lr: 2.9506e-04 gnorm: 0.89 [ 2:06:35<19:58:59] +[titan] 2025-07-09 15:41:35,661 - root - INFO - step: 9555 loss: 17.5274 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.7706 global_avg_mtp_loss: 14.7568 +[titan] 2025-07-09 15:41:35,662 - root - INFO - lr: 2.9505e-04 gnorm: 0.90 [ 2:06:39<19:58:54] +[titan] 2025-07-09 15:41:39,577 - root - INFO - step: 9560 loss: 17.4766 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.7606 global_avg_mtp_loss: 14.7161 +[titan] 2025-07-09 15:41:39,577 - root - INFO - lr: 2.9505e-04 gnorm: 0.92 [ 2:06:43<19:58:49] +[titan] 2025-07-09 15:41:43,497 - root - INFO - step: 9565 loss: 17.2726 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.7144 global_avg_mtp_loss: 14.5582 +[titan] 2025-07-09 15:41:43,497 - root - INFO - lr: 2.9504e-04 gnorm: 0.95 [ 2:06:47<19:58:45] +[titan] 2025-07-09 15:41:47,391 - root - INFO - step: 9570 loss: 17.1757 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.6926 global_avg_mtp_loss: 14.4830 +[titan] 2025-07-09 15:41:47,391 - root - INFO - lr: 2.9504e-04 gnorm: 1.02 [ 2:06:51<19:58:40] +[titan] 2025-07-09 15:41:51,302 - root - INFO - step: 9575 loss: 17.6137 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.8003 global_avg_mtp_loss: 14.8135 +[titan] 2025-07-09 15:41:51,303 - root - INFO - lr: 2.9503e-04 gnorm: 0.98 [ 2:06:55<19:58:35] +[titan] 2025-07-09 15:41:55,190 - root - INFO - step: 9580 loss: 17.4330 memory: 44.58GiB(31.99%) tps: 84,296 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.7516 global_avg_mtp_loss: 14.6814 +[titan] 2025-07-09 15:41:55,190 - root - INFO - lr: 2.9503e-04 gnorm: 0.89 [ 2:06:58<19:58:31] +[titan] 2025-07-09 15:41:59,100 - root - INFO - step: 9585 loss: 17.4619 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.7510 global_avg_mtp_loss: 14.7109 +[titan] 2025-07-09 15:41:59,101 - root - INFO - lr: 2.9502e-04 gnorm: 0.88 [ 2:07:02<19:58:26] +[titan] 2025-07-09 15:42:03,039 - root - INFO - step: 9590 loss: 17.1622 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.6916 global_avg_mtp_loss: 14.4706 +[titan] 2025-07-09 15:42:03,039 - root - INFO - lr: 2.9501e-04 gnorm: 0.92 [ 2:07:06<19:58:22] +[titan] 2025-07-09 15:42:06,941 - root - INFO - step: 9595 loss: 17.5053 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.7741 global_avg_mtp_loss: 14.7313 +[titan] 2025-07-09 15:42:06,942 - root - INFO - lr: 2.9501e-04 gnorm: 0.93 [ 2:07:10<19:58:17] +[titan] 2025-07-09 15:42:10,057 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:42:10,846 - root - INFO - step: 9600 loss: 17.4253 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7433 global_avg_mtp_loss: 14.6820 +[titan] 2025-07-09 15:42:10,846 - root - INFO - lr: 2.9500e-04 gnorm: 0.93 [ 2:07:14<19:58:12] +[titan] 2025-07-09 15:42:14,777 - root - INFO - step: 9605 loss: 17.4261 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.7563 global_avg_mtp_loss: 14.6698 +[titan] 2025-07-09 15:42:14,777 - root - INFO - lr: 2.9500e-04 gnorm: 0.91 [ 2:07:18<19:58:08] +[titan] 2025-07-09 15:42:18,673 - root - INFO - step: 9610 loss: 17.6776 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.8103 global_avg_mtp_loss: 14.8673 +[titan] 2025-07-09 15:42:18,673 - root - INFO - lr: 2.9499e-04 gnorm: 0.84 [ 2:07:22<19:58:03] +[titan] 2025-07-09 15:42:22,596 - root - INFO - step: 9615 loss: 17.4491 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.7593 global_avg_mtp_loss: 14.6898 +[titan] 2025-07-09 15:42:22,597 - root - INFO - lr: 2.9499e-04 gnorm: 0.87 [ 2:07:26<19:57:59] +[titan] 2025-07-09 15:42:26,492 - root - INFO - step: 9620 loss: 17.3261 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7405 global_avg_mtp_loss: 14.5857 +[titan] 2025-07-09 15:42:26,492 - root - INFO - lr: 2.9498e-04 gnorm: 0.89 [ 2:07:30<19:57:54] +[titan] 2025-07-09 15:42:30,411 - root - INFO - step: 9625 loss: 17.3621 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.7280 global_avg_mtp_loss: 14.6341 +[titan] 2025-07-09 15:42:30,412 - root - INFO - lr: 2.9497e-04 gnorm: 0.89 [ 2:07:34<19:57:49] +[titan] 2025-07-09 15:42:34,336 - root - INFO - step: 9630 loss: 17.5103 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.7598 global_avg_mtp_loss: 14.7505 +[titan] 2025-07-09 15:42:34,336 - root - INFO - lr: 2.9497e-04 gnorm: 0.86 [ 2:07:38<19:57:45] +[titan] 2025-07-09 15:42:38,249 - root - INFO - step: 9635 loss: 17.0474 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6883 global_avg_mtp_loss: 14.3590 +[titan] 2025-07-09 15:42:38,250 - root - INFO - lr: 2.9496e-04 gnorm: 0.90 [ 2:07:42<19:57:40] +[titan] 2025-07-09 15:42:42,144 - root - INFO - step: 9640 loss: 17.4570 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.7519 global_avg_mtp_loss: 14.7051 +[titan] 2025-07-09 15:42:42,144 - root - INFO - lr: 2.9496e-04 gnorm: 0.89 [ 2:07:45<19:57:36] +[titan] 2025-07-09 15:42:46,034 - root - INFO - step: 9645 loss: 17.4126 memory: 44.58GiB(31.99%) tps: 84,253 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.7405 global_avg_mtp_loss: 14.6721 +[titan] 2025-07-09 15:42:46,034 - root - INFO - lr: 2.9495e-04 gnorm: 0.98 [ 2:07:49<19:57:31] +[titan] 2025-07-09 15:42:49,167 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:42:49,953 - root - INFO - step: 9650 loss: 17.4807 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.7512 global_avg_mtp_loss: 14.7295 +[titan] 2025-07-09 15:42:49,954 - root - INFO - lr: 2.9494e-04 gnorm: 1.03 [ 2:07:53<19:57:26] +[titan] 2025-07-09 15:42:53,841 - root - INFO - step: 9655 loss: 17.5331 memory: 44.58GiB(31.99%) tps: 84,302 tflops: 290.94 mfu: 29.42% global_avg_ntp_loss: 2.7924 global_avg_mtp_loss: 14.7407 +[titan] 2025-07-09 15:42:53,841 - root - INFO - lr: 2.9494e-04 gnorm: 0.97 [ 2:07:57<19:57:22] +[titan] 2025-07-09 15:42:57,752 - root - INFO - step: 9660 loss: 17.4275 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.7458 global_avg_mtp_loss: 14.6817 +[titan] 2025-07-09 15:42:57,752 - root - INFO - lr: 2.9493e-04 gnorm: 1.04 [ 2:08:01<19:57:17] +[titan] 2025-07-09 15:43:01,680 - root - INFO - step: 9665 loss: 17.2618 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.7233 global_avg_mtp_loss: 14.5385 +[titan] 2025-07-09 15:43:01,681 - root - INFO - lr: 2.9493e-04 gnorm: 0.87 [ 2:08:05<19:57:13] +[titan] 2025-07-09 15:43:05,585 - root - INFO - step: 9670 loss: 17.5013 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7646 global_avg_mtp_loss: 14.7367 +[titan] 2025-07-09 15:43:05,585 - root - INFO - lr: 2.9492e-04 gnorm: 0.92 [ 2:08:09<19:57:08] +[titan] 2025-07-09 15:43:09,516 - root - INFO - step: 9675 loss: 17.4241 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.7459 global_avg_mtp_loss: 14.6782 +[titan] 2025-07-09 15:43:09,516 - root - INFO - lr: 2.9492e-04 gnorm: 0.86 [ 2:08:13<19:57:04] +[titan] 2025-07-09 15:43:13,423 - root - INFO - step: 9680 loss: 17.5421 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7635 global_avg_mtp_loss: 14.7787 +[titan] 2025-07-09 15:43:13,423 - root - INFO - lr: 2.9491e-04 gnorm: 0.98 [ 2:08:17<19:56:59] +[titan] 2025-07-09 15:43:17,314 - root - INFO - step: 9685 loss: 17.4529 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.7453 global_avg_mtp_loss: 14.7076 +[titan] 2025-07-09 15:43:17,314 - root - INFO - lr: 2.9490e-04 gnorm: 0.95 [ 2:08:21<19:56:54] +[titan] 2025-07-09 15:43:21,215 - root - INFO - step: 9690 loss: 17.5747 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.7698 global_avg_mtp_loss: 14.8049 +[titan] 2025-07-09 15:43:21,215 - root - INFO - lr: 2.9490e-04 gnorm: 0.97 [ 2:08:24<19:56:49] +[titan] 2025-07-09 15:43:25,127 - root - INFO - step: 9695 loss: 17.5420 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.7633 global_avg_mtp_loss: 14.7787 +[titan] 2025-07-09 15:43:25,127 - root - INFO - lr: 2.9489e-04 gnorm: 0.84 [ 2:08:28<19:56:45] +[titan] 2025-07-09 15:43:28,246 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:43:29,030 - root - INFO - step: 9700 loss: 17.2475 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.7116 global_avg_mtp_loss: 14.5359 +[titan] 2025-07-09 15:43:29,030 - root - INFO - lr: 2.9489e-04 gnorm: 0.93 [ 2:08:32<19:56:40] +[titan] 2025-07-09 15:43:32,933 - root - INFO - step: 9705 loss: 17.4181 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.7415 global_avg_mtp_loss: 14.6766 +[titan] 2025-07-09 15:43:32,934 - root - INFO - lr: 2.9488e-04 gnorm: 0.92 [ 2:08:36<19:56:36] +[titan] 2025-07-09 15:43:36,928 - root - INFO - step: 9710 loss: 17.3461 memory: 44.58GiB(31.99%) tps: 82,036 tflops: 283.12 mfu: 28.63% global_avg_ntp_loss: 2.7319 global_avg_mtp_loss: 14.6142 +[titan] 2025-07-09 15:43:36,928 - root - INFO - lr: 2.9488e-04 gnorm: 0.85 [ 2:08:40<19:56:32] +[titan] 2025-07-09 15:43:40,838 - root - INFO - step: 9715 loss: 17.6005 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.7752 global_avg_mtp_loss: 14.8253 +[titan] 2025-07-09 15:43:40,838 - root - INFO - lr: 2.9487e-04 gnorm: 0.90 [ 2:08:44<19:56:27] +[titan] 2025-07-09 15:43:44,750 - root - INFO - step: 9720 loss: 17.0530 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6755 global_avg_mtp_loss: 14.3775 +[titan] 2025-07-09 15:43:44,750 - root - INFO - lr: 2.9486e-04 gnorm: 1.00 [ 2:08:48<19:56:23] +[titan] 2025-07-09 15:43:48,745 - root - INFO - step: 9725 loss: 17.3013 memory: 44.58GiB(31.99%) tps: 82,016 tflops: 283.05 mfu: 28.62% global_avg_ntp_loss: 2.7264 global_avg_mtp_loss: 14.5749 +[titan] 2025-07-09 15:43:48,746 - root - INFO - lr: 2.9486e-04 gnorm: 1.00 [ 2:08:52<19:56:19] +[titan] 2025-07-09 15:43:51,261 - root - INFO - Dumping profiler traces at step 9728 +[titan] 2025-07-09 15:43:51,302 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-09 15:43:52,873 - root - INFO - step: 9730 loss: 17.5186 memory: 44.58GiB(31.99%) tps: 79,389 tflops: 273.98 mfu: 27.70% global_avg_ntp_loss: 2.7587 global_avg_mtp_loss: 14.7599 +[titan] 2025-07-09 15:43:52,874 - root - INFO - lr: 2.9485e-04 gnorm: 0.91 [ 2:08:56<19:56:16] +[titan] 2025-07-09 15:43:56,782 - root - INFO - step: 9735 loss: 17.3569 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.7376 global_avg_mtp_loss: 14.6194 +[titan] 2025-07-09 15:43:56,782 - root - INFO - lr: 2.9485e-04 gnorm: 0.85 [ 2:09:00<19:56:12] +[titan] 2025-07-09 15:44:00,697 - root - INFO - step: 9740 loss: 17.2695 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.7149 global_avg_mtp_loss: 14.5547 +[titan] 2025-07-09 15:44:00,698 - root - INFO - lr: 2.9484e-04 gnorm: 0.97 [ 2:09:04<19:56:07] +[titan] 2025-07-09 15:44:04,601 - root - INFO - step: 9745 loss: 17.1328 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6956 global_avg_mtp_loss: 14.4371 +[titan] 2025-07-09 15:44:04,602 - root - INFO - lr: 2.9483e-04 gnorm: 0.89 [ 2:09:08<19:56:02] +[titan] 2025-07-09 15:44:07,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:44:08,510 - root - INFO - step: 9750 loss: 17.4271 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.7493 global_avg_mtp_loss: 14.6778 +[titan] 2025-07-09 15:44:08,510 - root - INFO - lr: 2.9483e-04 gnorm: 1.04 [ 2:09:12<19:55:58] +[titan] 2025-07-09 15:44:12,426 - root - INFO - step: 9755 loss: 17.1752 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.7062 global_avg_mtp_loss: 14.4689 +[titan] 2025-07-09 15:44:12,426 - root - INFO - lr: 2.9482e-04 gnorm: 0.94 [ 2:09:16<19:55:53] +[titan] 2025-07-09 15:44:16,322 - root - INFO - step: 9760 loss: 17.5332 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7638 global_avg_mtp_loss: 14.7694 +[titan] 2025-07-09 15:44:16,322 - root - INFO - lr: 2.9482e-04 gnorm: 0.94 [ 2:09:20<19:55:49] +[titan] 2025-07-09 15:44:20,269 - root - INFO - step: 9765 loss: 17.3766 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.7417 global_avg_mtp_loss: 14.6349 +[titan] 2025-07-09 15:44:20,269 - root - INFO - lr: 2.9481e-04 gnorm: 0.89 [ 2:09:24<19:55:44] +[titan] 2025-07-09 15:44:24,175 - root - INFO - step: 9770 loss: 17.4632 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.7641 global_avg_mtp_loss: 14.6991 +[titan] 2025-07-09 15:44:24,175 - root - INFO - lr: 2.9480e-04 gnorm: 1.01 [ 2:09:27<19:55:40] +[titan] 2025-07-09 15:44:28,082 - root - INFO - step: 9775 loss: 17.6003 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.7722 global_avg_mtp_loss: 14.8282 +[titan] 2025-07-09 15:44:28,083 - root - INFO - lr: 2.9480e-04 gnorm: 0.95 [ 2:09:31<19:55:35] +[titan] 2025-07-09 15:44:31,997 - root - INFO - step: 9780 loss: 17.4162 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7479 global_avg_mtp_loss: 14.6683 +[titan] 2025-07-09 15:44:31,997 - root - INFO - lr: 2.9479e-04 gnorm: 1.00 [ 2:09:35<19:55:31] +[titan] 2025-07-09 15:44:35,908 - root - INFO - step: 9785 loss: 17.5994 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.7816 global_avg_mtp_loss: 14.8177 +[titan] 2025-07-09 15:44:35,908 - root - INFO - lr: 2.9479e-04 gnorm: 0.98 [ 2:09:39<19:55:26] +[titan] 2025-07-09 15:44:39,828 - root - INFO - step: 9790 loss: 17.4350 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.7595 global_avg_mtp_loss: 14.6755 +[titan] 2025-07-09 15:44:39,828 - root - INFO - lr: 2.9478e-04 gnorm: 0.90 [ 2:09:43<19:55:22] +[titan] 2025-07-09 15:44:43,755 - root - INFO - step: 9795 loss: 17.4282 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.7540 global_avg_mtp_loss: 14.6742 +[titan] 2025-07-09 15:44:43,755 - root - INFO - lr: 2.9478e-04 gnorm: 0.90 [ 2:09:47<19:55:17] +[titan] 2025-07-09 15:44:46,892 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:44:47,679 - root - INFO - step: 9800 loss: 17.3404 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.7343 global_avg_mtp_loss: 14.6061 +[titan] 2025-07-09 15:44:47,680 - root - INFO - lr: 2.9477e-04 gnorm: 0.90 [ 2:09:51<19:55:13] +[titan] 2025-07-09 15:44:51,612 - root - INFO - step: 9805 loss: 17.3507 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.7217 global_avg_mtp_loss: 14.6289 +[titan] 2025-07-09 15:44:51,612 - root - INFO - lr: 2.9476e-04 gnorm: 0.99 [ 2:09:55<19:55:08] +[titan] 2025-07-09 15:44:55,516 - root - INFO - step: 9810 loss: 17.1707 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6966 global_avg_mtp_loss: 14.4741 +[titan] 2025-07-09 15:44:55,516 - root - INFO - lr: 2.9476e-04 gnorm: 0.88 [ 2:09:59<19:55:04] +[titan] 2025-07-09 15:44:59,451 - root - INFO - step: 9815 loss: 17.2449 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.7309 global_avg_mtp_loss: 14.5140 +[titan] 2025-07-09 15:44:59,451 - root - INFO - lr: 2.9475e-04 gnorm: 0.88 [ 2:10:03<19:54:59] +[titan] 2025-07-09 15:45:03,395 - root - INFO - step: 9820 loss: 17.3367 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.7277 global_avg_mtp_loss: 14.6090 +[titan] 2025-07-09 15:45:03,395 - root - INFO - lr: 2.9475e-04 gnorm: 1.03 [ 2:10:07<19:54:55] +[titan] 2025-07-09 15:45:07,304 - root - INFO - step: 9825 loss: 17.4156 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.7448 global_avg_mtp_loss: 14.6709 +[titan] 2025-07-09 15:45:07,304 - root - INFO - lr: 2.9474e-04 gnorm: 0.93 [ 2:10:11<19:54:50] +[titan] 2025-07-09 15:45:11,216 - root - INFO - step: 9830 loss: 17.3294 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7498 global_avg_mtp_loss: 14.5797 +[titan] 2025-07-09 15:45:11,217 - root - INFO - lr: 2.9473e-04 gnorm: 1.05 [ 2:10:14<19:54:46] +[titan] 2025-07-09 15:45:15,115 - root - INFO - step: 9835 loss: 17.4080 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.7364 global_avg_mtp_loss: 14.6715 +[titan] 2025-07-09 15:45:15,115 - root - INFO - lr: 2.9473e-04 gnorm: 0.93 [ 2:10:18<19:54:41] +[titan] 2025-07-09 15:45:19,018 - root - INFO - step: 9840 loss: 17.3639 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.7489 global_avg_mtp_loss: 14.6150 +[titan] 2025-07-09 15:45:19,019 - root - INFO - lr: 2.9472e-04 gnorm: 0.97 [ 2:10:22<19:54:37] +[titan] 2025-07-09 15:45:22,923 - root - INFO - step: 9845 loss: 17.5364 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7719 global_avg_mtp_loss: 14.7644 +[titan] 2025-07-09 15:45:22,923 - root - INFO - lr: 2.9472e-04 gnorm: 0.97 [ 2:10:26<19:54:32] +[titan] 2025-07-09 15:45:26,040 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:45:26,844 - root - INFO - step: 9850 loss: 17.6360 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.7955 global_avg_mtp_loss: 14.8405 +[titan] 2025-07-09 15:45:26,844 - root - INFO - lr: 2.9471e-04 gnorm: 1.00 [ 2:10:30<19:54:27] +[titan] 2025-07-09 15:45:30,756 - root - INFO - step: 9855 loss: 17.3630 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.7336 global_avg_mtp_loss: 14.6293 +[titan] 2025-07-09 15:45:30,756 - root - INFO - lr: 2.9470e-04 gnorm: 0.84 [ 2:10:34<19:54:23] +[titan] 2025-07-09 15:45:34,650 - root - INFO - step: 9860 loss: 17.2859 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.7235 global_avg_mtp_loss: 14.5624 +[titan] 2025-07-09 15:45:34,651 - root - INFO - lr: 2.9470e-04 gnorm: 0.92 [ 2:10:38<19:54:18] +[titan] 2025-07-09 15:45:38,544 - root - INFO - step: 9865 loss: 17.2897 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.7202 global_avg_mtp_loss: 14.5695 +[titan] 2025-07-09 15:45:38,544 - root - INFO - lr: 2.9469e-04 gnorm: 0.85 [ 2:10:42<19:54:13] +[titan] 2025-07-09 15:45:42,464 - root - INFO - step: 9870 loss: 16.9799 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.3089 +[titan] 2025-07-09 15:45:42,464 - root - INFO - lr: 2.9469e-04 gnorm: 1.02 [ 2:10:46<19:54:09] +[titan] 2025-07-09 15:45:46,364 - root - INFO - step: 9875 loss: 17.1587 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.7155 global_avg_mtp_loss: 14.4432 +[titan] 2025-07-09 15:45:46,365 - root - INFO - lr: 2.9468e-04 gnorm: 0.95 [ 2:10:50<19:54:04] +[titan] 2025-07-09 15:45:50,274 - root - INFO - step: 9880 loss: 17.4697 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.7611 global_avg_mtp_loss: 14.7087 +[titan] 2025-07-09 15:45:50,274 - root - INFO - lr: 2.9467e-04 gnorm: 0.91 [ 2:10:54<19:54:00] +[titan] 2025-07-09 15:45:54,159 - root - INFO - step: 9885 loss: 17.2845 memory: 44.58GiB(31.99%) tps: 84,345 tflops: 291.09 mfu: 29.43% global_avg_ntp_loss: 2.7152 global_avg_mtp_loss: 14.5693 +[titan] 2025-07-09 15:45:54,160 - root - INFO - lr: 2.9467e-04 gnorm: 0.89 [ 2:10:57<19:53:55] +[titan] 2025-07-09 15:45:58,077 - root - INFO - step: 9890 loss: 17.6362 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.7878 global_avg_mtp_loss: 14.8484 +[titan] 2025-07-09 15:45:58,077 - root - INFO - lr: 2.9466e-04 gnorm: 0.95 [ 2:11:01<19:53:50] +[titan] 2025-07-09 15:46:02,024 - root - INFO - step: 9895 loss: 17.1523 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.7012 global_avg_mtp_loss: 14.4510 +[titan] 2025-07-09 15:46:02,024 - root - INFO - lr: 2.9466e-04 gnorm: 0.92 [ 2:11:05<19:53:46] +[titan] 2025-07-09 15:46:05,142 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:46:05,948 - root - INFO - step: 9900 loss: 17.3966 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.7470 global_avg_mtp_loss: 14.6496 +[titan] 2025-07-09 15:46:05,948 - root - INFO - lr: 2.9465e-04 gnorm: 0.86 [ 2:11:09<19:53:42] +[titan] 2025-07-09 15:46:09,905 - root - INFO - step: 9905 loss: 17.2743 memory: 44.58GiB(31.99%) tps: 82,809 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.7355 global_avg_mtp_loss: 14.5389 +[titan] 2025-07-09 15:46:09,906 - root - INFO - lr: 2.9464e-04 gnorm: 0.93 [ 2:11:13<19:53:38] +[titan] 2025-07-09 15:46:13,802 - root - INFO - step: 9910 loss: 17.5145 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.7566 global_avg_mtp_loss: 14.7579 +[titan] 2025-07-09 15:46:13,802 - root - INFO - lr: 2.9464e-04 gnorm: 0.83 [ 2:11:17<19:53:33] +[titan] 2025-07-09 15:46:17,710 - root - INFO - step: 9915 loss: 17.3326 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.7269 global_avg_mtp_loss: 14.6057 +[titan] 2025-07-09 15:46:17,710 - root - INFO - lr: 2.9463e-04 gnorm: 0.96 [ 2:11:21<19:53:28] +[titan] 2025-07-09 15:46:21,619 - root - INFO - step: 9920 loss: 17.4202 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7462 global_avg_mtp_loss: 14.6740 +[titan] 2025-07-09 15:46:21,619 - root - INFO - lr: 2.9463e-04 gnorm: 0.96 [ 2:11:25<19:53:24] +[titan] 2025-07-09 15:46:25,546 - root - INFO - step: 9925 loss: 17.5787 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.7730 global_avg_mtp_loss: 14.8057 +[titan] 2025-07-09 15:46:25,546 - root - INFO - lr: 2.9462e-04 gnorm: 0.96 [ 2:11:29<19:53:19] +[titan] 2025-07-09 15:46:29,471 - root - INFO - step: 9930 loss: 17.5472 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.7733 global_avg_mtp_loss: 14.7739 +[titan] 2025-07-09 15:46:29,471 - root - INFO - lr: 2.9461e-04 gnorm: 0.93 [ 2:11:33<19:53:15] +[titan] 2025-07-09 15:46:33,410 - root - INFO - step: 9935 loss: 17.3309 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.7215 global_avg_mtp_loss: 14.6094 +[titan] 2025-07-09 15:46:33,410 - root - INFO - lr: 2.9461e-04 gnorm: 0.95 [ 2:11:37<19:53:11] +[titan] 2025-07-09 15:46:37,338 - root - INFO - step: 9940 loss: 17.2596 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.7117 global_avg_mtp_loss: 14.5478 +[titan] 2025-07-09 15:46:37,338 - root - INFO - lr: 2.9460e-04 gnorm: 0.96 [ 2:11:41<19:53:06] +[titan] 2025-07-09 15:46:41,277 - root - INFO - step: 9945 loss: 17.4396 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.7375 global_avg_mtp_loss: 14.7021 +[titan] 2025-07-09 15:46:41,277 - root - INFO - lr: 2.9460e-04 gnorm: 0.98 [ 2:11:45<19:53:02] +[titan] 2025-07-09 15:46:44,400 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:46:45,188 - root - INFO - step: 9950 loss: 17.4434 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.7538 global_avg_mtp_loss: 14.6896 +[titan] 2025-07-09 15:46:45,188 - root - INFO - lr: 2.9459e-04 gnorm: 0.92 [ 2:11:48<19:52:57] +[titan] 2025-07-09 15:46:49,089 - root - INFO - step: 9955 loss: 17.0342 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6792 global_avg_mtp_loss: 14.3551 +[titan] 2025-07-09 15:46:49,089 - root - INFO - lr: 2.9458e-04 gnorm: 0.93 [ 2:11:52<19:52:53] +[titan] 2025-07-09 15:46:52,999 - root - INFO - step: 9960 loss: 17.6026 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7778 global_avg_mtp_loss: 14.8248 +[titan] 2025-07-09 15:46:52,999 - root - INFO - lr: 2.9458e-04 gnorm: 0.89 [ 2:11:56<19:52:48] +[titan] 2025-07-09 15:46:56,905 - root - INFO - step: 9965 loss: 17.3194 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.7374 global_avg_mtp_loss: 14.5820 +[titan] 2025-07-09 15:46:56,905 - root - INFO - lr: 2.9457e-04 gnorm: 0.93 [ 2:12:00<19:52:44] +[titan] 2025-07-09 15:47:00,804 - root - INFO - step: 9970 loss: 17.3530 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.7364 global_avg_mtp_loss: 14.6166 +[titan] 2025-07-09 15:47:00,804 - root - INFO - lr: 2.9457e-04 gnorm: 0.93 [ 2:12:04<19:52:39] +[titan] 2025-07-09 15:47:04,748 - root - INFO - step: 9975 loss: 17.7350 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.8259 global_avg_mtp_loss: 14.9091 +[titan] 2025-07-09 15:47:04,748 - root - INFO - lr: 2.9456e-04 gnorm: 0.96 [ 2:12:08<19:52:35] +[titan] 2025-07-09 15:47:08,652 - root - INFO - step: 9980 loss: 17.3598 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.7381 global_avg_mtp_loss: 14.6217 +[titan] 2025-07-09 15:47:08,653 - root - INFO - lr: 2.9455e-04 gnorm: 0.89 [ 2:12:12<19:52:30] +[titan] 2025-07-09 15:47:12,561 - root - INFO - step: 9985 loss: 17.4246 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.7679 global_avg_mtp_loss: 14.6567 +[titan] 2025-07-09 15:47:12,561 - root - INFO - lr: 2.9455e-04 gnorm: 0.85 [ 2:12:16<19:52:26] +[titan] 2025-07-09 15:47:16,480 - root - INFO - step: 9990 loss: 17.5149 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.7767 global_avg_mtp_loss: 14.7382 +[titan] 2025-07-09 15:47:16,480 - root - INFO - lr: 2.9454e-04 gnorm: 0.89 [ 2:12:20<19:52:21] +[titan] 2025-07-09 15:47:20,409 - root - INFO - step: 9995 loss: 17.3168 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.7242 global_avg_mtp_loss: 14.5926 +[titan] 2025-07-09 15:47:20,409 - root - INFO - lr: 2.9454e-04 gnorm: 0.87 [ 2:12:24<19:52:17] +[titan] 2025-07-09 15:47:23,524 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:47:24,312 - root - INFO - step: 10000 loss: 17.5924 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.7775 global_avg_mtp_loss: 14.8149 +[titan] 2025-07-09 15:47:24,312 - root - INFO - lr: 2.9453e-04 gnorm: 0.84 [ 2:12:28<19:52:12] +[titan] 2025-07-09 15:47:24,313 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-09 15:47:26,462 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-09 15:47:26,462 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.15 seconds. +[titan] 2025-07-09 15:49:20,984 - root - INFO - step: 10005 loss: 17.8731 memory: 44.58GiB(31.99%) tps: 2,809 tflops: 9.69 mfu: 0.98% global_avg_ntp_loss: 2.8404 global_avg_mtp_loss: 15.0327 +[titan] 2025-07-09 15:49:20,984 - root - INFO - lr: 2.9452e-04 gnorm: 0.91 [ 2:14:24<20:09:02] +[titan] 2025-07-09 15:49:24,913 - root - INFO - step: 10010 loss: 17.3197 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.7252 global_avg_mtp_loss: 14.5946 +[titan] 2025-07-09 15:49:24,913 - root - INFO - lr: 2.9452e-04 gnorm: 0.98 [ 2:14:28<20:08:57] +[titan] 2025-07-09 15:49:28,828 - root - INFO - step: 10015 loss: 17.3596 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.7457 global_avg_mtp_loss: 14.6139 +[titan] 2025-07-09 15:49:28,829 - root - INFO - lr: 2.9451e-04 gnorm: 0.89 [ 2:14:32<20:08:52] +[titan] 2025-07-09 15:49:32,824 - root - INFO - step: 10020 loss: 17.2986 memory: 44.58GiB(31.99%) tps: 82,016 tflops: 283.05 mfu: 28.62% global_avg_ntp_loss: 2.7177 global_avg_mtp_loss: 14.5809 +[titan] 2025-07-09 15:49:32,824 - root - INFO - lr: 2.9451e-04 gnorm: 1.03 [ 2:14:36<20:08:47] +[titan] 2025-07-09 15:49:36,768 - root - INFO - step: 10025 loss: 17.3652 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.7393 global_avg_mtp_loss: 14.6259 +[titan] 2025-07-09 15:49:36,768 - root - INFO - lr: 2.9450e-04 gnorm: 0.90 [ 2:14:40<20:08:43] +[titan] 2025-07-09 15:49:40,677 - root - INFO - step: 10030 loss: 17.3869 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.7346 global_avg_mtp_loss: 14.6523 +[titan] 2025-07-09 15:49:40,677 - root - INFO - lr: 2.9449e-04 gnorm: 0.84 [ 2:14:44<20:08:38] +[titan] 2025-07-09 15:49:44,605 - root - INFO - step: 10035 loss: 17.5682 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.7800 global_avg_mtp_loss: 14.7882 +[titan] 2025-07-09 15:49:44,606 - root - INFO - lr: 2.9449e-04 gnorm: 1.02 [ 2:14:48<20:08:33] +[titan] 2025-07-09 15:49:48,533 - root - INFO - step: 10040 loss: 16.9046 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6526 global_avg_mtp_loss: 14.2520 +[titan] 2025-07-09 15:49:48,533 - root - INFO - lr: 2.9448e-04 gnorm: 1.00 [ 2:14:52<20:08:28] +[titan] 2025-07-09 15:49:52,465 - root - INFO - step: 10045 loss: 17.5305 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.7583 global_avg_mtp_loss: 14.7722 +[titan] 2025-07-09 15:49:52,465 - root - INFO - lr: 2.9448e-04 gnorm: 0.91 [ 2:14:56<20:08:23] +[titan] 2025-07-09 15:49:55,636 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:49:56,431 - root - INFO - step: 10050 loss: 17.6331 memory: 44.58GiB(31.99%) tps: 82,634 tflops: 285.18 mfu: 28.84% global_avg_ntp_loss: 2.7816 global_avg_mtp_loss: 14.8516 +[titan] 2025-07-09 15:49:56,431 - root - INFO - lr: 2.9447e-04 gnorm: 0.86 [ 2:15:00<20:08:18] +[titan] 2025-07-09 15:50:00,402 - root - INFO - step: 10055 loss: 17.4040 memory: 44.58GiB(31.99%) tps: 82,512 tflops: 284.76 mfu: 28.79% global_avg_ntp_loss: 2.7345 global_avg_mtp_loss: 14.6695 +[titan] 2025-07-09 15:50:00,403 - root - INFO - lr: 2.9446e-04 gnorm: 0.86 [ 2:15:04<20:08:14] +[titan] 2025-07-09 15:50:04,359 - root - INFO - step: 10060 loss: 17.5145 memory: 44.58GiB(31.99%) tps: 82,821 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.7610 global_avg_mtp_loss: 14.7535 +[titan] 2025-07-09 15:50:04,359 - root - INFO - lr: 2.9446e-04 gnorm: 0.92 [ 2:15:08<20:08:09] +[titan] 2025-07-09 15:50:08,314 - root - INFO - step: 10065 loss: 17.5003 memory: 44.58GiB(31.99%) tps: 82,858 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.7657 global_avg_mtp_loss: 14.7346 +[titan] 2025-07-09 15:50:08,315 - root - INFO - lr: 2.9445e-04 gnorm: 0.90 [ 2:15:12<20:08:04] +[titan] 2025-07-09 15:50:12,313 - root - INFO - step: 10070 loss: 17.3118 memory: 44.58GiB(31.99%) tps: 81,953 tflops: 282.83 mfu: 28.60% global_avg_ntp_loss: 2.7230 global_avg_mtp_loss: 14.5888 +[titan] 2025-07-09 15:50:12,313 - root - INFO - lr: 2.9445e-04 gnorm: 0.90 [ 2:15:16<20:08:00] +[titan] 2025-07-09 15:50:16,244 - root - INFO - step: 10075 loss: 17.4419 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.7531 global_avg_mtp_loss: 14.6888 +[titan] 2025-07-09 15:50:16,245 - root - INFO - lr: 2.9444e-04 gnorm: 0.91 [ 2:15:19<20:07:55] +[titan] 2025-07-09 15:50:20,203 - root - INFO - step: 10080 loss: 17.2751 memory: 44.58GiB(31.99%) tps: 82,785 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.7154 global_avg_mtp_loss: 14.5597 +[titan] 2025-07-09 15:50:20,203 - root - INFO - lr: 2.9443e-04 gnorm: 0.82 [ 2:15:23<20:07:50] +[titan] 2025-07-09 15:50:24,171 - root - INFO - step: 10085 loss: 17.5942 memory: 44.58GiB(31.99%) tps: 82,590 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.7656 global_avg_mtp_loss: 14.8286 +[titan] 2025-07-09 15:50:24,171 - root - INFO - lr: 2.9443e-04 gnorm: 0.92 [ 2:15:27<20:07:46] +[titan] 2025-07-09 15:50:28,126 - root - INFO - step: 10090 loss: 17.3315 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.7327 global_avg_mtp_loss: 14.5988 +[titan] 2025-07-09 15:50:28,126 - root - INFO - lr: 2.9442e-04 gnorm: 0.88 [ 2:15:31<20:07:41] +[titan] 2025-07-09 15:50:32,047 - root - INFO - step: 10095 loss: 17.2686 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7078 global_avg_mtp_loss: 14.5608 +[titan] 2025-07-09 15:50:32,048 - root - INFO - lr: 2.9442e-04 gnorm: 0.89 [ 2:15:35<20:07:36] +[titan] 2025-07-09 15:50:35,202 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:50:36,010 - root - INFO - step: 10100 loss: 17.2311 memory: 44.58GiB(31.99%) tps: 82,704 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.7196 global_avg_mtp_loss: 14.5114 +[titan] 2025-07-09 15:50:36,010 - root - INFO - lr: 2.9441e-04 gnorm: 0.90 [ 2:15:39<20:07:31] +[titan] 2025-07-09 15:50:39,939 - root - INFO - step: 10105 loss: 17.2069 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.7009 global_avg_mtp_loss: 14.5061 +[titan] 2025-07-09 15:50:39,939 - root - INFO - lr: 2.9440e-04 gnorm: 0.85 [ 2:15:43<20:07:26] +[titan] 2025-07-09 15:50:43,895 - root - INFO - step: 10110 loss: 17.5263 memory: 44.58GiB(31.99%) tps: 82,841 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.7750 global_avg_mtp_loss: 14.7513 +[titan] 2025-07-09 15:50:43,895 - root - INFO - lr: 2.9440e-04 gnorm: 0.83 [ 2:15:47<20:07:22] +[titan] 2025-07-09 15:50:47,813 - root - INFO - step: 10115 loss: 17.2386 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7048 global_avg_mtp_loss: 14.5338 +[titan] 2025-07-09 15:50:47,814 - root - INFO - lr: 2.9439e-04 gnorm: 0.94 [ 2:15:51<20:07:17] +[titan] 2025-07-09 15:50:51,707 - root - INFO - step: 10120 loss: 17.4539 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.7479 global_avg_mtp_loss: 14.7060 +[titan] 2025-07-09 15:50:51,708 - root - INFO - lr: 2.9438e-04 gnorm: 0.93 [ 2:15:55<20:07:11] +[titan] 2025-07-09 15:50:55,627 - root - INFO - step: 10125 loss: 17.2364 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.7256 global_avg_mtp_loss: 14.5108 +[titan] 2025-07-09 15:50:55,627 - root - INFO - lr: 2.9438e-04 gnorm: 0.88 [ 2:15:59<20:07:06] +[titan] 2025-07-09 15:50:59,547 - root - INFO - step: 10130 loss: 17.1740 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.7110 global_avg_mtp_loss: 14.4630 +[titan] 2025-07-09 15:50:59,547 - root - INFO - lr: 2.9437e-04 gnorm: 0.89 [ 2:16:03<20:07:01] +[titan] 2025-07-09 15:51:03,478 - root - INFO - step: 10135 loss: 17.4127 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.7469 global_avg_mtp_loss: 14.6658 +[titan] 2025-07-09 15:51:03,478 - root - INFO - lr: 2.9437e-04 gnorm: 0.95 [ 2:16:07<20:06:57] +[titan] 2025-07-09 15:51:07,408 - root - INFO - step: 10140 loss: 17.4029 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.7459 global_avg_mtp_loss: 14.6570 +[titan] 2025-07-09 15:51:07,408 - root - INFO - lr: 2.9436e-04 gnorm: 0.93 [ 2:16:11<20:06:52] +[titan] 2025-07-09 15:51:11,333 - root - INFO - step: 10145 loss: 17.6217 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.7813 global_avg_mtp_loss: 14.8404 +[titan] 2025-07-09 15:51:11,334 - root - INFO - lr: 2.9435e-04 gnorm: 0.88 [ 2:16:15<20:06:47] +[titan] 2025-07-09 15:51:14,525 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:51:15,334 - root - INFO - step: 10150 loss: 17.3940 memory: 44.58GiB(31.99%) tps: 81,908 tflops: 282.68 mfu: 28.58% global_avg_ntp_loss: 2.7587 global_avg_mtp_loss: 14.6353 +[titan] 2025-07-09 15:51:15,335 - root - INFO - lr: 2.9435e-04 gnorm: 0.86 [ 2:16:19<20:06:42] +[titan] 2025-07-09 15:51:19,241 - root - INFO - step: 10155 loss: 17.5339 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7711 global_avg_mtp_loss: 14.7627 +[titan] 2025-07-09 15:51:19,241 - root - INFO - lr: 2.9434e-04 gnorm: 0.93 [ 2:16:22<20:06:37] +[titan] 2025-07-09 15:51:23,142 - root - INFO - step: 10160 loss: 17.2193 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.7057 global_avg_mtp_loss: 14.5136 +[titan] 2025-07-09 15:51:23,143 - root - INFO - lr: 2.9434e-04 gnorm: 0.93 [ 2:16:26<20:06:32] +[titan] 2025-07-09 15:51:27,054 - root - INFO - step: 10165 loss: 17.0968 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6834 global_avg_mtp_loss: 14.4134 +[titan] 2025-07-09 15:51:27,054 - root - INFO - lr: 2.9433e-04 gnorm: 0.88 [ 2:16:30<20:06:27] +[titan] 2025-07-09 15:51:31,122 - root - INFO - step: 10170 loss: 17.3161 memory: 44.58GiB(31.99%) tps: 80,564 tflops: 278.04 mfu: 28.11% global_avg_ntp_loss: 2.7104 global_avg_mtp_loss: 14.6057 +[titan] 2025-07-09 15:51:31,122 - root - INFO - lr: 2.9432e-04 gnorm: 0.88 [ 2:16:34<20:06:23] +[titan] 2025-07-09 15:51:35,019 - root - INFO - step: 10175 loss: 17.3879 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.7360 global_avg_mtp_loss: 14.6519 +[titan] 2025-07-09 15:51:35,020 - root - INFO - lr: 2.9432e-04 gnorm: 0.94 [ 2:16:38<20:06:18] +[titan] 2025-07-09 15:51:38,900 - root - INFO - step: 10180 loss: 17.4380 memory: 44.58GiB(31.99%) tps: 84,456 tflops: 291.47 mfu: 29.47% global_avg_ntp_loss: 2.7513 global_avg_mtp_loss: 14.6867 +[titan] 2025-07-09 15:51:38,900 - root - INFO - lr: 2.9431e-04 gnorm: 0.98 [ 2:16:42<20:06:13] +[titan] 2025-07-09 15:51:42,800 - root - INFO - step: 10185 loss: 17.5008 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.7648 global_avg_mtp_loss: 14.7360 +[titan] 2025-07-09 15:51:42,801 - root - INFO - lr: 2.9430e-04 gnorm: 0.92 [ 2:16:46<20:06:08] +[titan] 2025-07-09 15:51:46,705 - root - INFO - step: 10190 loss: 17.3663 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.7378 global_avg_mtp_loss: 14.6284 +[titan] 2025-07-09 15:51:46,705 - root - INFO - lr: 2.9430e-04 gnorm: 0.95 [ 2:16:50<20:06:03] +[titan] 2025-07-09 15:51:50,622 - root - INFO - step: 10195 loss: 17.2646 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.7046 global_avg_mtp_loss: 14.5601 +[titan] 2025-07-09 15:51:50,623 - root - INFO - lr: 2.9429e-04 gnorm: 0.93 [ 2:16:54<20:05:58] +[titan] 2025-07-09 15:51:53,749 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:51:54,535 - root - INFO - step: 10200 loss: 17.3880 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7324 global_avg_mtp_loss: 14.6555 +[titan] 2025-07-09 15:51:54,536 - root - INFO - lr: 2.9429e-04 gnorm: 0.93 [ 2:16:58<20:05:53] +[titan] 2025-07-09 15:51:58,469 - root - INFO - step: 10205 loss: 17.2857 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.7041 global_avg_mtp_loss: 14.5816 +[titan] 2025-07-09 15:51:58,469 - root - INFO - lr: 2.9428e-04 gnorm: 0.95 [ 2:17:02<20:05:48] +[titan] 2025-07-09 15:52:02,435 - root - INFO - step: 10210 loss: 17.3634 memory: 44.58GiB(31.99%) tps: 82,629 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.7407 global_avg_mtp_loss: 14.6227 +[titan] 2025-07-09 15:52:02,435 - root - INFO - lr: 2.9427e-04 gnorm: 0.90 [ 2:17:06<20:05:43] +[titan] 2025-07-09 15:52:06,362 - root - INFO - step: 10215 loss: 16.9690 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.6681 global_avg_mtp_loss: 14.3008 +[titan] 2025-07-09 15:52:06,363 - root - INFO - lr: 2.9427e-04 gnorm: 0.92 [ 2:17:10<20:05:38] +[titan] 2025-07-09 15:52:10,283 - root - INFO - step: 10220 loss: 17.6338 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.7883 global_avg_mtp_loss: 14.8455 +[titan] 2025-07-09 15:52:10,284 - root - INFO - lr: 2.9426e-04 gnorm: 0.92 [ 2:17:14<20:05:33] +[titan] 2025-07-09 15:52:14,187 - root - INFO - step: 10225 loss: 17.3631 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7455 global_avg_mtp_loss: 14.6176 +[titan] 2025-07-09 15:52:14,187 - root - INFO - lr: 2.9426e-04 gnorm: 0.85 [ 2:17:17<20:05:28] +[titan] 2025-07-09 15:52:18,146 - root - INFO - step: 10230 loss: 17.5533 memory: 44.58GiB(31.99%) tps: 82,777 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.7903 global_avg_mtp_loss: 14.7630 +[titan] 2025-07-09 15:52:18,146 - root - INFO - lr: 2.9425e-04 gnorm: 0.93 [ 2:17:21<20:05:23] +[titan] 2025-07-09 15:52:22,063 - root - INFO - step: 10235 loss: 17.4845 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.7520 global_avg_mtp_loss: 14.7325 +[titan] 2025-07-09 15:52:22,063 - root - INFO - lr: 2.9424e-04 gnorm: 0.88 [ 2:17:25<20:05:18] +[titan] 2025-07-09 15:52:26,060 - root - INFO - step: 10240 loss: 17.5249 memory: 44.58GiB(31.99%) tps: 81,987 tflops: 282.95 mfu: 28.61% global_avg_ntp_loss: 2.7649 global_avg_mtp_loss: 14.7600 +[titan] 2025-07-09 15:52:26,061 - root - INFO - lr: 2.9424e-04 gnorm: 0.90 [ 2:17:29<20:05:14] +[titan] 2025-07-09 15:52:26,211 - root - INFO - Dumping profiler traces at step 10240 +[titan] 2025-07-09 15:52:26,244 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:52:30,162 - root - INFO - step: 10245 loss: 17.5400 memory: 44.58GiB(31.99%) tps: 79,891 tflops: 275.72 mfu: 27.88% global_avg_ntp_loss: 2.7588 global_avg_mtp_loss: 14.7812 +[titan] 2025-07-09 15:52:30,163 - root - INFO - lr: 2.9423e-04 gnorm: 0.87 [ 2:17:33<20:05:11] +[titan] 2025-07-09 15:52:33,274 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:52:34,062 - root - INFO - step: 10250 loss: 17.0478 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6888 global_avg_mtp_loss: 14.3590 +[titan] 2025-07-09 15:52:34,062 - root - INFO - lr: 2.9422e-04 gnorm: 0.88 [ 2:17:37<20:05:06] +[titan] 2025-07-09 15:52:38,009 - root - INFO - step: 10255 loss: 17.5837 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.7951 global_avg_mtp_loss: 14.7886 +[titan] 2025-07-09 15:52:38,010 - root - INFO - lr: 2.9422e-04 gnorm: 1.01 [ 2:17:41<20:05:01] +[titan] 2025-07-09 15:52:41,935 - root - INFO - step: 10260 loss: 17.5324 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.7682 global_avg_mtp_loss: 14.7643 +[titan] 2025-07-09 15:52:41,935 - root - INFO - lr: 2.9421e-04 gnorm: 0.92 [ 2:17:45<20:04:56] +[titan] 2025-07-09 15:52:45,854 - root - INFO - step: 10265 loss: 17.4836 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.7571 global_avg_mtp_loss: 14.7265 +[titan] 2025-07-09 15:52:45,854 - root - INFO - lr: 2.9421e-04 gnorm: 0.93 [ 2:17:49<20:04:51] +[titan] 2025-07-09 15:52:49,754 - root - INFO - step: 10270 loss: 17.5963 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.7783 global_avg_mtp_loss: 14.8180 +[titan] 2025-07-09 15:52:49,754 - root - INFO - lr: 2.9420e-04 gnorm: 0.87 [ 2:17:53<20:04:46] +[titan] 2025-07-09 15:52:53,710 - root - INFO - step: 10275 loss: 17.4403 memory: 44.58GiB(31.99%) tps: 82,831 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.7540 global_avg_mtp_loss: 14.6863 +[titan] 2025-07-09 15:52:53,710 - root - INFO - lr: 2.9419e-04 gnorm: 0.84 [ 2:17:57<20:04:41] +[titan] 2025-07-09 15:52:57,630 - root - INFO - step: 10280 loss: 17.3481 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.7218 global_avg_mtp_loss: 14.6263 +[titan] 2025-07-09 15:52:57,630 - root - INFO - lr: 2.9419e-04 gnorm: 0.85 [ 2:18:01<20:04:36] +[titan] 2025-07-09 15:53:01,526 - root - INFO - step: 10285 loss: 17.5449 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7691 global_avg_mtp_loss: 14.7757 +[titan] 2025-07-09 15:53:01,526 - root - INFO - lr: 2.9418e-04 gnorm: 0.90 [ 2:18:05<20:04:31] +[titan] 2025-07-09 15:53:05,414 - root - INFO - step: 10290 loss: 17.2878 memory: 44.58GiB(31.99%) tps: 84,278 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.7003 global_avg_mtp_loss: 14.5875 +[titan] 2025-07-09 15:53:05,414 - root - INFO - lr: 2.9417e-04 gnorm: 0.89 [ 2:18:09<20:04:26] +[titan] 2025-07-09 15:53:09,304 - root - INFO - step: 10295 loss: 17.7521 memory: 44.58GiB(31.99%) tps: 84,242 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.7924 global_avg_mtp_loss: 14.9597 +[titan] 2025-07-09 15:53:09,305 - root - INFO - lr: 2.9417e-04 gnorm: 1.01 [ 2:18:13<20:04:20] +[titan] 2025-07-09 15:53:12,409 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:53:13,210 - root - INFO - step: 10300 loss: 17.1659 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7056 global_avg_mtp_loss: 14.4604 +[titan] 2025-07-09 15:53:13,210 - root - INFO - lr: 2.9416e-04 gnorm: 0.90 [ 2:18:16<20:04:15] +[titan] 2025-07-09 15:53:17,109 - root - INFO - step: 10305 loss: 17.3871 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.7310 global_avg_mtp_loss: 14.6561 +[titan] 2025-07-09 15:53:17,109 - root - INFO - lr: 2.9416e-04 gnorm: 0.95 [ 2:18:20<20:04:10] +[titan] 2025-07-09 15:53:21,017 - root - INFO - step: 10310 loss: 17.2366 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6959 global_avg_mtp_loss: 14.5406 +[titan] 2025-07-09 15:53:21,017 - root - INFO - lr: 2.9415e-04 gnorm: 0.94 [ 2:18:24<20:04:05] +[titan] 2025-07-09 15:53:24,938 - root - INFO - step: 10315 loss: 17.2168 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.7005 global_avg_mtp_loss: 14.5162 +[titan] 2025-07-09 15:53:24,938 - root - INFO - lr: 2.9414e-04 gnorm: 0.93 [ 2:18:28<20:04:00] +[titan] 2025-07-09 15:53:28,870 - root - INFO - step: 10320 loss: 17.3548 memory: 44.58GiB(31.99%) tps: 83,339 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.7343 global_avg_mtp_loss: 14.6205 +[titan] 2025-07-09 15:53:28,871 - root - INFO - lr: 2.9414e-04 gnorm: 0.94 [ 2:18:32<20:03:55] +[titan] 2025-07-09 15:53:32,788 - root - INFO - step: 10325 loss: 17.3307 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7346 global_avg_mtp_loss: 14.5961 +[titan] 2025-07-09 15:53:32,788 - root - INFO - lr: 2.9413e-04 gnorm: 1.03 [ 2:18:36<20:03:50] +[titan] 2025-07-09 15:53:36,671 - root - INFO - step: 10330 loss: 17.4349 memory: 44.58GiB(31.99%) tps: 84,402 tflops: 291.28 mfu: 29.45% global_avg_ntp_loss: 2.7447 global_avg_mtp_loss: 14.6902 +[titan] 2025-07-09 15:53:36,671 - root - INFO - lr: 2.9413e-04 gnorm: 0.91 [ 2:18:40<20:03:45] +[titan] 2025-07-09 15:53:40,567 - root - INFO - step: 10335 loss: 17.2042 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.7011 global_avg_mtp_loss: 14.5031 +[titan] 2025-07-09 15:53:40,567 - root - INFO - lr: 2.9412e-04 gnorm: 0.93 [ 2:18:44<20:03:40] +[titan] 2025-07-09 15:53:44,477 - root - INFO - step: 10340 loss: 17.2550 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7124 global_avg_mtp_loss: 14.5426 +[titan] 2025-07-09 15:53:44,477 - root - INFO - lr: 2.9411e-04 gnorm: 0.94 [ 2:18:48<20:03:35] +[titan] 2025-07-09 15:53:48,390 - root - INFO - step: 10345 loss: 17.3878 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.7388 global_avg_mtp_loss: 14.6490 +[titan] 2025-07-09 15:53:48,391 - root - INFO - lr: 2.9411e-04 gnorm: 1.01 [ 2:18:52<20:03:30] +[titan] 2025-07-09 15:53:51,505 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:53:52,293 - root - INFO - step: 10350 loss: 17.0271 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6787 global_avg_mtp_loss: 14.3484 +[titan] 2025-07-09 15:53:52,293 - root - INFO - lr: 2.9410e-04 gnorm: 0.85 [ 2:18:56<20:03:25] +[titan] 2025-07-09 15:53:56,184 - root - INFO - step: 10355 loss: 17.2989 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.7275 global_avg_mtp_loss: 14.5714 +[titan] 2025-07-09 15:53:56,184 - root - INFO - lr: 2.9409e-04 gnorm: 0.91 [ 2:18:59<20:03:19] +[titan] 2025-07-09 15:54:00,127 - root - INFO - step: 10360 loss: 17.3533 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.7279 global_avg_mtp_loss: 14.6254 +[titan] 2025-07-09 15:54:00,128 - root - INFO - lr: 2.9409e-04 gnorm: 0.94 [ 2:19:03<20:03:15] +[titan] 2025-07-09 15:54:04,223 - root - INFO - step: 10365 loss: 17.2760 memory: 44.58GiB(31.99%) tps: 80,018 tflops: 276.15 mfu: 27.92% global_avg_ntp_loss: 2.7130 global_avg_mtp_loss: 14.5630 +[titan] 2025-07-09 15:54:04,223 - root - INFO - lr: 2.9408e-04 gnorm: 0.87 [ 2:19:07<20:03:11] +[titan] 2025-07-09 15:54:08,109 - root - INFO - step: 10370 loss: 17.2755 memory: 44.58GiB(31.99%) tps: 84,339 tflops: 291.07 mfu: 29.43% global_avg_ntp_loss: 2.7236 global_avg_mtp_loss: 14.5519 +[titan] 2025-07-09 15:54:08,109 - root - INFO - lr: 2.9407e-04 gnorm: 0.94 [ 2:19:11<20:03:06] +[titan] 2025-07-09 15:54:12,013 - root - INFO - step: 10375 loss: 17.2295 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.7127 global_avg_mtp_loss: 14.5167 +[titan] 2025-07-09 15:54:12,014 - root - INFO - lr: 2.9407e-04 gnorm: 0.89 [ 2:19:15<20:03:01] +[titan] 2025-07-09 15:54:15,932 - root - INFO - step: 10380 loss: 17.2181 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6992 global_avg_mtp_loss: 14.5189 +[titan] 2025-07-09 15:54:15,932 - root - INFO - lr: 2.9406e-04 gnorm: 0.96 [ 2:19:19<20:02:56] +[titan] 2025-07-09 15:54:19,855 - root - INFO - step: 10385 loss: 17.4671 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.7463 global_avg_mtp_loss: 14.7208 +[titan] 2025-07-09 15:54:19,855 - root - INFO - lr: 2.9406e-04 gnorm: 0.95 [ 2:19:23<20:02:51] +[titan] 2025-07-09 15:54:23,757 - root - INFO - step: 10390 loss: 17.2344 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.7043 global_avg_mtp_loss: 14.5301 +[titan] 2025-07-09 15:54:23,757 - root - INFO - lr: 2.9405e-04 gnorm: 0.94 [ 2:19:27<20:02:46] +[titan] 2025-07-09 15:54:27,656 - root - INFO - step: 10395 loss: 17.1815 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6900 global_avg_mtp_loss: 14.4915 +[titan] 2025-07-09 15:54:27,656 - root - INFO - lr: 2.9404e-04 gnorm: 0.87 [ 2:19:31<20:02:41] +[titan] 2025-07-09 15:54:30,790 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:54:31,576 - root - INFO - step: 10400 loss: 17.3704 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.7305 global_avg_mtp_loss: 14.6399 +[titan] 2025-07-09 15:54:31,577 - root - INFO - lr: 2.9404e-04 gnorm: 0.90 [ 2:19:35<20:02:36] +[titan] 2025-07-09 15:54:35,485 - root - INFO - step: 10405 loss: 17.1463 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.7050 global_avg_mtp_loss: 14.4413 +[titan] 2025-07-09 15:54:35,485 - root - INFO - lr: 2.9403e-04 gnorm: 0.90 [ 2:19:39<20:02:31] +[titan] 2025-07-09 15:54:39,415 - root - INFO - step: 10410 loss: 17.3998 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.7264 global_avg_mtp_loss: 14.6733 +[titan] 2025-07-09 15:54:39,416 - root - INFO - lr: 2.9402e-04 gnorm: 0.98 [ 2:19:43<20:02:26] +[titan] 2025-07-09 15:54:43,315 - root - INFO - step: 10415 loss: 17.1952 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6967 global_avg_mtp_loss: 14.4986 +[titan] 2025-07-09 15:54:43,315 - root - INFO - lr: 2.9402e-04 gnorm: 1.02 [ 2:19:47<20:02:21] +[titan] 2025-07-09 15:54:47,216 - root - INFO - step: 10420 loss: 17.5632 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.7760 global_avg_mtp_loss: 14.7871 +[titan] 2025-07-09 15:54:47,216 - root - INFO - lr: 2.9401e-04 gnorm: 0.93 [ 2:19:50<20:02:16] +[titan] 2025-07-09 15:54:51,114 - root - INFO - step: 10425 loss: 17.4138 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.7375 global_avg_mtp_loss: 14.6763 +[titan] 2025-07-09 15:54:51,115 - root - INFO - lr: 2.9401e-04 gnorm: 0.94 [ 2:19:54<20:02:11] +[titan] 2025-07-09 15:54:55,033 - root - INFO - step: 10430 loss: 17.4646 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.7518 global_avg_mtp_loss: 14.7128 +[titan] 2025-07-09 15:54:55,033 - root - INFO - lr: 2.9400e-04 gnorm: 0.88 [ 2:19:58<20:02:06] +[titan] 2025-07-09 15:54:59,000 - root - INFO - step: 10435 loss: 17.6035 memory: 44.58GiB(31.99%) tps: 82,606 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 2.7758 global_avg_mtp_loss: 14.8277 +[titan] 2025-07-09 15:54:59,000 - root - INFO - lr: 2.9399e-04 gnorm: 0.92 [ 2:20:02<20:02:01] +[titan] 2025-07-09 15:55:02,924 - root - INFO - step: 10440 loss: 17.2144 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6961 global_avg_mtp_loss: 14.5183 +[titan] 2025-07-09 15:55:02,924 - root - INFO - lr: 2.9399e-04 gnorm: 0.90 [ 2:20:06<20:01:56] +[titan] 2025-07-09 15:55:06,815 - root - INFO - step: 10445 loss: 17.3276 memory: 44.58GiB(31.99%) tps: 84,230 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.7230 global_avg_mtp_loss: 14.6046 +[titan] 2025-07-09 15:55:06,815 - root - INFO - lr: 2.9398e-04 gnorm: 0.97 [ 2:20:10<20:01:51] +[titan] 2025-07-09 15:55:09,975 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:55:10,767 - root - INFO - step: 10450 loss: 17.2293 memory: 44.58GiB(31.99%) tps: 82,908 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.7009 global_avg_mtp_loss: 14.5284 +[titan] 2025-07-09 15:55:10,768 - root - INFO - lr: 2.9397e-04 gnorm: 0.92 [ 2:20:14<20:01:46] +[titan] 2025-07-09 15:55:14,693 - root - INFO - step: 10455 loss: 17.6277 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.7866 global_avg_mtp_loss: 14.8411 +[titan] 2025-07-09 15:55:14,693 - root - INFO - lr: 2.9397e-04 gnorm: 0.99 [ 2:20:18<20:01:41] +[titan] 2025-07-09 15:55:18,584 - root - INFO - step: 10460 loss: 17.1394 memory: 44.58GiB(31.99%) tps: 84,216 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.6940 global_avg_mtp_loss: 14.4454 +[titan] 2025-07-09 15:55:18,585 - root - INFO - lr: 2.9396e-04 gnorm: 0.89 [ 2:20:22<20:01:36] +[titan] 2025-07-09 15:55:22,494 - root - INFO - step: 10465 loss: 17.1816 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6999 global_avg_mtp_loss: 14.4817 +[titan] 2025-07-09 15:55:22,495 - root - INFO - lr: 2.9396e-04 gnorm: 0.96 [ 2:20:26<20:01:31] +[titan] 2025-07-09 15:55:26,396 - root - INFO - step: 10470 loss: 17.7052 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.8099 global_avg_mtp_loss: 14.8953 +[titan] 2025-07-09 15:55:26,396 - root - INFO - lr: 2.9395e-04 gnorm: 0.90 [ 2:20:30<20:01:26] +[titan] 2025-07-09 15:55:30,302 - root - INFO - step: 10475 loss: 17.2800 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.7170 global_avg_mtp_loss: 14.5630 +[titan] 2025-07-09 15:55:30,302 - root - INFO - lr: 2.9394e-04 gnorm: 0.94 [ 2:20:34<20:01:21] +[titan] 2025-07-09 15:55:34,251 - root - INFO - step: 10480 loss: 17.4880 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.7680 global_avg_mtp_loss: 14.7200 +[titan] 2025-07-09 15:55:34,251 - root - INFO - lr: 2.9394e-04 gnorm: 1.06 [ 2:20:37<20:01:16] +[titan] 2025-07-09 15:55:38,177 - root - INFO - step: 10485 loss: 17.1693 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.7061 global_avg_mtp_loss: 14.4632 +[titan] 2025-07-09 15:55:38,177 - root - INFO - lr: 2.9393e-04 gnorm: 0.88 [ 2:20:41<20:01:12] +[titan] 2025-07-09 15:55:42,064 - root - INFO - step: 10490 loss: 17.2591 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.7137 global_avg_mtp_loss: 14.5453 +[titan] 2025-07-09 15:55:42,064 - root - INFO - lr: 2.9392e-04 gnorm: 0.96 [ 2:20:45<20:01:06] +[titan] 2025-07-09 15:55:45,966 - root - INFO - step: 10495 loss: 17.3280 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.7226 global_avg_mtp_loss: 14.6054 +[titan] 2025-07-09 15:55:45,967 - root - INFO - lr: 2.9392e-04 gnorm: 0.97 [ 2:20:49<20:01:01] +[titan] 2025-07-09 15:55:49,112 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:55:49,906 - root - INFO - step: 10500 loss: 17.3106 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.7246 global_avg_mtp_loss: 14.5860 +[titan] 2025-07-09 15:55:49,906 - root - INFO - lr: 2.9391e-04 gnorm: 0.94 [ 2:20:53<20:00:56] +[titan] 2025-07-09 15:55:53,800 - root - INFO - step: 10505 loss: 17.2854 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.7241 global_avg_mtp_loss: 14.5613 +[titan] 2025-07-09 15:55:53,800 - root - INFO - lr: 2.9390e-04 gnorm: 0.86 [ 2:20:57<20:00:51] +[titan] 2025-07-09 15:55:57,720 - root - INFO - step: 10510 loss: 17.5193 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.7618 global_avg_mtp_loss: 14.7575 +[titan] 2025-07-09 15:55:57,721 - root - INFO - lr: 2.9390e-04 gnorm: 0.95 [ 2:21:01<20:00:46] +[titan] 2025-07-09 15:56:01,629 - root - INFO - step: 10515 loss: 17.1117 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6779 global_avg_mtp_loss: 14.4338 +[titan] 2025-07-09 15:56:01,629 - root - INFO - lr: 2.9389e-04 gnorm: 0.87 [ 2:21:05<20:00:41] +[titan] 2025-07-09 15:56:05,570 - root - INFO - step: 10520 loss: 17.1919 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6961 global_avg_mtp_loss: 14.4959 +[titan] 2025-07-09 15:56:05,570 - root - INFO - lr: 2.9389e-04 gnorm: 0.89 [ 2:21:09<20:00:37] +[titan] 2025-07-09 15:56:09,487 - root - INFO - step: 10525 loss: 17.3340 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7233 global_avg_mtp_loss: 14.6106 +[titan] 2025-07-09 15:56:09,487 - root - INFO - lr: 2.9388e-04 gnorm: 0.91 [ 2:21:13<20:00:32] +[titan] 2025-07-09 15:56:13,446 - root - INFO - step: 10530 loss: 17.0722 memory: 44.58GiB(31.99%) tps: 82,757 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.6724 global_avg_mtp_loss: 14.3998 +[titan] 2025-07-09 15:56:13,447 - root - INFO - lr: 2.9387e-04 gnorm: 0.86 [ 2:21:17<20:00:27] +[titan] 2025-07-09 15:56:17,383 - root - INFO - step: 10535 loss: 17.5444 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.7700 global_avg_mtp_loss: 14.7744 +[titan] 2025-07-09 15:56:17,384 - root - INFO - lr: 2.9387e-04 gnorm: 0.86 [ 2:21:21<20:00:22] +[titan] 2025-07-09 15:56:21,281 - root - INFO - step: 10540 loss: 17.3965 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7343 global_avg_mtp_loss: 14.6622 +[titan] 2025-07-09 15:56:21,281 - root - INFO - lr: 2.9386e-04 gnorm: 0.88 [ 2:21:24<20:00:17] +[titan] 2025-07-09 15:56:25,207 - root - INFO - step: 10545 loss: 17.2417 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.7040 global_avg_mtp_loss: 14.5377 +[titan] 2025-07-09 15:56:25,207 - root - INFO - lr: 2.9385e-04 gnorm: 0.97 [ 2:21:28<20:00:12] +[titan] 2025-07-09 15:56:28,356 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:56:29,143 - root - INFO - step: 10550 loss: 17.4691 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7530 global_avg_mtp_loss: 14.7161 +[titan] 2025-07-09 15:56:29,144 - root - INFO - lr: 2.9385e-04 gnorm: 0.95 [ 2:21:32<20:00:08] +[titan] 2025-07-09 15:56:33,044 - root - INFO - step: 10555 loss: 17.0641 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6621 global_avg_mtp_loss: 14.4020 +[titan] 2025-07-09 15:56:33,044 - root - INFO - lr: 2.9384e-04 gnorm: 0.91 [ 2:21:36<20:00:02] +[titan] 2025-07-09 15:56:36,952 - root - INFO - step: 10560 loss: 17.2912 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7181 global_avg_mtp_loss: 14.5731 +[titan] 2025-07-09 15:56:36,952 - root - INFO - lr: 2.9383e-04 gnorm: 0.97 [ 2:21:40<19:59:57] +[titan] 2025-07-09 15:56:40,833 - root - INFO - step: 10565 loss: 17.6338 memory: 44.58GiB(31.99%) tps: 84,427 tflops: 291.37 mfu: 29.46% global_avg_ntp_loss: 2.7775 global_avg_mtp_loss: 14.8563 +[titan] 2025-07-09 15:56:40,833 - root - INFO - lr: 2.9383e-04 gnorm: 0.92 [ 2:21:44<19:59:52] +[titan] 2025-07-09 15:56:44,725 - root - INFO - step: 10570 loss: 17.5265 memory: 44.58GiB(31.99%) tps: 84,216 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.7534 global_avg_mtp_loss: 14.7731 +[titan] 2025-07-09 15:56:44,725 - root - INFO - lr: 2.9382e-04 gnorm: 0.83 [ 2:21:48<19:59:47] +[titan] 2025-07-09 15:56:48,629 - root - INFO - step: 10575 loss: 17.1629 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6948 global_avg_mtp_loss: 14.4681 +[titan] 2025-07-09 15:56:48,629 - root - INFO - lr: 2.9381e-04 gnorm: 0.83 [ 2:21:52<19:59:42] +[titan] 2025-07-09 15:56:52,538 - root - INFO - step: 10580 loss: 17.2729 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.7243 global_avg_mtp_loss: 14.5486 +[titan] 2025-07-09 15:56:52,538 - root - INFO - lr: 2.9381e-04 gnorm: 0.96 [ 2:21:56<19:59:37] +[titan] 2025-07-09 15:56:56,447 - root - INFO - step: 10585 loss: 17.3331 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7232 global_avg_mtp_loss: 14.6099 +[titan] 2025-07-09 15:56:56,447 - root - INFO - lr: 2.9380e-04 gnorm: 0.92 [ 2:22:00<19:59:32] +[titan] 2025-07-09 15:57:00,395 - root - INFO - step: 10590 loss: 17.2350 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.7237 global_avg_mtp_loss: 14.5113 +[titan] 2025-07-09 15:57:00,395 - root - INFO - lr: 2.9380e-04 gnorm: 0.86 [ 2:22:04<19:59:27] +[titan] 2025-07-09 15:57:04,347 - root - INFO - step: 10595 loss: 17.3498 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.7252 global_avg_mtp_loss: 14.6247 +[titan] 2025-07-09 15:57:04,347 - root - INFO - lr: 2.9379e-04 gnorm: 0.94 [ 2:22:08<19:59:23] +[titan] 2025-07-09 15:57:07,503 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:57:08,295 - root - INFO - step: 10600 loss: 17.0494 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.6643 global_avg_mtp_loss: 14.3851 +[titan] 2025-07-09 15:57:08,295 - root - INFO - lr: 2.9378e-04 gnorm: 0.97 [ 2:22:11<19:59:18] +[titan] 2025-07-09 15:57:12,207 - root - INFO - step: 10605 loss: 17.3469 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.7230 global_avg_mtp_loss: 14.6239 +[titan] 2025-07-09 15:57:12,208 - root - INFO - lr: 2.9378e-04 gnorm: 0.93 [ 2:22:15<19:59:13] +[titan] 2025-07-09 15:57:16,112 - root - INFO - step: 10610 loss: 17.2945 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.7224 global_avg_mtp_loss: 14.5721 +[titan] 2025-07-09 15:57:16,113 - root - INFO - lr: 2.9377e-04 gnorm: 0.87 [ 2:22:19<19:59:08] +[titan] 2025-07-09 15:57:20,020 - root - INFO - step: 10615 loss: 17.0002 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6723 global_avg_mtp_loss: 14.3280 +[titan] 2025-07-09 15:57:20,020 - root - INFO - lr: 2.9376e-04 gnorm: 0.91 [ 2:22:23<19:59:03] +[titan] 2025-07-09 15:57:23,911 - root - INFO - step: 10620 loss: 17.2709 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.7078 global_avg_mtp_loss: 14.5631 +[titan] 2025-07-09 15:57:23,911 - root - INFO - lr: 2.9376e-04 gnorm: 0.94 [ 2:22:27<19:58:58] +[titan] 2025-07-09 15:57:27,830 - root - INFO - step: 10625 loss: 17.2681 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.7096 global_avg_mtp_loss: 14.5586 +[titan] 2025-07-09 15:57:27,830 - root - INFO - lr: 2.9375e-04 gnorm: 0.87 [ 2:22:31<19:58:53] +[titan] 2025-07-09 15:57:31,811 - root - INFO - step: 10630 loss: 17.3565 memory: 44.58GiB(31.99%) tps: 82,327 tflops: 284.12 mfu: 28.73% global_avg_ntp_loss: 2.7307 global_avg_mtp_loss: 14.6258 +[titan] 2025-07-09 15:57:31,811 - root - INFO - lr: 2.9374e-04 gnorm: 0.91 [ 2:22:35<19:58:49] +[titan] 2025-07-09 15:57:35,709 - root - INFO - step: 10635 loss: 17.0486 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6808 global_avg_mtp_loss: 14.3678 +[titan] 2025-07-09 15:57:35,709 - root - INFO - lr: 2.9374e-04 gnorm: 0.83 [ 2:22:39<19:58:43] +[titan] 2025-07-09 15:57:39,645 - root - INFO - step: 10640 loss: 17.1218 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.6892 global_avg_mtp_loss: 14.4326 +[titan] 2025-07-09 15:57:39,645 - root - INFO - lr: 2.9373e-04 gnorm: 0.87 [ 2:22:43<19:58:39] +[titan] 2025-07-09 15:57:43,564 - root - INFO - step: 10645 loss: 17.3770 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7431 global_avg_mtp_loss: 14.6339 +[titan] 2025-07-09 15:57:43,564 - root - INFO - lr: 2.9372e-04 gnorm: 0.91 [ 2:22:47<19:58:34] +[titan] 2025-07-09 15:57:46,668 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:57:47,462 - root - INFO - step: 10650 loss: 17.2696 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.7238 global_avg_mtp_loss: 14.5458 +[titan] 2025-07-09 15:57:47,462 - root - INFO - lr: 2.9372e-04 gnorm: 1.07 [ 2:22:51<19:58:29] +[titan] 2025-07-09 15:57:51,410 - root - INFO - step: 10655 loss: 17.1552 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.6836 global_avg_mtp_loss: 14.4716 +[titan] 2025-07-09 15:57:51,410 - root - INFO - lr: 2.9371e-04 gnorm: 0.92 [ 2:22:55<19:58:24] +[titan] 2025-07-09 15:57:55,385 - root - INFO - step: 10660 loss: 17.5446 memory: 44.58GiB(31.99%) tps: 82,440 tflops: 284.52 mfu: 28.77% global_avg_ntp_loss: 2.7703 global_avg_mtp_loss: 14.7743 +[titan] 2025-07-09 15:57:55,385 - root - INFO - lr: 2.9371e-04 gnorm: 0.90 [ 2:22:59<19:58:20] +[titan] 2025-07-09 15:57:59,317 - root - INFO - step: 10665 loss: 17.4689 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.7555 global_avg_mtp_loss: 14.7134 +[titan] 2025-07-09 15:57:59,317 - root - INFO - lr: 2.9370e-04 gnorm: 0.97 [ 2:23:03<19:58:15] +[titan] 2025-07-09 15:58:03,219 - root - INFO - step: 10670 loss: 17.2853 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.7196 global_avg_mtp_loss: 14.5657 +[titan] 2025-07-09 15:58:03,219 - root - INFO - lr: 2.9369e-04 gnorm: 0.86 [ 2:23:06<19:58:10] +[titan] 2025-07-09 15:58:07,167 - root - INFO - step: 10675 loss: 17.3147 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.7320 global_avg_mtp_loss: 14.5827 +[titan] 2025-07-09 15:58:07,167 - root - INFO - lr: 2.9369e-04 gnorm: 0.93 [ 2:23:10<19:58:05] +[titan] 2025-07-09 15:58:11,062 - root - INFO - step: 10680 loss: 17.2472 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.7134 global_avg_mtp_loss: 14.5338 +[titan] 2025-07-09 15:58:11,063 - root - INFO - lr: 2.9368e-04 gnorm: 0.89 [ 2:23:14<19:58:00] +[titan] 2025-07-09 15:58:14,972 - root - INFO - step: 10685 loss: 17.0863 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6760 global_avg_mtp_loss: 14.4104 +[titan] 2025-07-09 15:58:14,972 - root - INFO - lr: 2.9367e-04 gnorm: 0.89 [ 2:23:18<19:57:55] +[titan] 2025-07-09 15:58:18,893 - root - INFO - step: 10690 loss: 17.4987 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.7558 global_avg_mtp_loss: 14.7429 +[titan] 2025-07-09 15:58:18,894 - root - INFO - lr: 2.9367e-04 gnorm: 0.93 [ 2:23:22<19:57:50] +[titan] 2025-07-09 15:58:22,802 - root - INFO - step: 10695 loss: 17.5334 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.7632 global_avg_mtp_loss: 14.7701 +[titan] 2025-07-09 15:58:22,803 - root - INFO - lr: 2.9366e-04 gnorm: 0.89 [ 2:23:26<19:57:45] +[titan] 2025-07-09 15:58:25,911 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:58:26,700 - root - INFO - step: 10700 loss: 17.4523 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.7453 global_avg_mtp_loss: 14.7070 +[titan] 2025-07-09 15:58:26,700 - root - INFO - lr: 2.9365e-04 gnorm: 0.89 [ 2:23:30<19:57:40] +[titan] 2025-07-09 15:58:30,682 - root - INFO - step: 10705 loss: 17.4317 memory: 44.58GiB(31.99%) tps: 82,305 tflops: 284.05 mfu: 28.72% global_avg_ntp_loss: 2.7534 global_avg_mtp_loss: 14.6782 +[titan] 2025-07-09 15:58:30,682 - root - INFO - lr: 2.9365e-04 gnorm: 0.96 [ 2:23:34<19:57:36] +[titan] 2025-07-09 15:58:34,636 - root - INFO - step: 10710 loss: 17.8328 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.8507 global_avg_mtp_loss: 14.9822 +[titan] 2025-07-09 15:58:34,636 - root - INFO - lr: 2.9364e-04 gnorm: 1.00 [ 2:23:38<19:57:31] +[titan] 2025-07-09 15:58:38,553 - root - INFO - step: 10715 loss: 17.5939 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.7765 global_avg_mtp_loss: 14.8174 +[titan] 2025-07-09 15:58:38,553 - root - INFO - lr: 2.9363e-04 gnorm: 0.90 [ 2:23:42<19:57:26] +[titan] 2025-07-09 15:58:42,522 - root - INFO - step: 10720 loss: 17.4724 memory: 44.58GiB(31.99%) tps: 82,559 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.7497 global_avg_mtp_loss: 14.7228 +[titan] 2025-07-09 15:58:42,522 - root - INFO - lr: 2.9363e-04 gnorm: 0.90 [ 2:23:46<19:57:22] +[titan] 2025-07-09 15:58:46,424 - root - INFO - step: 10725 loss: 17.4676 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.7608 global_avg_mtp_loss: 14.7068 +[titan] 2025-07-09 15:58:46,424 - root - INFO - lr: 2.9362e-04 gnorm: 0.90 [ 2:23:50<19:57:17] +[titan] 2025-07-09 15:58:50,338 - root - INFO - step: 10730 loss: 17.2569 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.7126 global_avg_mtp_loss: 14.5443 +[titan] 2025-07-09 15:58:50,339 - root - INFO - lr: 2.9361e-04 gnorm: 0.88 [ 2:23:54<19:57:12] +[titan] 2025-07-09 15:58:54,251 - root - INFO - step: 10735 loss: 17.4746 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.7518 global_avg_mtp_loss: 14.7227 +[titan] 2025-07-09 15:58:54,251 - root - INFO - lr: 2.9361e-04 gnorm: 0.93 [ 2:23:57<19:57:07] +[titan] 2025-07-09 15:58:58,188 - root - INFO - step: 10740 loss: 17.4528 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.7499 global_avg_mtp_loss: 14.7029 +[titan] 2025-07-09 15:58:58,188 - root - INFO - lr: 2.9360e-04 gnorm: 0.92 [ 2:24:01<19:57:02] +[titan] 2025-07-09 15:59:02,096 - root - INFO - step: 10745 loss: 17.0805 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6906 global_avg_mtp_loss: 14.3899 +[titan] 2025-07-09 15:59:02,097 - root - INFO - lr: 2.9360e-04 gnorm: 0.86 [ 2:24:05<19:56:57] +[titan] 2025-07-09 15:59:05,298 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:59:06,101 - root - INFO - step: 10750 loss: 17.3244 memory: 44.58GiB(31.99%) tps: 81,838 tflops: 282.44 mfu: 28.56% global_avg_ntp_loss: 2.7202 global_avg_mtp_loss: 14.6043 +[titan] 2025-07-09 15:59:06,101 - root - INFO - lr: 2.9359e-04 gnorm: 0.95 [ 2:24:09<19:56:53] +[titan] 2025-07-09 15:59:07,806 - root - INFO - Dumping profiler traces at step 10752 +[titan] 2025-07-09 15:59:07,839 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 15:59:10,198 - root - INFO - step: 10755 loss: 17.4833 memory: 44.58GiB(31.99%) tps: 79,972 tflops: 276.00 mfu: 27.91% global_avg_ntp_loss: 2.7459 global_avg_mtp_loss: 14.7374 +[titan] 2025-07-09 15:59:10,199 - root - INFO - lr: 2.9358e-04 gnorm: 0.91 [ 2:24:13<19:56:49] +[titan] 2025-07-09 15:59:14,154 - root - INFO - step: 10760 loss: 17.2632 memory: 44.58GiB(31.99%) tps: 82,855 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.7219 global_avg_mtp_loss: 14.5413 +[titan] 2025-07-09 15:59:14,154 - root - INFO - lr: 2.9358e-04 gnorm: 0.86 [ 2:24:17<19:56:45] +[titan] 2025-07-09 15:59:18,051 - root - INFO - step: 10765 loss: 17.3372 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.7395 global_avg_mtp_loss: 14.5977 +[titan] 2025-07-09 15:59:18,052 - root - INFO - lr: 2.9357e-04 gnorm: 0.93 [ 2:24:21<19:56:40] +[titan] 2025-07-09 15:59:21,967 - root - INFO - step: 10770 loss: 17.4137 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.7382 global_avg_mtp_loss: 14.6755 +[titan] 2025-07-09 15:59:21,967 - root - INFO - lr: 2.9356e-04 gnorm: 0.93 [ 2:24:25<19:56:35] +[titan] 2025-07-09 15:59:25,863 - root - INFO - step: 10775 loss: 17.3560 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.7263 global_avg_mtp_loss: 14.6297 +[titan] 2025-07-09 15:59:25,863 - root - INFO - lr: 2.9356e-04 gnorm: 0.91 [ 2:24:29<19:56:30] +[titan] 2025-07-09 15:59:29,836 - root - INFO - step: 10780 loss: 17.3517 memory: 44.58GiB(31.99%) tps: 82,482 tflops: 284.66 mfu: 28.78% global_avg_ntp_loss: 2.7289 global_avg_mtp_loss: 14.6228 +[titan] 2025-07-09 15:59:29,836 - root - INFO - lr: 2.9355e-04 gnorm: 0.88 [ 2:24:33<19:56:25] +[titan] 2025-07-09 15:59:33,730 - root - INFO - step: 10785 loss: 17.4673 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.7590 global_avg_mtp_loss: 14.7084 +[titan] 2025-07-09 15:59:33,730 - root - INFO - lr: 2.9354e-04 gnorm: 0.80 [ 2:24:37<19:56:20] +[titan] 2025-07-09 15:59:37,630 - root - INFO - step: 10790 loss: 17.3468 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.7356 global_avg_mtp_loss: 14.6112 +[titan] 2025-07-09 15:59:37,630 - root - INFO - lr: 2.9354e-04 gnorm: 0.85 [ 2:24:41<19:56:15] +[titan] 2025-07-09 15:59:41,559 - root - INFO - step: 10795 loss: 17.2025 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.7036 global_avg_mtp_loss: 14.4989 +[titan] 2025-07-09 15:59:41,560 - root - INFO - lr: 2.9353e-04 gnorm: 0.87 [ 2:24:45<19:56:10] +[titan] 2025-07-09 15:59:44,676 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 15:59:45,496 - root - INFO - step: 10800 loss: 17.3252 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.7413 global_avg_mtp_loss: 14.5839 +[titan] 2025-07-09 15:59:45,496 - root - INFO - lr: 2.9352e-04 gnorm: 0.91 [ 2:24:49<19:56:06] +[titan] 2025-07-09 15:59:49,399 - root - INFO - step: 10805 loss: 17.2090 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7001 global_avg_mtp_loss: 14.5089 +[titan] 2025-07-09 15:59:49,399 - root - INFO - lr: 2.9352e-04 gnorm: 0.90 [ 2:24:53<19:56:01] +[titan] 2025-07-09 15:59:53,281 - root - INFO - step: 10810 loss: 17.3471 memory: 44.58GiB(31.99%) tps: 84,416 tflops: 291.33 mfu: 29.46% global_avg_ntp_loss: 2.7164 global_avg_mtp_loss: 14.6307 +[titan] 2025-07-09 15:59:53,282 - root - INFO - lr: 2.9351e-04 gnorm: 0.85 [ 2:24:56<19:55:55] +[titan] 2025-07-09 15:59:57,209 - root - INFO - step: 10815 loss: 17.3150 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.7190 global_avg_mtp_loss: 14.5961 +[titan] 2025-07-09 15:59:57,209 - root - INFO - lr: 2.9350e-04 gnorm: 0.88 [ 2:25:00<19:55:51] +[titan] 2025-07-09 16:00:01,122 - root - INFO - step: 10820 loss: 17.1276 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6905 global_avg_mtp_loss: 14.4371 +[titan] 2025-07-09 16:00:01,122 - root - INFO - lr: 2.9350e-04 gnorm: 0.92 [ 2:25:04<19:55:46] +[titan] 2025-07-09 16:00:05,019 - root - INFO - step: 10825 loss: 17.0326 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6695 global_avg_mtp_loss: 14.3631 +[titan] 2025-07-09 16:00:05,020 - root - INFO - lr: 2.9349e-04 gnorm: 1.02 [ 2:25:08<19:55:41] +[titan] 2025-07-09 16:00:08,929 - root - INFO - step: 10830 loss: 17.3211 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.7177 global_avg_mtp_loss: 14.6034 +[titan] 2025-07-09 16:00:08,929 - root - INFO - lr: 2.9348e-04 gnorm: 0.94 [ 2:25:12<19:55:36] +[titan] 2025-07-09 16:00:12,862 - root - INFO - step: 10835 loss: 17.3083 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.7087 global_avg_mtp_loss: 14.5996 +[titan] 2025-07-09 16:00:12,862 - root - INFO - lr: 2.9348e-04 gnorm: 0.93 [ 2:25:16<19:55:31] +[titan] 2025-07-09 16:00:16,778 - root - INFO - step: 10840 loss: 17.4204 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.7306 global_avg_mtp_loss: 14.6898 +[titan] 2025-07-09 16:00:16,778 - root - INFO - lr: 2.9347e-04 gnorm: 0.86 [ 2:25:20<19:55:26] +[titan] 2025-07-09 16:00:20,687 - root - INFO - step: 10845 loss: 17.2972 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.7270 global_avg_mtp_loss: 14.5702 +[titan] 2025-07-09 16:00:20,687 - root - INFO - lr: 2.9346e-04 gnorm: 0.84 [ 2:25:24<19:55:21] +[titan] 2025-07-09 16:00:23,801 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:00:24,590 - root - INFO - step: 10850 loss: 17.3993 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.7606 global_avg_mtp_loss: 14.6387 +[titan] 2025-07-09 16:00:24,590 - root - INFO - lr: 2.9346e-04 gnorm: 0.85 [ 2:25:28<19:55:16] +[titan] 2025-07-09 16:00:28,524 - root - INFO - step: 10855 loss: 17.0600 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6940 global_avg_mtp_loss: 14.3659 +[titan] 2025-07-09 16:00:28,524 - root - INFO - lr: 2.9345e-04 gnorm: 0.90 [ 2:25:32<19:55:11] +[titan] 2025-07-09 16:00:32,434 - root - INFO - step: 10860 loss: 17.4460 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.7387 global_avg_mtp_loss: 14.7072 +[titan] 2025-07-09 16:00:32,434 - root - INFO - lr: 2.9344e-04 gnorm: 0.82 [ 2:25:36<19:55:06] +[titan] 2025-07-09 16:00:36,325 - root - INFO - step: 10865 loss: 17.5553 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.7671 global_avg_mtp_loss: 14.7882 +[titan] 2025-07-09 16:00:36,325 - root - INFO - lr: 2.9344e-04 gnorm: 0.87 [ 2:25:39<19:55:01] +[titan] 2025-07-09 16:00:40,220 - root - INFO - step: 10870 loss: 17.2962 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.7280 global_avg_mtp_loss: 14.5682 +[titan] 2025-07-09 16:00:40,221 - root - INFO - lr: 2.9343e-04 gnorm: 0.87 [ 2:25:43<19:54:56] +[titan] 2025-07-09 16:00:44,154 - root - INFO - step: 10875 loss: 17.6335 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.7861 global_avg_mtp_loss: 14.8474 +[titan] 2025-07-09 16:00:44,154 - root - INFO - lr: 2.9342e-04 gnorm: 0.84 [ 2:25:47<19:54:51] +[titan] 2025-07-09 16:00:48,055 - root - INFO - step: 10880 loss: 17.2914 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.7167 global_avg_mtp_loss: 14.5746 +[titan] 2025-07-09 16:00:48,056 - root - INFO - lr: 2.9342e-04 gnorm: 0.89 [ 2:25:51<19:54:46] +[titan] 2025-07-09 16:00:51,949 - root - INFO - step: 10885 loss: 17.3967 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.7327 global_avg_mtp_loss: 14.6641 +[titan] 2025-07-09 16:00:51,950 - root - INFO - lr: 2.9341e-04 gnorm: 0.82 [ 2:25:55<19:54:41] +[titan] 2025-07-09 16:00:55,857 - root - INFO - step: 10890 loss: 17.1342 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6924 global_avg_mtp_loss: 14.4417 +[titan] 2025-07-09 16:00:55,858 - root - INFO - lr: 2.9340e-04 gnorm: 0.88 [ 2:25:59<19:54:36] +[titan] 2025-07-09 16:00:59,838 - root - INFO - step: 10895 loss: 17.1769 memory: 44.58GiB(31.99%) tps: 82,324 tflops: 284.11 mfu: 28.73% global_avg_ntp_loss: 2.6942 global_avg_mtp_loss: 14.4827 +[titan] 2025-07-09 16:00:59,838 - root - INFO - lr: 2.9340e-04 gnorm: 1.04 [ 2:26:03<19:54:32] +[titan] 2025-07-09 16:01:02,969 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:01:03,763 - root - INFO - step: 10900 loss: 17.4025 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.7429 global_avg_mtp_loss: 14.6596 +[titan] 2025-07-09 16:01:03,763 - root - INFO - lr: 2.9339e-04 gnorm: 0.90 [ 2:26:07<19:54:27] +[titan] 2025-07-09 16:01:07,664 - root - INFO - step: 10905 loss: 17.4182 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.7384 global_avg_mtp_loss: 14.6798 +[titan] 2025-07-09 16:01:07,664 - root - INFO - lr: 2.9338e-04 gnorm: 1.02 [ 2:26:11<19:54:22] +[titan] 2025-07-09 16:01:11,552 - root - INFO - step: 10910 loss: 17.4923 memory: 44.58GiB(31.99%) tps: 84,283 tflops: 290.87 mfu: 29.41% global_avg_ntp_loss: 2.7590 global_avg_mtp_loss: 14.7333 +[titan] 2025-07-09 16:01:11,552 - root - INFO - lr: 2.9338e-04 gnorm: 0.98 [ 2:26:15<19:54:17] +[titan] 2025-07-09 16:01:15,441 - root - INFO - step: 10915 loss: 17.2133 memory: 44.58GiB(31.99%) tps: 84,262 tflops: 290.80 mfu: 29.40% global_avg_ntp_loss: 2.7030 global_avg_mtp_loss: 14.5103 +[titan] 2025-07-09 16:01:15,441 - root - INFO - lr: 2.9337e-04 gnorm: 0.84 [ 2:26:19<19:54:12] +[titan] 2025-07-09 16:01:19,345 - root - INFO - step: 10920 loss: 17.4086 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.7623 global_avg_mtp_loss: 14.6463 +[titan] 2025-07-09 16:01:19,345 - root - INFO - lr: 2.9336e-04 gnorm: 0.95 [ 2:26:23<19:54:07] +[titan] 2025-07-09 16:01:23,230 - root - INFO - step: 10925 loss: 17.0425 memory: 44.58GiB(31.99%) tps: 84,353 tflops: 291.11 mfu: 29.44% global_avg_ntp_loss: 2.6662 global_avg_mtp_loss: 14.3763 +[titan] 2025-07-09 16:01:23,230 - root - INFO - lr: 2.9336e-04 gnorm: 0.95 [ 2:26:26<19:54:02] +[titan] 2025-07-09 16:01:27,150 - root - INFO - step: 10930 loss: 17.4568 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.7465 global_avg_mtp_loss: 14.7103 +[titan] 2025-07-09 16:01:27,150 - root - INFO - lr: 2.9335e-04 gnorm: 0.93 [ 2:26:30<19:53:57] +[titan] 2025-07-09 16:01:31,050 - root - INFO - step: 10935 loss: 17.0737 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6834 global_avg_mtp_loss: 14.3903 +[titan] 2025-07-09 16:01:31,050 - root - INFO - lr: 2.9335e-04 gnorm: 0.91 [ 2:26:34<19:53:52] +[titan] 2025-07-09 16:01:34,945 - root - INFO - step: 10940 loss: 17.1328 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6824 global_avg_mtp_loss: 14.4504 +[titan] 2025-07-09 16:01:34,945 - root - INFO - lr: 2.9334e-04 gnorm: 0.87 [ 2:26:38<19:53:47] +[titan] 2025-07-09 16:01:38,850 - root - INFO - step: 10945 loss: 17.4767 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.7413 global_avg_mtp_loss: 14.7354 +[titan] 2025-07-09 16:01:38,850 - root - INFO - lr: 2.9333e-04 gnorm: 1.07 [ 2:26:42<19:53:42] +[titan] 2025-07-09 16:01:41,985 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:01:42,767 - root - INFO - step: 10950 loss: 17.2693 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6992 global_avg_mtp_loss: 14.5701 +[titan] 2025-07-09 16:01:42,767 - root - INFO - lr: 2.9333e-04 gnorm: 0.94 [ 2:26:46<19:53:37] +[titan] 2025-07-09 16:01:46,712 - root - INFO - step: 10955 loss: 17.2481 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.7272 global_avg_mtp_loss: 14.5210 +[titan] 2025-07-09 16:01:46,712 - root - INFO - lr: 2.9332e-04 gnorm: 0.97 [ 2:26:50<19:53:32] +[titan] 2025-07-09 16:01:50,599 - root - INFO - step: 10960 loss: 16.9890 memory: 44.58GiB(31.99%) tps: 84,296 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.6608 global_avg_mtp_loss: 14.3281 +[titan] 2025-07-09 16:01:50,599 - root - INFO - lr: 2.9331e-04 gnorm: 0.84 [ 2:26:54<19:53:27] +[titan] 2025-07-09 16:01:54,489 - root - INFO - step: 10965 loss: 17.3371 memory: 44.58GiB(31.99%) tps: 84,258 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.7171 global_avg_mtp_loss: 14.6200 +[titan] 2025-07-09 16:01:54,489 - root - INFO - lr: 2.9331e-04 gnorm: 0.93 [ 2:26:58<19:53:22] +[titan] 2025-07-09 16:01:58,434 - root - INFO - step: 10970 loss: 17.4009 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.7313 global_avg_mtp_loss: 14.6696 +[titan] 2025-07-09 16:01:58,434 - root - INFO - lr: 2.9330e-04 gnorm: 0.89 [ 2:27:02<19:53:18] +[titan] 2025-07-09 16:02:02,346 - root - INFO - step: 10975 loss: 17.5418 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7629 global_avg_mtp_loss: 14.7789 +[titan] 2025-07-09 16:02:02,347 - root - INFO - lr: 2.9329e-04 gnorm: 0.98 [ 2:27:06<19:53:13] +[titan] 2025-07-09 16:02:06,262 - root - INFO - step: 10980 loss: 17.2322 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.7150 global_avg_mtp_loss: 14.5172 +[titan] 2025-07-09 16:02:06,262 - root - INFO - lr: 2.9329e-04 gnorm: 0.87 [ 2:27:09<19:53:08] +[titan] 2025-07-09 16:02:10,156 - root - INFO - step: 10985 loss: 17.6037 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.7798 global_avg_mtp_loss: 14.8238 +[titan] 2025-07-09 16:02:10,156 - root - INFO - lr: 2.9328e-04 gnorm: 1.00 [ 2:27:13<19:53:03] +[titan] 2025-07-09 16:02:14,065 - root - INFO - step: 10990 loss: 17.6590 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7894 global_avg_mtp_loss: 14.8696 +[titan] 2025-07-09 16:02:14,065 - root - INFO - lr: 2.9327e-04 gnorm: 0.91 [ 2:27:17<19:52:58] +[titan] 2025-07-09 16:02:18,014 - root - INFO - step: 10995 loss: 17.3573 memory: 44.58GiB(31.99%) tps: 82,987 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.7381 global_avg_mtp_loss: 14.6192 +[titan] 2025-07-09 16:02:18,014 - root - INFO - lr: 2.9327e-04 gnorm: 0.92 [ 2:27:21<19:52:53] +[titan] 2025-07-09 16:02:21,176 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:02:21,978 - root - INFO - step: 11000 loss: 17.4141 memory: 44.58GiB(31.99%) tps: 82,664 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.7465 global_avg_mtp_loss: 14.6676 +[titan] 2025-07-09 16:02:21,978 - root - INFO - lr: 2.9326e-04 gnorm: 0.87 [ 2:27:25<19:52:49] +[titan] 2025-07-09 16:02:25,923 - root - INFO - step: 11005 loss: 17.2435 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.7126 global_avg_mtp_loss: 14.5309 +[titan] 2025-07-09 16:02:25,924 - root - INFO - lr: 2.9325e-04 gnorm: 0.91 [ 2:27:29<19:52:44] +[titan] 2025-07-09 16:02:29,933 - root - INFO - step: 11010 loss: 17.3482 memory: 44.58GiB(31.99%) tps: 81,726 tflops: 282.05 mfu: 28.52% global_avg_ntp_loss: 2.7250 global_avg_mtp_loss: 14.6232 +[titan] 2025-07-09 16:02:29,934 - root - INFO - lr: 2.9325e-04 gnorm: 0.87 [ 2:27:33<19:52:40] +[titan] 2025-07-09 16:02:33,834 - root - INFO - step: 11015 loss: 17.2607 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6980 global_avg_mtp_loss: 14.5627 +[titan] 2025-07-09 16:02:33,834 - root - INFO - lr: 2.9324e-04 gnorm: 0.91 [ 2:27:37<19:52:35] +[titan] 2025-07-09 16:02:37,816 - root - INFO - step: 11020 loss: 17.4471 memory: 44.58GiB(31.99%) tps: 82,298 tflops: 284.03 mfu: 28.72% global_avg_ntp_loss: 2.7432 global_avg_mtp_loss: 14.7039 +[titan] 2025-07-09 16:02:37,817 - root - INFO - lr: 2.9323e-04 gnorm: 0.86 [ 2:27:41<19:52:31] +[titan] 2025-07-09 16:02:41,776 - root - INFO - step: 11025 loss: 17.0660 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.6832 global_avg_mtp_loss: 14.3828 +[titan] 2025-07-09 16:02:41,776 - root - INFO - lr: 2.9322e-04 gnorm: 0.84 [ 2:27:45<19:52:26] +[titan] 2025-07-09 16:02:45,744 - root - INFO - step: 11030 loss: 17.0485 memory: 44.58GiB(31.99%) tps: 82,596 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.6785 global_avg_mtp_loss: 14.3700 +[titan] 2025-07-09 16:02:45,744 - root - INFO - lr: 2.9322e-04 gnorm: 0.85 [ 2:27:49<19:52:22] +[titan] 2025-07-09 16:02:49,657 - root - INFO - step: 11035 loss: 17.2013 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7001 global_avg_mtp_loss: 14.5012 +[titan] 2025-07-09 16:02:49,657 - root - INFO - lr: 2.9321e-04 gnorm: 0.88 [ 2:27:53<19:52:17] +[titan] 2025-07-09 16:02:53,564 - root - INFO - step: 11040 loss: 17.4440 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.7451 global_avg_mtp_loss: 14.6989 +[titan] 2025-07-09 16:02:53,564 - root - INFO - lr: 2.9320e-04 gnorm: 0.90 [ 2:27:57<19:52:12] +[titan] 2025-07-09 16:02:57,519 - root - INFO - step: 11045 loss: 17.4728 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.7453 global_avg_mtp_loss: 14.7275 +[titan] 2025-07-09 16:02:57,520 - root - INFO - lr: 2.9320e-04 gnorm: 0.89 [ 2:28:01<19:52:07] +[titan] 2025-07-09 16:03:00,661 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:03:01,451 - root - INFO - step: 11050 loss: 17.4355 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.7536 global_avg_mtp_loss: 14.6819 +[titan] 2025-07-09 16:03:01,451 - root - INFO - lr: 2.9319e-04 gnorm: 0.90 [ 2:28:05<19:52:03] +[titan] 2025-07-09 16:03:05,379 - root - INFO - step: 11055 loss: 17.3024 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.7066 global_avg_mtp_loss: 14.5958 +[titan] 2025-07-09 16:03:05,379 - root - INFO - lr: 2.9318e-04 gnorm: 0.92 [ 2:28:09<19:51:58] +[titan] 2025-07-09 16:03:09,287 - root - INFO - step: 11060 loss: 17.7946 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.8127 global_avg_mtp_loss: 14.9819 +[titan] 2025-07-09 16:03:09,287 - root - INFO - lr: 2.9318e-04 gnorm: 0.87 [ 2:28:12<19:51:53] +[titan] 2025-07-09 16:03:13,202 - root - INFO - step: 11065 loss: 17.3544 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.7394 global_avg_mtp_loss: 14.6150 +[titan] 2025-07-09 16:03:13,202 - root - INFO - lr: 2.9317e-04 gnorm: 0.90 [ 2:28:16<19:51:48] +[titan] 2025-07-09 16:03:17,110 - root - INFO - step: 11070 loss: 17.1389 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6959 global_avg_mtp_loss: 14.4430 +[titan] 2025-07-09 16:03:17,110 - root - INFO - lr: 2.9316e-04 gnorm: 0.87 [ 2:28:20<19:51:43] +[titan] 2025-07-09 16:03:20,998 - root - INFO - step: 11075 loss: 17.5233 memory: 44.58GiB(31.99%) tps: 84,283 tflops: 290.87 mfu: 29.41% global_avg_ntp_loss: 2.7616 global_avg_mtp_loss: 14.7617 +[titan] 2025-07-09 16:03:20,998 - root - INFO - lr: 2.9316e-04 gnorm: 0.89 [ 2:28:24<19:51:38] +[titan] 2025-07-09 16:03:24,911 - root - INFO - step: 11080 loss: 17.5296 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7687 global_avg_mtp_loss: 14.7608 +[titan] 2025-07-09 16:03:24,911 - root - INFO - lr: 2.9315e-04 gnorm: 0.91 [ 2:28:28<19:51:33] +[titan] 2025-07-09 16:03:28,820 - root - INFO - step: 11085 loss: 17.1256 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6846 global_avg_mtp_loss: 14.4410 +[titan] 2025-07-09 16:03:28,821 - root - INFO - lr: 2.9314e-04 gnorm: 0.94 [ 2:28:32<19:51:28] +[titan] 2025-07-09 16:03:32,741 - root - INFO - step: 11090 loss: 17.1842 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.7117 global_avg_mtp_loss: 14.4724 +[titan] 2025-07-09 16:03:32,741 - root - INFO - lr: 2.9314e-04 gnorm: 0.86 [ 2:28:36<19:51:23] +[titan] 2025-07-09 16:03:36,648 - root - INFO - step: 11095 loss: 17.3414 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.7303 global_avg_mtp_loss: 14.6111 +[titan] 2025-07-09 16:03:36,648 - root - INFO - lr: 2.9313e-04 gnorm: 0.91 [ 2:28:40<19:51:18] +[titan] 2025-07-09 16:03:39,764 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:03:40,558 - root - INFO - step: 11100 loss: 17.0952 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.6855 global_avg_mtp_loss: 14.4097 +[titan] 2025-07-09 16:03:40,559 - root - INFO - lr: 2.9312e-04 gnorm: 0.88 [ 2:28:44<19:51:14] +[titan] 2025-07-09 16:03:44,511 - root - INFO - step: 11105 loss: 17.3717 memory: 44.58GiB(31.99%) tps: 82,906 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.7451 global_avg_mtp_loss: 14.6265 +[titan] 2025-07-09 16:03:44,511 - root - INFO - lr: 2.9312e-04 gnorm: 1.01 [ 2:28:48<19:51:09] +[titan] 2025-07-09 16:03:48,436 - root - INFO - step: 11110 loss: 17.2215 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.7036 global_avg_mtp_loss: 14.5179 +[titan] 2025-07-09 16:03:48,436 - root - INFO - lr: 2.9311e-04 gnorm: 0.87 [ 2:28:52<19:51:04] +[titan] 2025-07-09 16:03:52,370 - root - INFO - step: 11115 loss: 17.2249 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.7038 global_avg_mtp_loss: 14.5211 +[titan] 2025-07-09 16:03:52,371 - root - INFO - lr: 2.9310e-04 gnorm: 0.83 [ 2:28:56<19:51:00] +[titan] 2025-07-09 16:03:56,293 - root - INFO - step: 11120 loss: 17.1141 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6785 global_avg_mtp_loss: 14.4356 +[titan] 2025-07-09 16:03:56,293 - root - INFO - lr: 2.9310e-04 gnorm: 0.98 [ 2:28:59<19:50:55] +[titan] 2025-07-09 16:04:00,219 - root - INFO - step: 11125 loss: 17.0464 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6564 global_avg_mtp_loss: 14.3900 +[titan] 2025-07-09 16:04:00,220 - root - INFO - lr: 2.9309e-04 gnorm: 0.91 [ 2:29:03<19:50:50] +[titan] 2025-07-09 16:04:04,136 - root - INFO - step: 11130 loss: 17.1739 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.7089 global_avg_mtp_loss: 14.4650 +[titan] 2025-07-09 16:04:04,136 - root - INFO - lr: 2.9308e-04 gnorm: 0.80 [ 2:29:07<19:50:45] +[titan] 2025-07-09 16:04:08,230 - root - INFO - step: 11135 loss: 17.5796 memory: 44.58GiB(31.99%) tps: 80,047 tflops: 276.25 mfu: 27.93% global_avg_ntp_loss: 2.7886 global_avg_mtp_loss: 14.7910 +[titan] 2025-07-09 16:04:08,230 - root - INFO - lr: 2.9308e-04 gnorm: 0.87 [ 2:29:11<19:50:42] +[titan] 2025-07-09 16:04:12,159 - root - INFO - step: 11140 loss: 17.5323 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.7727 global_avg_mtp_loss: 14.7596 +[titan] 2025-07-09 16:04:12,159 - root - INFO - lr: 2.9307e-04 gnorm: 1.00 [ 2:29:15<19:50:37] +[titan] 2025-07-09 16:04:16,090 - root - INFO - step: 11145 loss: 17.2844 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.7135 global_avg_mtp_loss: 14.5709 +[titan] 2025-07-09 16:04:16,090 - root - INFO - lr: 2.9306e-04 gnorm: 0.89 [ 2:29:19<19:50:32] +[titan] 2025-07-09 16:04:19,204 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:04:19,999 - root - INFO - step: 11150 loss: 17.1036 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6927 global_avg_mtp_loss: 14.4109 +[titan] 2025-07-09 16:04:19,999 - root - INFO - lr: 2.9306e-04 gnorm: 0.94 [ 2:29:23<19:50:27] +[titan] 2025-07-09 16:04:23,913 - root - INFO - step: 11155 loss: 17.2817 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.7107 global_avg_mtp_loss: 14.5710 +[titan] 2025-07-09 16:04:23,913 - root - INFO - lr: 2.9305e-04 gnorm: 0.95 [ 2:29:27<19:50:22] +[titan] 2025-07-09 16:04:27,853 - root - INFO - step: 11160 loss: 17.4088 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.7442 global_avg_mtp_loss: 14.6646 +[titan] 2025-07-09 16:04:27,854 - root - INFO - lr: 2.9304e-04 gnorm: 0.99 [ 2:29:31<19:50:18] +[titan] 2025-07-09 16:04:31,778 - root - INFO - step: 11165 loss: 17.3105 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.7236 global_avg_mtp_loss: 14.5870 +[titan] 2025-07-09 16:04:31,778 - root - INFO - lr: 2.9304e-04 gnorm: 0.85 [ 2:29:35<19:50:13] +[titan] 2025-07-09 16:04:35,862 - root - INFO - step: 11170 loss: 17.2436 memory: 44.58GiB(31.99%) tps: 80,235 tflops: 276.90 mfu: 28.00% global_avg_ntp_loss: 2.7043 global_avg_mtp_loss: 14.5393 +[titan] 2025-07-09 16:04:35,862 - root - INFO - lr: 2.9303e-04 gnorm: 0.89 [ 2:29:39<19:50:10] +[titan] 2025-07-09 16:04:39,806 - root - INFO - step: 11175 loss: 17.2105 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.6935 global_avg_mtp_loss: 14.5170 +[titan] 2025-07-09 16:04:39,806 - root - INFO - lr: 2.9302e-04 gnorm: 0.83 [ 2:29:43<19:50:05] +[titan] 2025-07-09 16:04:43,716 - root - INFO - step: 11180 loss: 17.2279 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.7213 global_avg_mtp_loss: 14.5066 +[titan] 2025-07-09 16:04:43,716 - root - INFO - lr: 2.9302e-04 gnorm: 0.82 [ 2:29:47<19:50:00] +[titan] 2025-07-09 16:04:47,658 - root - INFO - step: 11185 loss: 17.4528 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.7420 global_avg_mtp_loss: 14.7108 +[titan] 2025-07-09 16:04:47,658 - root - INFO - lr: 2.9301e-04 gnorm: 0.91 [ 2:29:51<19:49:55] +[titan] 2025-07-09 16:04:51,578 - root - INFO - step: 11190 loss: 17.4355 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.7498 global_avg_mtp_loss: 14.6857 +[titan] 2025-07-09 16:04:51,578 - root - INFO - lr: 2.9300e-04 gnorm: 0.98 [ 2:29:55<19:49:51] +[titan] 2025-07-09 16:04:55,484 - root - INFO - step: 11195 loss: 17.4372 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.7533 global_avg_mtp_loss: 14.6838 +[titan] 2025-07-09 16:04:55,485 - root - INFO - lr: 2.9300e-04 gnorm: 0.91 [ 2:29:59<19:49:46] +[titan] 2025-07-09 16:04:58,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:04:59,405 - root - INFO - step: 11200 loss: 17.4662 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.7698 global_avg_mtp_loss: 14.6964 +[titan] 2025-07-09 16:04:59,405 - root - INFO - lr: 2.9299e-04 gnorm: 0.96 [ 2:30:03<19:49:41] +[titan] 2025-07-09 16:05:03,322 - root - INFO - step: 11205 loss: 17.4599 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.7292 global_avg_mtp_loss: 14.7307 +[titan] 2025-07-09 16:05:03,322 - root - INFO - lr: 2.9298e-04 gnorm: 0.90 [ 2:30:06<19:49:36] +[titan] 2025-07-09 16:05:07,279 - root - INFO - step: 11210 loss: 17.3782 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.7395 global_avg_mtp_loss: 14.6387 +[titan] 2025-07-09 16:05:07,280 - root - INFO - lr: 2.9297e-04 gnorm: 0.82 [ 2:30:10<19:49:31] +[titan] 2025-07-09 16:05:11,179 - root - INFO - step: 11215 loss: 17.2984 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.7323 global_avg_mtp_loss: 14.5661 +[titan] 2025-07-09 16:05:11,179 - root - INFO - lr: 2.9297e-04 gnorm: 0.82 [ 2:30:14<19:49:27] +[titan] 2025-07-09 16:05:15,079 - root - INFO - step: 11220 loss: 17.3003 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.7218 global_avg_mtp_loss: 14.5785 +[titan] 2025-07-09 16:05:15,079 - root - INFO - lr: 2.9296e-04 gnorm: 0.87 [ 2:30:18<19:49:22] +[titan] 2025-07-09 16:05:18,979 - root - INFO - step: 11225 loss: 17.5594 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.7766 global_avg_mtp_loss: 14.7828 +[titan] 2025-07-09 16:05:18,980 - root - INFO - lr: 2.9295e-04 gnorm: 0.86 [ 2:30:22<19:49:17] +[titan] 2025-07-09 16:05:22,895 - root - INFO - step: 11230 loss: 17.2329 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.7120 global_avg_mtp_loss: 14.5209 +[titan] 2025-07-09 16:05:22,895 - root - INFO - lr: 2.9295e-04 gnorm: 0.87 [ 2:30:26<19:49:12] +[titan] 2025-07-09 16:05:26,798 - root - INFO - step: 11235 loss: 17.2806 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.7115 global_avg_mtp_loss: 14.5690 +[titan] 2025-07-09 16:05:26,799 - root - INFO - lr: 2.9294e-04 gnorm: 0.84 [ 2:30:30<19:49:07] +[titan] 2025-07-09 16:05:30,715 - root - INFO - step: 11240 loss: 17.2918 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7140 global_avg_mtp_loss: 14.5778 +[titan] 2025-07-09 16:05:30,715 - root - INFO - lr: 2.9293e-04 gnorm: 0.82 [ 2:30:34<19:49:02] +[titan] 2025-07-09 16:05:34,641 - root - INFO - step: 11245 loss: 17.1755 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6798 global_avg_mtp_loss: 14.4957 +[titan] 2025-07-09 16:05:34,641 - root - INFO - lr: 2.9293e-04 gnorm: 0.88 [ 2:30:38<19:48:57] +[titan] 2025-07-09 16:05:37,759 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:05:38,548 - root - INFO - step: 11250 loss: 17.1340 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6964 global_avg_mtp_loss: 14.4376 +[titan] 2025-07-09 16:05:38,548 - root - INFO - lr: 2.9292e-04 gnorm: 0.87 [ 2:30:42<19:48:52] +[titan] 2025-07-09 16:05:42,453 - root - INFO - step: 11255 loss: 17.3410 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.7271 global_avg_mtp_loss: 14.6139 +[titan] 2025-07-09 16:05:42,453 - root - INFO - lr: 2.9291e-04 gnorm: 0.87 [ 2:30:46<19:48:47] +[titan] 2025-07-09 16:05:46,358 - root - INFO - step: 11260 loss: 17.2685 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7054 global_avg_mtp_loss: 14.5631 +[titan] 2025-07-09 16:05:46,358 - root - INFO - lr: 2.9291e-04 gnorm: 0.92 [ 2:30:49<19:48:42] +[titan] 2025-07-09 16:05:49,704 - root - INFO - Dumping profiler traces at step 11264 +[titan] 2025-07-09 16:05:49,737 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:05:50,554 - root - INFO - step: 11265 loss: 17.6719 memory: 44.58GiB(31.99%) tps: 78,104 tflops: 269.55 mfu: 27.25% global_avg_ntp_loss: 2.7855 global_avg_mtp_loss: 14.8864 +[titan] 2025-07-09 16:05:50,554 - root - INFO - lr: 2.9290e-04 gnorm: 0.95 [ 2:30:54<19:48:40] +[titan] 2025-07-09 16:05:54,458 - root - INFO - step: 11270 loss: 17.5783 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7879 global_avg_mtp_loss: 14.7904 +[titan] 2025-07-09 16:05:54,459 - root - INFO - lr: 2.9289e-04 gnorm: 0.93 [ 2:30:58<19:48:35] +[titan] 2025-07-09 16:05:58,389 - root - INFO - step: 11275 loss: 17.3408 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.7257 global_avg_mtp_loss: 14.6151 +[titan] 2025-07-09 16:05:58,389 - root - INFO - lr: 2.9289e-04 gnorm: 0.93 [ 2:31:02<19:48:30] +[titan] 2025-07-09 16:06:02,345 - root - INFO - step: 11280 loss: 17.0575 memory: 44.58GiB(31.99%) tps: 82,844 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.6663 global_avg_mtp_loss: 14.3911 +[titan] 2025-07-09 16:06:02,345 - root - INFO - lr: 2.9288e-04 gnorm: 0.85 [ 2:31:05<19:48:26] +[titan] 2025-07-09 16:06:06,260 - root - INFO - step: 11285 loss: 17.0656 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6842 global_avg_mtp_loss: 14.3814 +[titan] 2025-07-09 16:06:06,261 - root - INFO - lr: 2.9287e-04 gnorm: 0.90 [ 2:31:09<19:48:21] +[titan] 2025-07-09 16:06:10,146 - root - INFO - step: 11290 loss: 17.4752 memory: 44.58GiB(31.99%) tps: 84,336 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 2.7644 global_avg_mtp_loss: 14.7108 +[titan] 2025-07-09 16:06:10,147 - root - INFO - lr: 2.9287e-04 gnorm: 0.89 [ 2:31:13<19:48:16] +[titan] 2025-07-09 16:06:14,041 - root - INFO - step: 11295 loss: 17.2238 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.7165 global_avg_mtp_loss: 14.5074 +[titan] 2025-07-09 16:06:14,041 - root - INFO - lr: 2.9286e-04 gnorm: 0.81 [ 2:31:17<19:48:11] +[titan] 2025-07-09 16:06:17,166 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:06:18,017 - root - INFO - step: 11300 loss: 17.3734 memory: 44.58GiB(31.99%) tps: 82,428 tflops: 284.47 mfu: 28.76% global_avg_ntp_loss: 2.7266 global_avg_mtp_loss: 14.6469 +[titan] 2025-07-09 16:06:18,017 - root - INFO - lr: 2.9285e-04 gnorm: 0.88 [ 2:31:21<19:48:06] +[titan] 2025-07-09 16:06:21,978 - root - INFO - step: 11305 loss: 17.3874 memory: 44.58GiB(31.99%) tps: 82,733 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.7465 global_avg_mtp_loss: 14.6409 +[titan] 2025-07-09 16:06:21,978 - root - INFO - lr: 2.9284e-04 gnorm: 0.89 [ 2:31:25<19:48:02] +[titan] 2025-07-09 16:06:25,883 - root - INFO - step: 11310 loss: 17.3310 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.7316 global_avg_mtp_loss: 14.5994 +[titan] 2025-07-09 16:06:25,883 - root - INFO - lr: 2.9284e-04 gnorm: 1.01 [ 2:31:29<19:47:57] +[titan] 2025-07-09 16:06:29,793 - root - INFO - step: 11315 loss: 17.2780 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.7102 global_avg_mtp_loss: 14.5678 +[titan] 2025-07-09 16:06:29,794 - root - INFO - lr: 2.9283e-04 gnorm: 0.90 [ 2:31:33<19:47:52] +[titan] 2025-07-09 16:06:33,713 - root - INFO - step: 11320 loss: 17.3980 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.7392 global_avg_mtp_loss: 14.6588 +[titan] 2025-07-09 16:06:33,713 - root - INFO - lr: 2.9282e-04 gnorm: 0.86 [ 2:31:37<19:47:47] +[titan] 2025-07-09 16:06:37,603 - root - INFO - step: 11325 loss: 17.2093 memory: 44.58GiB(31.99%) tps: 84,251 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.6973 global_avg_mtp_loss: 14.5121 +[titan] 2025-07-09 16:06:37,603 - root - INFO - lr: 2.9282e-04 gnorm: 0.86 [ 2:31:41<19:47:42] +[titan] 2025-07-09 16:06:41,527 - root - INFO - step: 11330 loss: 17.3672 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.7493 global_avg_mtp_loss: 14.6179 +[titan] 2025-07-09 16:06:41,528 - root - INFO - lr: 2.9281e-04 gnorm: 0.86 [ 2:31:45<19:47:38] +[titan] 2025-07-09 16:06:45,446 - root - INFO - step: 11335 loss: 17.4903 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.7451 global_avg_mtp_loss: 14.7453 +[titan] 2025-07-09 16:06:45,446 - root - INFO - lr: 2.9280e-04 gnorm: 0.92 [ 2:31:49<19:47:33] +[titan] 2025-07-09 16:06:49,343 - root - INFO - step: 11340 loss: 17.2845 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6998 global_avg_mtp_loss: 14.5847 +[titan] 2025-07-09 16:06:49,343 - root - INFO - lr: 2.9280e-04 gnorm: 0.91 [ 2:31:52<19:47:28] +[titan] 2025-07-09 16:06:53,228 - root - INFO - step: 11345 loss: 17.3665 memory: 44.58GiB(31.99%) tps: 84,354 tflops: 291.12 mfu: 29.44% global_avg_ntp_loss: 2.7311 global_avg_mtp_loss: 14.6354 +[titan] 2025-07-09 16:06:53,228 - root - INFO - lr: 2.9279e-04 gnorm: 0.87 [ 2:31:56<19:47:23] +[titan] 2025-07-09 16:06:56,344 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:06:57,133 - root - INFO - step: 11350 loss: 17.0048 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6732 global_avg_mtp_loss: 14.3316 +[titan] 2025-07-09 16:06:57,133 - root - INFO - lr: 2.9278e-04 gnorm: 0.85 [ 2:32:00<19:47:18] +[titan] 2025-07-09 16:07:01,027 - root - INFO - step: 11355 loss: 17.2512 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.7185 global_avg_mtp_loss: 14.5328 +[titan] 2025-07-09 16:07:01,027 - root - INFO - lr: 2.9278e-04 gnorm: 0.96 [ 2:32:04<19:47:13] +[titan] 2025-07-09 16:07:04,990 - root - INFO - step: 11360 loss: 17.2657 memory: 44.58GiB(31.99%) tps: 82,686 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.7187 global_avg_mtp_loss: 14.5470 +[titan] 2025-07-09 16:07:04,990 - root - INFO - lr: 2.9277e-04 gnorm: 0.87 [ 2:32:08<19:47:08] +[titan] 2025-07-09 16:07:08,959 - root - INFO - step: 11365 loss: 17.2759 memory: 44.58GiB(31.99%) tps: 82,562 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.7222 global_avg_mtp_loss: 14.5537 +[titan] 2025-07-09 16:07:08,959 - root - INFO - lr: 2.9276e-04 gnorm: 0.90 [ 2:32:12<19:47:04] +[titan] 2025-07-09 16:07:12,875 - root - INFO - step: 11370 loss: 17.1878 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6975 global_avg_mtp_loss: 14.4903 +[titan] 2025-07-09 16:07:12,875 - root - INFO - lr: 2.9275e-04 gnorm: 0.90 [ 2:32:16<19:46:59] +[titan] 2025-07-09 16:07:16,812 - root - INFO - step: 11375 loss: 17.3875 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.7500 global_avg_mtp_loss: 14.6375 +[titan] 2025-07-09 16:07:16,813 - root - INFO - lr: 2.9275e-04 gnorm: 0.90 [ 2:32:20<19:46:55] +[titan] 2025-07-09 16:07:20,709 - root - INFO - step: 11380 loss: 17.0503 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.3708 +[titan] 2025-07-09 16:07:20,710 - root - INFO - lr: 2.9274e-04 gnorm: 0.85 [ 2:32:24<19:46:50] +[titan] 2025-07-09 16:07:24,639 - root - INFO - step: 11385 loss: 17.2033 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.7024 global_avg_mtp_loss: 14.5009 +[titan] 2025-07-09 16:07:24,639 - root - INFO - lr: 2.9273e-04 gnorm: 0.87 [ 2:32:28<19:46:45] +[titan] 2025-07-09 16:07:28,531 - root - INFO - step: 11390 loss: 17.3024 memory: 44.58GiB(31.99%) tps: 84,199 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.7249 global_avg_mtp_loss: 14.5775 +[titan] 2025-07-09 16:07:28,531 - root - INFO - lr: 2.9273e-04 gnorm: 0.92 [ 2:32:32<19:46:40] +[titan] 2025-07-09 16:07:32,459 - root - INFO - step: 11395 loss: 17.0803 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6786 global_avg_mtp_loss: 14.4017 +[titan] 2025-07-09 16:07:32,459 - root - INFO - lr: 2.9272e-04 gnorm: 0.97 [ 2:32:36<19:46:35] +[titan] 2025-07-09 16:07:35,586 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:07:36,371 - root - INFO - step: 11400 loss: 17.0797 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6986 global_avg_mtp_loss: 14.3812 +[titan] 2025-07-09 16:07:36,371 - root - INFO - lr: 2.9271e-04 gnorm: 0.96 [ 2:32:39<19:46:30] +[titan] 2025-07-09 16:07:40,281 - root - INFO - step: 11405 loss: 17.2986 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.7184 global_avg_mtp_loss: 14.5803 +[titan] 2025-07-09 16:07:40,282 - root - INFO - lr: 2.9271e-04 gnorm: 0.86 [ 2:32:43<19:46:25] +[titan] 2025-07-09 16:07:44,216 - root - INFO - step: 11410 loss: 17.4145 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.7335 global_avg_mtp_loss: 14.6810 +[titan] 2025-07-09 16:07:44,216 - root - INFO - lr: 2.9270e-04 gnorm: 0.89 [ 2:32:47<19:46:21] +[titan] 2025-07-09 16:07:48,123 - root - INFO - step: 11415 loss: 17.1252 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6846 global_avg_mtp_loss: 14.4406 +[titan] 2025-07-09 16:07:48,124 - root - INFO - lr: 2.9269e-04 gnorm: 0.94 [ 2:32:51<19:46:16] +[titan] 2025-07-09 16:07:52,035 - root - INFO - step: 11420 loss: 17.4914 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.7872 global_avg_mtp_loss: 14.7042 +[titan] 2025-07-09 16:07:52,035 - root - INFO - lr: 2.9269e-04 gnorm: 0.87 [ 2:32:55<19:46:11] +[titan] 2025-07-09 16:07:55,944 - root - INFO - step: 11425 loss: 17.2938 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.7268 global_avg_mtp_loss: 14.5669 +[titan] 2025-07-09 16:07:55,944 - root - INFO - lr: 2.9268e-04 gnorm: 0.81 [ 2:32:59<19:46:06] +[titan] 2025-07-09 16:07:59,858 - root - INFO - step: 11430 loss: 17.1151 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6887 global_avg_mtp_loss: 14.4265 +[titan] 2025-07-09 16:07:59,858 - root - INFO - lr: 2.9267e-04 gnorm: 0.90 [ 2:33:03<19:46:01] +[titan] 2025-07-09 16:08:03,776 - root - INFO - step: 11435 loss: 17.4024 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.7403 global_avg_mtp_loss: 14.6621 +[titan] 2025-07-09 16:08:03,777 - root - INFO - lr: 2.9266e-04 gnorm: 0.93 [ 2:33:07<19:45:57] +[titan] 2025-07-09 16:08:07,691 - root - INFO - step: 11440 loss: 17.4317 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.7524 global_avg_mtp_loss: 14.6793 +[titan] 2025-07-09 16:08:07,692 - root - INFO - lr: 2.9266e-04 gnorm: 0.90 [ 2:33:11<19:45:52] +[titan] 2025-07-09 16:08:11,579 - root - INFO - step: 11445 loss: 17.4473 memory: 44.58GiB(31.99%) tps: 84,294 tflops: 290.91 mfu: 29.41% global_avg_ntp_loss: 2.7447 global_avg_mtp_loss: 14.7026 +[titan] 2025-07-09 16:08:11,579 - root - INFO - lr: 2.9265e-04 gnorm: 0.92 [ 2:33:15<19:45:47] +[titan] 2025-07-09 16:08:14,762 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:08:15,552 - root - INFO - step: 11450 loss: 17.3868 memory: 44.58GiB(31.99%) tps: 82,482 tflops: 284.66 mfu: 28.78% global_avg_ntp_loss: 2.7349 global_avg_mtp_loss: 14.6519 +[titan] 2025-07-09 16:08:15,552 - root - INFO - lr: 2.9264e-04 gnorm: 0.95 [ 2:33:19<19:45:42] +[titan] 2025-07-09 16:08:19,470 - root - INFO - step: 11455 loss: 17.3256 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.7268 global_avg_mtp_loss: 14.5988 +[titan] 2025-07-09 16:08:19,471 - root - INFO - lr: 2.9264e-04 gnorm: 0.89 [ 2:33:23<19:45:38] +[titan] 2025-07-09 16:08:23,368 - root - INFO - step: 11460 loss: 17.1912 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7015 global_avg_mtp_loss: 14.4897 +[titan] 2025-07-09 16:08:23,368 - root - INFO - lr: 2.9263e-04 gnorm: 0.90 [ 2:33:26<19:45:33] +[titan] 2025-07-09 16:08:27,258 - root - INFO - step: 11465 loss: 17.4128 memory: 44.58GiB(31.99%) tps: 84,236 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.7396 global_avg_mtp_loss: 14.6731 +[titan] 2025-07-09 16:08:27,258 - root - INFO - lr: 2.9262e-04 gnorm: 0.94 [ 2:33:30<19:45:28] +[titan] 2025-07-09 16:08:31,162 - root - INFO - step: 11470 loss: 17.0991 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.7044 global_avg_mtp_loss: 14.3946 +[titan] 2025-07-09 16:08:31,163 - root - INFO - lr: 2.9262e-04 gnorm: 0.94 [ 2:33:34<19:45:23] +[titan] 2025-07-09 16:08:35,063 - root - INFO - step: 11475 loss: 17.4210 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.7431 global_avg_mtp_loss: 14.6779 +[titan] 2025-07-09 16:08:35,064 - root - INFO - lr: 2.9261e-04 gnorm: 0.85 [ 2:33:38<19:45:18] +[titan] 2025-07-09 16:08:38,996 - root - INFO - step: 11480 loss: 17.0220 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6708 global_avg_mtp_loss: 14.3512 +[titan] 2025-07-09 16:08:38,996 - root - INFO - lr: 2.9260e-04 gnorm: 0.94 [ 2:33:42<19:45:13] +[titan] 2025-07-09 16:08:42,890 - root - INFO - step: 11485 loss: 17.3841 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.7398 global_avg_mtp_loss: 14.6443 +[titan] 2025-07-09 16:08:42,891 - root - INFO - lr: 2.9259e-04 gnorm: 1.06 [ 2:33:46<19:45:08] +[titan] 2025-07-09 16:08:46,802 - root - INFO - step: 11490 loss: 17.0474 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.3847 +[titan] 2025-07-09 16:08:46,802 - root - INFO - lr: 2.9259e-04 gnorm: 0.96 [ 2:33:50<19:45:03] +[titan] 2025-07-09 16:08:50,763 - root - INFO - step: 11495 loss: 17.4393 memory: 44.58GiB(31.99%) tps: 82,735 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.7443 global_avg_mtp_loss: 14.6950 +[titan] 2025-07-09 16:08:50,763 - root - INFO - lr: 2.9258e-04 gnorm: 0.93 [ 2:33:54<19:44:59] +[titan] 2025-07-09 16:08:53,895 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:08:54,680 - root - INFO - step: 11500 loss: 17.8516 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.8401 global_avg_mtp_loss: 15.0115 +[titan] 2025-07-09 16:08:54,680 - root - INFO - lr: 2.9257e-04 gnorm: 0.96 [ 2:33:58<19:44:54] +[titan] 2025-07-09 16:08:58,591 - root - INFO - step: 11505 loss: 17.2611 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.7326 global_avg_mtp_loss: 14.5284 +[titan] 2025-07-09 16:08:58,591 - root - INFO - lr: 2.9257e-04 gnorm: 1.14 [ 2:34:02<19:44:49] +[titan] 2025-07-09 16:09:02,548 - root - INFO - step: 11510 loss: 17.4096 memory: 44.58GiB(31.99%) tps: 82,820 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.7365 global_avg_mtp_loss: 14.6731 +[titan] 2025-07-09 16:09:02,548 - root - INFO - lr: 2.9256e-04 gnorm: 0.90 [ 2:34:06<19:44:45] +[titan] 2025-07-09 16:09:06,435 - root - INFO - step: 11515 loss: 17.1980 memory: 44.58GiB(31.99%) tps: 84,299 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6985 global_avg_mtp_loss: 14.4995 +[titan] 2025-07-09 16:09:06,436 - root - INFO - lr: 2.9255e-04 gnorm: 0.95 [ 2:34:10<19:44:40] +[titan] 2025-07-09 16:09:10,327 - root - INFO - step: 11520 loss: 17.0313 memory: 44.58GiB(31.99%) tps: 84,215 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.3686 +[titan] 2025-07-09 16:09:10,327 - root - INFO - lr: 2.9255e-04 gnorm: 0.87 [ 2:34:13<19:44:35] +[titan] 2025-07-09 16:09:14,230 - root - INFO - step: 11525 loss: 17.1460 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.6846 global_avg_mtp_loss: 14.4614 +[titan] 2025-07-09 16:09:14,231 - root - INFO - lr: 2.9254e-04 gnorm: 0.86 [ 2:34:17<19:44:30] +[titan] 2025-07-09 16:09:18,144 - root - INFO - step: 11530 loss: 17.2274 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5152 +[titan] 2025-07-09 16:09:18,145 - root - INFO - lr: 2.9253e-04 gnorm: 0.84 [ 2:34:21<19:44:25] +[titan] 2025-07-09 16:09:22,039 - root - INFO - step: 11535 loss: 17.1953 memory: 44.58GiB(31.99%) tps: 84,144 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.7015 global_avg_mtp_loss: 14.4938 +[titan] 2025-07-09 16:09:22,039 - root - INFO - lr: 2.9252e-04 gnorm: 0.90 [ 2:34:25<19:44:20] +[titan] 2025-07-09 16:09:25,993 - root - INFO - step: 11540 loss: 17.2626 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.7326 global_avg_mtp_loss: 14.5301 +[titan] 2025-07-09 16:09:25,993 - root - INFO - lr: 2.9252e-04 gnorm: 0.96 [ 2:34:29<19:44:16] +[titan] 2025-07-09 16:09:29,897 - root - INFO - step: 11545 loss: 17.1652 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7186 global_avg_mtp_loss: 14.4466 +[titan] 2025-07-09 16:09:29,897 - root - INFO - lr: 2.9251e-04 gnorm: 0.90 [ 2:34:33<19:44:11] +[titan] 2025-07-09 16:09:33,025 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:09:33,814 - root - INFO - step: 11550 loss: 17.3771 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.7447 global_avg_mtp_loss: 14.6324 +[titan] 2025-07-09 16:09:33,815 - root - INFO - lr: 2.9250e-04 gnorm: 0.85 [ 2:34:37<19:44:06] +[titan] 2025-07-09 16:09:37,709 - root - INFO - step: 11555 loss: 17.3440 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.7171 global_avg_mtp_loss: 14.6269 +[titan] 2025-07-09 16:09:37,709 - root - INFO - lr: 2.9250e-04 gnorm: 0.92 [ 2:34:41<19:44:01] +[titan] 2025-07-09 16:09:41,603 - root - INFO - step: 11560 loss: 17.4519 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.7370 global_avg_mtp_loss: 14.7148 +[titan] 2025-07-09 16:09:41,603 - root - INFO - lr: 2.9249e-04 gnorm: 0.85 [ 2:34:45<19:43:56] +[titan] 2025-07-09 16:09:45,522 - root - INFO - step: 11565 loss: 17.2850 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.7082 global_avg_mtp_loss: 14.5768 +[titan] 2025-07-09 16:09:45,522 - root - INFO - lr: 2.9248e-04 gnorm: 0.96 [ 2:34:49<19:43:51] +[titan] 2025-07-09 16:09:49,419 - root - INFO - step: 11570 loss: 17.2515 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.7106 global_avg_mtp_loss: 14.5409 +[titan] 2025-07-09 16:09:49,420 - root - INFO - lr: 2.9248e-04 gnorm: 0.88 [ 2:34:53<19:43:47] +[titan] 2025-07-09 16:09:53,313 - root - INFO - step: 11575 loss: 17.1057 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6665 global_avg_mtp_loss: 14.4392 +[titan] 2025-07-09 16:09:53,314 - root - INFO - lr: 2.9247e-04 gnorm: 0.93 [ 2:34:56<19:43:42] +[titan] 2025-07-09 16:09:57,231 - root - INFO - step: 11580 loss: 17.3412 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.7226 global_avg_mtp_loss: 14.6185 +[titan] 2025-07-09 16:09:57,232 - root - INFO - lr: 2.9246e-04 gnorm: 0.89 [ 2:35:00<19:43:37] +[titan] 2025-07-09 16:10:01,164 - root - INFO - step: 11585 loss: 17.2611 memory: 44.58GiB(31.99%) tps: 83,339 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.7049 global_avg_mtp_loss: 14.5562 +[titan] 2025-07-09 16:10:01,164 - root - INFO - lr: 2.9245e-04 gnorm: 0.97 [ 2:35:04<19:43:32] +[titan] 2025-07-09 16:10:05,118 - root - INFO - step: 11590 loss: 17.2929 memory: 44.58GiB(31.99%) tps: 82,869 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.7306 global_avg_mtp_loss: 14.5623 +[titan] 2025-07-09 16:10:05,119 - root - INFO - lr: 2.9245e-04 gnorm: 0.82 [ 2:35:08<19:43:28] +[titan] 2025-07-09 16:10:09,056 - root - INFO - step: 11595 loss: 17.2819 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.7288 global_avg_mtp_loss: 14.5531 +[titan] 2025-07-09 16:10:09,056 - root - INFO - lr: 2.9244e-04 gnorm: 0.89 [ 2:35:12<19:43:23] +[titan] 2025-07-09 16:10:12,189 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:10:12,974 - root - INFO - step: 11600 loss: 17.5482 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.7556 global_avg_mtp_loss: 14.7926 +[titan] 2025-07-09 16:10:12,974 - root - INFO - lr: 2.9243e-04 gnorm: 7.66 [ 2:35:16<19:43:18] +[titan] 2025-07-09 16:10:16,878 - root - INFO - step: 11605 loss: 17.4014 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.7527 global_avg_mtp_loss: 14.6486 +[titan] 2025-07-09 16:10:16,878 - root - INFO - lr: 2.9243e-04 gnorm: 0.98 [ 2:35:20<19:43:13] +[titan] 2025-07-09 16:10:20,787 - root - INFO - step: 11610 loss: 17.5356 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.7656 global_avg_mtp_loss: 14.7700 +[titan] 2025-07-09 16:10:20,787 - root - INFO - lr: 2.9242e-04 gnorm: 0.94 [ 2:35:24<19:43:09] +[titan] 2025-07-09 16:10:24,684 - root - INFO - step: 11615 loss: 17.2482 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.7145 global_avg_mtp_loss: 14.5337 +[titan] 2025-07-09 16:10:24,685 - root - INFO - lr: 2.9241e-04 gnorm: 0.92 [ 2:35:28<19:43:04] +[titan] 2025-07-09 16:10:28,634 - root - INFO - step: 11620 loss: 17.0750 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.6730 global_avg_mtp_loss: 14.4020 +[titan] 2025-07-09 16:10:28,634 - root - INFO - lr: 2.9240e-04 gnorm: 0.89 [ 2:35:32<19:42:59] +[titan] 2025-07-09 16:10:32,603 - root - INFO - step: 11625 loss: 17.3716 memory: 44.58GiB(31.99%) tps: 82,553 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.7302 global_avg_mtp_loss: 14.6413 +[titan] 2025-07-09 16:10:32,604 - root - INFO - lr: 2.9240e-04 gnorm: 0.88 [ 2:35:36<19:42:55] +[titan] 2025-07-09 16:10:36,552 - root - INFO - step: 11630 loss: 17.2718 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.7024 global_avg_mtp_loss: 14.5694 +[titan] 2025-07-09 16:10:36,552 - root - INFO - lr: 2.9239e-04 gnorm: 0.87 [ 2:35:40<19:42:50] +[titan] 2025-07-09 16:10:40,480 - root - INFO - step: 11635 loss: 17.3935 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.7320 global_avg_mtp_loss: 14.6616 +[titan] 2025-07-09 16:10:40,481 - root - INFO - lr: 2.9238e-04 gnorm: 0.90 [ 2:35:44<19:42:46] +[titan] 2025-07-09 16:10:44,400 - root - INFO - step: 11640 loss: 17.4813 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.7464 global_avg_mtp_loss: 14.7348 +[titan] 2025-07-09 16:10:44,400 - root - INFO - lr: 2.9238e-04 gnorm: 0.88 [ 2:35:48<19:42:41] +[titan] 2025-07-09 16:10:48,307 - root - INFO - step: 11645 loss: 17.2679 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7208 global_avg_mtp_loss: 14.5471 +[titan] 2025-07-09 16:10:48,307 - root - INFO - lr: 2.9237e-04 gnorm: 0.83 [ 2:35:51<19:42:36] +[titan] 2025-07-09 16:10:51,436 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:10:52,226 - root - INFO - step: 11650 loss: 17.1929 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.7094 global_avg_mtp_loss: 14.4836 +[titan] 2025-07-09 16:10:52,227 - root - INFO - lr: 2.9236e-04 gnorm: 0.95 [ 2:35:55<19:42:31] +[titan] 2025-07-09 16:10:56,150 - root - INFO - step: 11655 loss: 17.5000 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.7562 global_avg_mtp_loss: 14.7438 +[titan] 2025-07-09 16:10:56,151 - root - INFO - lr: 2.9236e-04 gnorm: 0.84 [ 2:35:59<19:42:26] +[titan] 2025-07-09 16:11:00,065 - root - INFO - step: 11660 loss: 17.2187 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6981 global_avg_mtp_loss: 14.5206 +[titan] 2025-07-09 16:11:00,066 - root - INFO - lr: 2.9235e-04 gnorm: 0.98 [ 2:36:03<19:42:22] +[titan] 2025-07-09 16:11:04,002 - root - INFO - step: 11665 loss: 17.5101 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.7548 global_avg_mtp_loss: 14.7553 +[titan] 2025-07-09 16:11:04,002 - root - INFO - lr: 2.9234e-04 gnorm: 0.91 [ 2:36:07<19:42:17] +[titan] 2025-07-09 16:11:07,977 - root - INFO - step: 11670 loss: 17.2761 memory: 44.58GiB(31.99%) tps: 82,451 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.7212 global_avg_mtp_loss: 14.5550 +[titan] 2025-07-09 16:11:07,977 - root - INFO - lr: 2.9233e-04 gnorm: 0.90 [ 2:36:11<19:42:13] +[titan] 2025-07-09 16:11:11,942 - root - INFO - step: 11675 loss: 17.3775 memory: 44.58GiB(31.99%) tps: 82,652 tflops: 285.25 mfu: 28.84% global_avg_ntp_loss: 2.7311 global_avg_mtp_loss: 14.6465 +[titan] 2025-07-09 16:11:11,942 - root - INFO - lr: 2.9233e-04 gnorm: 0.87 [ 2:36:15<19:42:08] +[titan] 2025-07-09 16:11:15,868 - root - INFO - step: 11680 loss: 17.1950 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6903 global_avg_mtp_loss: 14.5047 +[titan] 2025-07-09 16:11:15,868 - root - INFO - lr: 2.9232e-04 gnorm: 0.93 [ 2:36:19<19:42:04] +[titan] 2025-07-09 16:11:19,747 - root - INFO - step: 11685 loss: 17.1773 memory: 44.58GiB(31.99%) tps: 84,480 tflops: 291.56 mfu: 29.48% global_avg_ntp_loss: 2.6904 global_avg_mtp_loss: 14.4869 +[titan] 2025-07-09 16:11:19,747 - root - INFO - lr: 2.9231e-04 gnorm: 0.83 [ 2:36:23<19:41:59] +[titan] 2025-07-09 16:11:23,715 - root - INFO - step: 11690 loss: 17.2520 memory: 44.58GiB(31.99%) tps: 82,603 tflops: 285.07 mfu: 28.82% global_avg_ntp_loss: 2.7207 global_avg_mtp_loss: 14.5313 +[titan] 2025-07-09 16:11:23,715 - root - INFO - lr: 2.9231e-04 gnorm: 0.85 [ 2:36:27<19:41:54] +[titan] 2025-07-09 16:11:27,657 - root - INFO - step: 11695 loss: 17.1873 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.7156 global_avg_mtp_loss: 14.4718 +[titan] 2025-07-09 16:11:27,657 - root - INFO - lr: 2.9230e-04 gnorm: 0.91 [ 2:36:31<19:41:50] +[titan] 2025-07-09 16:11:30,787 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:11:31,574 - root - INFO - step: 11700 loss: 17.3158 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.7286 global_avg_mtp_loss: 14.5872 +[titan] 2025-07-09 16:11:31,574 - root - INFO - lr: 2.9229e-04 gnorm: 0.92 [ 2:36:35<19:41:45] +[titan] 2025-07-09 16:11:35,472 - root - INFO - step: 11705 loss: 16.9817 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6615 global_avg_mtp_loss: 14.3202 +[titan] 2025-07-09 16:11:35,472 - root - INFO - lr: 2.9228e-04 gnorm: 0.90 [ 2:36:39<19:41:40] +[titan] 2025-07-09 16:11:39,374 - root - INFO - step: 11710 loss: 17.3364 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.7176 global_avg_mtp_loss: 14.6188 +[titan] 2025-07-09 16:11:39,374 - root - INFO - lr: 2.9228e-04 gnorm: 0.94 [ 2:36:42<19:41:35] +[titan] 2025-07-09 16:11:43,278 - root - INFO - step: 11715 loss: 17.1088 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6700 global_avg_mtp_loss: 14.4388 +[titan] 2025-07-09 16:11:43,278 - root - INFO - lr: 2.9227e-04 gnorm: 0.86 [ 2:36:46<19:41:30] +[titan] 2025-07-09 16:11:47,169 - root - INFO - step: 11720 loss: 17.5623 memory: 44.58GiB(31.99%) tps: 84,224 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.7672 global_avg_mtp_loss: 14.7951 +[titan] 2025-07-09 16:11:47,169 - root - INFO - lr: 2.9226e-04 gnorm: 0.90 [ 2:36:50<19:41:25] +[titan] 2025-07-09 16:11:51,087 - root - INFO - step: 11725 loss: 17.1855 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.7039 global_avg_mtp_loss: 14.4816 +[titan] 2025-07-09 16:11:51,088 - root - INFO - lr: 2.9226e-04 gnorm: 0.93 [ 2:36:54<19:41:21] +[titan] 2025-07-09 16:11:54,987 - root - INFO - step: 11730 loss: 17.1769 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6869 global_avg_mtp_loss: 14.4900 +[titan] 2025-07-09 16:11:54,987 - root - INFO - lr: 2.9225e-04 gnorm: 0.97 [ 2:36:58<19:41:16] +[titan] 2025-07-09 16:11:58,910 - root - INFO - step: 11735 loss: 17.2193 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.7103 global_avg_mtp_loss: 14.5090 +[titan] 2025-07-09 16:11:58,910 - root - INFO - lr: 2.9224e-04 gnorm: 0.89 [ 2:37:02<19:41:11] +[titan] 2025-07-09 16:12:02,822 - root - INFO - step: 11740 loss: 17.0985 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.4162 +[titan] 2025-07-09 16:12:02,822 - root - INFO - lr: 2.9223e-04 gnorm: 0.88 [ 2:37:06<19:41:06] +[titan] 2025-07-09 16:12:06,732 - root - INFO - step: 11745 loss: 17.2465 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7251 global_avg_mtp_loss: 14.5214 +[titan] 2025-07-09 16:12:06,732 - root - INFO - lr: 2.9223e-04 gnorm: 0.90 [ 2:37:10<19:41:01] +[titan] 2025-07-09 16:12:09,862 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:12:10,649 - root - INFO - step: 11750 loss: 17.3265 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.7349 global_avg_mtp_loss: 14.5916 +[titan] 2025-07-09 16:12:10,650 - root - INFO - lr: 2.9222e-04 gnorm: 0.93 [ 2:37:14<19:40:57] +[titan] 2025-07-09 16:12:14,538 - root - INFO - step: 11755 loss: 17.2173 memory: 44.58GiB(31.99%) tps: 84,280 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.6873 global_avg_mtp_loss: 14.5300 +[titan] 2025-07-09 16:12:14,538 - root - INFO - lr: 2.9221e-04 gnorm: 0.91 [ 2:37:18<19:40:52] +[titan] 2025-07-09 16:12:18,434 - root - INFO - step: 11760 loss: 17.2954 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.7163 global_avg_mtp_loss: 14.5791 +[titan] 2025-07-09 16:12:18,434 - root - INFO - lr: 2.9221e-04 gnorm: 0.82 [ 2:37:22<19:40:47] +[titan] 2025-07-09 16:12:22,330 - root - INFO - step: 11765 loss: 17.4793 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7546 global_avg_mtp_loss: 14.7246 +[titan] 2025-07-09 16:12:22,331 - root - INFO - lr: 2.9220e-04 gnorm: 0.92 [ 2:37:25<19:40:42] +[titan] 2025-07-09 16:12:26,229 - root - INFO - step: 11770 loss: 17.0180 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6550 global_avg_mtp_loss: 14.3631 +[titan] 2025-07-09 16:12:26,229 - root - INFO - lr: 2.9219e-04 gnorm: 0.95 [ 2:37:29<19:40:37] +[titan] 2025-07-09 16:12:30,221 - root - INFO - step: 11775 loss: 17.3016 memory: 44.58GiB(31.99%) tps: 82,103 tflops: 283.35 mfu: 28.65% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5894 +[titan] 2025-07-09 16:12:30,221 - root - INFO - lr: 2.9218e-04 gnorm: 0.88 [ 2:37:33<19:40:33] +[titan] 2025-07-09 16:12:31,147 - root - INFO - Dumping profiler traces at step 11776 +[titan] 2025-07-09 16:12:31,179 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:12:34,312 - root - INFO - step: 11780 loss: 17.3963 memory: 44.58GiB(31.99%) tps: 80,106 tflops: 276.46 mfu: 27.95% global_avg_ntp_loss: 2.7424 global_avg_mtp_loss: 14.6539 +[titan] 2025-07-09 16:12:34,312 - root - INFO - lr: 2.9218e-04 gnorm: 1.01 [ 2:37:37<19:40:29] +[titan] 2025-07-09 16:12:38,208 - root - INFO - step: 11785 loss: 17.3357 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7316 global_avg_mtp_loss: 14.6041 +[titan] 2025-07-09 16:12:38,208 - root - INFO - lr: 2.9217e-04 gnorm: 0.91 [ 2:37:41<19:40:24] +[titan] 2025-07-09 16:12:42,118 - root - INFO - step: 11790 loss: 17.3656 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7410 global_avg_mtp_loss: 14.6246 +[titan] 2025-07-09 16:12:42,119 - root - INFO - lr: 2.9216e-04 gnorm: 0.93 [ 2:37:45<19:40:20] +[titan] 2025-07-09 16:12:46,013 - root - INFO - step: 11795 loss: 17.3110 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.7222 global_avg_mtp_loss: 14.5888 +[titan] 2025-07-09 16:12:46,013 - root - INFO - lr: 2.9215e-04 gnorm: 0.88 [ 2:37:49<19:40:15] +[titan] 2025-07-09 16:12:49,118 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:12:49,929 - root - INFO - step: 11800 loss: 17.4210 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.7326 global_avg_mtp_loss: 14.6884 +[titan] 2025-07-09 16:12:49,929 - root - INFO - lr: 2.9215e-04 gnorm: 0.85 [ 2:37:53<19:40:10] +[titan] 2025-07-09 16:12:53,836 - root - INFO - step: 11805 loss: 17.2993 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7135 global_avg_mtp_loss: 14.5858 +[titan] 2025-07-09 16:12:53,836 - root - INFO - lr: 2.9214e-04 gnorm: 0.86 [ 2:37:57<19:40:05] +[titan] 2025-07-09 16:12:57,737 - root - INFO - step: 11810 loss: 17.1991 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.5002 +[titan] 2025-07-09 16:12:57,737 - root - INFO - lr: 2.9213e-04 gnorm: 0.87 [ 2:38:01<19:40:00] +[titan] 2025-07-09 16:13:01,649 - root - INFO - step: 11815 loss: 17.3113 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.7181 global_avg_mtp_loss: 14.5932 +[titan] 2025-07-09 16:13:01,649 - root - INFO - lr: 2.9213e-04 gnorm: 0.86 [ 2:38:05<19:39:56] +[titan] 2025-07-09 16:13:05,563 - root - INFO - step: 11820 loss: 17.3549 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7385 global_avg_mtp_loss: 14.6165 +[titan] 2025-07-09 16:13:05,564 - root - INFO - lr: 2.9212e-04 gnorm: 0.94 [ 2:38:09<19:39:51] +[titan] 2025-07-09 16:13:09,470 - root - INFO - step: 11825 loss: 17.3865 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7368 global_avg_mtp_loss: 14.6497 +[titan] 2025-07-09 16:13:09,471 - root - INFO - lr: 2.9211e-04 gnorm: 0.84 [ 2:38:13<19:39:46] +[titan] 2025-07-09 16:13:13,383 - root - INFO - step: 11830 loss: 17.3939 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.7409 global_avg_mtp_loss: 14.6530 +[titan] 2025-07-09 16:13:13,383 - root - INFO - lr: 2.9210e-04 gnorm: 0.91 [ 2:38:16<19:39:41] +[titan] 2025-07-09 16:13:17,270 - root - INFO - step: 11835 loss: 17.5170 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.7618 global_avg_mtp_loss: 14.7551 +[titan] 2025-07-09 16:13:17,271 - root - INFO - lr: 2.9210e-04 gnorm: 0.97 [ 2:38:20<19:39:36] +[titan] 2025-07-09 16:13:21,181 - root - INFO - step: 11840 loss: 17.2915 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.7112 global_avg_mtp_loss: 14.5802 +[titan] 2025-07-09 16:13:21,182 - root - INFO - lr: 2.9209e-04 gnorm: 0.97 [ 2:38:24<19:39:31] +[titan] 2025-07-09 16:13:25,076 - root - INFO - step: 11845 loss: 17.3311 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.7133 global_avg_mtp_loss: 14.6178 +[titan] 2025-07-09 16:13:25,077 - root - INFO - lr: 2.9208e-04 gnorm: 0.90 [ 2:38:28<19:39:27] +[titan] 2025-07-09 16:13:28,182 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:13:28,972 - root - INFO - step: 11850 loss: 17.1575 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6953 global_avg_mtp_loss: 14.4622 +[titan] 2025-07-09 16:13:28,972 - root - INFO - lr: 2.9208e-04 gnorm: 0.93 [ 2:38:32<19:39:22] +[titan] 2025-07-09 16:13:32,869 - root - INFO - step: 11855 loss: 17.4452 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7480 global_avg_mtp_loss: 14.6972 +[titan] 2025-07-09 16:13:32,869 - root - INFO - lr: 2.9207e-04 gnorm: 0.87 [ 2:38:36<19:39:17] +[titan] 2025-07-09 16:13:36,773 - root - INFO - step: 11860 loss: 17.1954 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6975 global_avg_mtp_loss: 14.4980 +[titan] 2025-07-09 16:13:36,774 - root - INFO - lr: 2.9206e-04 gnorm: 0.92 [ 2:38:40<19:39:12] +[titan] 2025-07-09 16:13:40,680 - root - INFO - step: 11865 loss: 17.2112 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6940 global_avg_mtp_loss: 14.5172 +[titan] 2025-07-09 16:13:40,681 - root - INFO - lr: 2.9205e-04 gnorm: 0.92 [ 2:38:44<19:39:07] +[titan] 2025-07-09 16:13:44,625 - root - INFO - step: 11870 loss: 17.4699 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.7557 global_avg_mtp_loss: 14.7142 +[titan] 2025-07-09 16:13:44,625 - root - INFO - lr: 2.9205e-04 gnorm: 0.84 [ 2:38:48<19:39:03] +[titan] 2025-07-09 16:13:48,520 - root - INFO - step: 11875 loss: 17.2779 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.7135 global_avg_mtp_loss: 14.5644 +[titan] 2025-07-09 16:13:48,520 - root - INFO - lr: 2.9204e-04 gnorm: 0.83 [ 2:38:52<19:38:58] +[titan] 2025-07-09 16:13:52,411 - root - INFO - step: 11880 loss: 17.3779 memory: 44.58GiB(31.99%) tps: 84,218 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.7313 global_avg_mtp_loss: 14.6466 +[titan] 2025-07-09 16:13:52,412 - root - INFO - lr: 2.9203e-04 gnorm: 0.90 [ 2:38:55<19:38:53] +[titan] 2025-07-09 16:13:56,336 - root - INFO - step: 11885 loss: 17.3180 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.7321 global_avg_mtp_loss: 14.5859 +[titan] 2025-07-09 16:13:56,336 - root - INFO - lr: 2.9202e-04 gnorm: 0.88 [ 2:38:59<19:38:48] +[titan] 2025-07-09 16:14:00,249 - root - INFO - step: 11890 loss: 17.3194 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.7228 global_avg_mtp_loss: 14.5966 +[titan] 2025-07-09 16:14:00,250 - root - INFO - lr: 2.9202e-04 gnorm: 1.00 [ 2:39:03<19:38:43] +[titan] 2025-07-09 16:14:04,146 - root - INFO - step: 11895 loss: 17.1804 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.6922 global_avg_mtp_loss: 14.4882 +[titan] 2025-07-09 16:14:04,147 - root - INFO - lr: 2.9201e-04 gnorm: 0.82 [ 2:39:07<19:38:38] +[titan] 2025-07-09 16:14:07,284 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:14:08,072 - root - INFO - step: 11900 loss: 17.2648 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.7213 global_avg_mtp_loss: 14.5434 +[titan] 2025-07-09 16:14:08,072 - root - INFO - lr: 2.9200e-04 gnorm: 0.99 [ 2:39:11<19:38:34] +[titan] 2025-07-09 16:14:11,969 - root - INFO - step: 11905 loss: 17.2843 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.7250 global_avg_mtp_loss: 14.5593 +[titan] 2025-07-09 16:14:11,970 - root - INFO - lr: 2.9200e-04 gnorm: 0.81 [ 2:39:15<19:38:29] +[titan] 2025-07-09 16:14:15,901 - root - INFO - step: 11910 loss: 17.1544 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.6837 global_avg_mtp_loss: 14.4707 +[titan] 2025-07-09 16:14:15,901 - root - INFO - lr: 2.9199e-04 gnorm: 0.83 [ 2:39:19<19:38:24] +[titan] 2025-07-09 16:14:19,808 - root - INFO - step: 11915 loss: 17.3762 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7339 global_avg_mtp_loss: 14.6423 +[titan] 2025-07-09 16:14:19,809 - root - INFO - lr: 2.9198e-04 gnorm: 0.88 [ 2:39:23<19:38:20] +[titan] 2025-07-09 16:14:23,785 - root - INFO - step: 11920 loss: 17.1900 memory: 44.58GiB(31.99%) tps: 82,403 tflops: 284.39 mfu: 28.76% global_avg_ntp_loss: 2.7050 global_avg_mtp_loss: 14.4851 +[titan] 2025-07-09 16:14:23,785 - root - INFO - lr: 2.9197e-04 gnorm: 0.93 [ 2:39:27<19:38:15] +[titan] 2025-07-09 16:14:27,735 - root - INFO - step: 11925 loss: 16.9741 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.6474 global_avg_mtp_loss: 14.3267 +[titan] 2025-07-09 16:14:27,735 - root - INFO - lr: 2.9197e-04 gnorm: 0.82 [ 2:39:31<19:38:11] +[titan] 2025-07-09 16:14:31,664 - root - INFO - step: 11930 loss: 17.2101 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.7064 global_avg_mtp_loss: 14.5036 +[titan] 2025-07-09 16:14:31,665 - root - INFO - lr: 2.9196e-04 gnorm: 0.91 [ 2:39:35<19:38:06] +[titan] 2025-07-09 16:14:35,583 - root - INFO - step: 11935 loss: 17.1389 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.7131 global_avg_mtp_loss: 14.4258 +[titan] 2025-07-09 16:14:35,583 - root - INFO - lr: 2.9195e-04 gnorm: 0.87 [ 2:39:39<19:38:01] +[titan] 2025-07-09 16:14:39,513 - root - INFO - step: 11940 loss: 17.2597 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.7138 global_avg_mtp_loss: 14.5459 +[titan] 2025-07-09 16:14:39,514 - root - INFO - lr: 2.9194e-04 gnorm: 0.90 [ 2:39:43<19:37:57] +[titan] 2025-07-09 16:14:43,462 - root - INFO - step: 11945 loss: 17.2818 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.7247 global_avg_mtp_loss: 14.5571 +[titan] 2025-07-09 16:14:43,462 - root - INFO - lr: 2.9194e-04 gnorm: 0.87 [ 2:39:47<19:37:52] +[titan] 2025-07-09 16:14:46,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:14:47,371 - root - INFO - step: 11950 loss: 17.2327 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.7011 global_avg_mtp_loss: 14.5316 +[titan] 2025-07-09 16:14:47,371 - root - INFO - lr: 2.9193e-04 gnorm: 0.85 [ 2:39:50<19:37:48] +[titan] 2025-07-09 16:14:51,321 - root - INFO - step: 11955 loss: 17.1023 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6798 global_avg_mtp_loss: 14.4225 +[titan] 2025-07-09 16:14:51,322 - root - INFO - lr: 2.9192e-04 gnorm: 0.85 [ 2:39:54<19:37:43] +[titan] 2025-07-09 16:14:55,241 - root - INFO - step: 11960 loss: 17.4326 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.7442 global_avg_mtp_loss: 14.6883 +[titan] 2025-07-09 16:14:55,241 - root - INFO - lr: 2.9192e-04 gnorm: 0.88 [ 2:39:58<19:37:38] +[titan] 2025-07-09 16:14:59,158 - root - INFO - step: 11965 loss: 17.0947 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6763 global_avg_mtp_loss: 14.4184 +[titan] 2025-07-09 16:14:59,159 - root - INFO - lr: 2.9191e-04 gnorm: 0.83 [ 2:40:02<19:37:34] +[titan] 2025-07-09 16:15:03,073 - root - INFO - step: 11970 loss: 17.5513 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.7648 global_avg_mtp_loss: 14.7865 +[titan] 2025-07-09 16:15:03,073 - root - INFO - lr: 2.9190e-04 gnorm: 0.86 [ 2:40:06<19:37:29] +[titan] 2025-07-09 16:15:06,970 - root - INFO - step: 11975 loss: 17.1405 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6833 global_avg_mtp_loss: 14.4572 +[titan] 2025-07-09 16:15:06,970 - root - INFO - lr: 2.9189e-04 gnorm: 0.85 [ 2:40:10<19:37:24] +[titan] 2025-07-09 16:15:10,861 - root - INFO - step: 11980 loss: 17.3028 memory: 44.58GiB(31.99%) tps: 84,226 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.7176 global_avg_mtp_loss: 14.5852 +[titan] 2025-07-09 16:15:10,861 - root - INFO - lr: 2.9189e-04 gnorm: 0.84 [ 2:40:14<19:37:19] +[titan] 2025-07-09 16:15:14,802 - root - INFO - step: 11985 loss: 17.2639 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.7053 global_avg_mtp_loss: 14.5586 +[titan] 2025-07-09 16:15:14,802 - root - INFO - lr: 2.9188e-04 gnorm: 0.90 [ 2:40:18<19:37:15] +[titan] 2025-07-09 16:15:18,727 - root - INFO - step: 11990 loss: 17.3947 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.7662 global_avg_mtp_loss: 14.6286 +[titan] 2025-07-09 16:15:18,727 - root - INFO - lr: 2.9187e-04 gnorm: 0.95 [ 2:40:22<19:37:10] +[titan] 2025-07-09 16:15:22,639 - root - INFO - step: 11995 loss: 17.1706 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6836 global_avg_mtp_loss: 14.4870 +[titan] 2025-07-09 16:15:22,640 - root - INFO - lr: 2.9186e-04 gnorm: 0.93 [ 2:40:26<19:37:05] +[titan] 2025-07-09 16:15:25,759 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:15:26,562 - root - INFO - step: 12000 loss: 17.3879 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.7376 global_avg_mtp_loss: 14.6503 +[titan] 2025-07-09 16:15:26,562 - root - INFO - lr: 2.9186e-04 gnorm: 0.89 [ 2:40:30<19:37:00] +[titan] 2025-07-09 16:15:30,470 - root - INFO - step: 12005 loss: 17.3244 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.7163 global_avg_mtp_loss: 14.6081 +[titan] 2025-07-09 16:15:30,470 - root - INFO - lr: 2.9185e-04 gnorm: 0.88 [ 2:40:34<19:36:56] +[titan] 2025-07-09 16:15:34,421 - root - INFO - step: 12010 loss: 17.0271 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6787 global_avg_mtp_loss: 14.3484 +[titan] 2025-07-09 16:15:34,421 - root - INFO - lr: 2.9184e-04 gnorm: 0.82 [ 2:40:37<19:36:51] +[titan] 2025-07-09 16:15:38,342 - root - INFO - step: 12015 loss: 17.4052 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.7300 global_avg_mtp_loss: 14.6752 +[titan] 2025-07-09 16:15:38,342 - root - INFO - lr: 2.9184e-04 gnorm: 0.84 [ 2:40:41<19:36:47] +[titan] 2025-07-09 16:15:42,261 - root - INFO - step: 12020 loss: 17.4159 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.7345 global_avg_mtp_loss: 14.6814 +[titan] 2025-07-09 16:15:42,261 - root - INFO - lr: 2.9183e-04 gnorm: 0.86 [ 2:40:45<19:36:42] +[titan] 2025-07-09 16:15:46,168 - root - INFO - step: 12025 loss: 17.1871 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6850 global_avg_mtp_loss: 14.5022 +[titan] 2025-07-09 16:15:46,168 - root - INFO - lr: 2.9182e-04 gnorm: 0.87 [ 2:40:49<19:36:37] +[titan] 2025-07-09 16:15:50,073 - root - INFO - step: 12030 loss: 17.3962 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.7425 global_avg_mtp_loss: 14.6537 +[titan] 2025-07-09 16:15:50,073 - root - INFO - lr: 2.9181e-04 gnorm: 0.85 [ 2:40:53<19:36:32] +[titan] 2025-07-09 16:15:53,988 - root - INFO - step: 12035 loss: 17.4738 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7492 global_avg_mtp_loss: 14.7246 +[titan] 2025-07-09 16:15:53,988 - root - INFO - lr: 2.9181e-04 gnorm: 0.91 [ 2:40:57<19:36:28] +[titan] 2025-07-09 16:15:57,891 - root - INFO - step: 12040 loss: 17.1652 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.4700 +[titan] 2025-07-09 16:15:57,891 - root - INFO - lr: 2.9180e-04 gnorm: 0.93 [ 2:41:01<19:36:23] +[titan] 2025-07-09 16:16:01,784 - root - INFO - step: 12045 loss: 17.2759 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.7053 global_avg_mtp_loss: 14.5706 +[titan] 2025-07-09 16:16:01,785 - root - INFO - lr: 2.9179e-04 gnorm: 0.82 [ 2:41:05<19:36:18] +[titan] 2025-07-09 16:16:04,923 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:16:05,712 - root - INFO - step: 12050 loss: 17.4753 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.7513 global_avg_mtp_loss: 14.7240 +[titan] 2025-07-09 16:16:05,712 - root - INFO - lr: 2.9178e-04 gnorm: 0.85 [ 2:41:09<19:36:13] +[titan] 2025-07-09 16:16:09,608 - root - INFO - step: 12055 loss: 17.4840 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.7446 global_avg_mtp_loss: 14.7395 +[titan] 2025-07-09 16:16:09,608 - root - INFO - lr: 2.9178e-04 gnorm: 0.87 [ 2:41:13<19:36:08] +[titan] 2025-07-09 16:16:13,531 - root - INFO - step: 12060 loss: 17.2861 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.7316 global_avg_mtp_loss: 14.5545 +[titan] 2025-07-09 16:16:13,531 - root - INFO - lr: 2.9177e-04 gnorm: 0.80 [ 2:41:17<19:36:04] +[titan] 2025-07-09 16:16:17,428 - root - INFO - step: 12065 loss: 17.1103 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6891 global_avg_mtp_loss: 14.4212 +[titan] 2025-07-09 16:16:17,428 - root - INFO - lr: 2.9176e-04 gnorm: 0.85 [ 2:41:20<19:35:59] +[titan] 2025-07-09 16:16:21,329 - root - INFO - step: 12070 loss: 17.0806 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6697 global_avg_mtp_loss: 14.4109 +[titan] 2025-07-09 16:16:21,329 - root - INFO - lr: 2.9175e-04 gnorm: 0.90 [ 2:41:24<19:35:54] +[titan] 2025-07-09 16:16:25,231 - root - INFO - step: 12075 loss: 17.3817 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7483 global_avg_mtp_loss: 14.6334 +[titan] 2025-07-09 16:16:25,231 - root - INFO - lr: 2.9175e-04 gnorm: 0.93 [ 2:41:28<19:35:49] +[titan] 2025-07-09 16:16:29,154 - root - INFO - step: 12080 loss: 17.1374 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.4579 +[titan] 2025-07-09 16:16:29,155 - root - INFO - lr: 2.9174e-04 gnorm: 0.86 [ 2:41:32<19:35:45] +[titan] 2025-07-09 16:16:33,080 - root - INFO - step: 12085 loss: 17.0672 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6805 global_avg_mtp_loss: 14.3867 +[titan] 2025-07-09 16:16:33,080 - root - INFO - lr: 2.9173e-04 gnorm: 0.83 [ 2:41:36<19:35:40] +[titan] 2025-07-09 16:16:36,992 - root - INFO - step: 12090 loss: 17.4163 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.7432 global_avg_mtp_loss: 14.6731 +[titan] 2025-07-09 16:16:36,992 - root - INFO - lr: 2.9172e-04 gnorm: 0.83 [ 2:41:40<19:35:35] +[titan] 2025-07-09 16:16:40,893 - root - INFO - step: 12095 loss: 17.3003 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.7244 global_avg_mtp_loss: 14.5759 +[titan] 2025-07-09 16:16:40,893 - root - INFO - lr: 2.9172e-04 gnorm: 0.88 [ 2:41:44<19:35:30] +[titan] 2025-07-09 16:16:44,016 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:16:44,800 - root - INFO - step: 12100 loss: 17.0779 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4118 +[titan] 2025-07-09 16:16:44,800 - root - INFO - lr: 2.9171e-04 gnorm: 0.88 [ 2:41:48<19:35:26] +[titan] 2025-07-09 16:16:48,702 - root - INFO - step: 12105 loss: 17.3636 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.7421 global_avg_mtp_loss: 14.6216 +[titan] 2025-07-09 16:16:48,703 - root - INFO - lr: 2.9170e-04 gnorm: 1.09 [ 2:41:52<19:35:21] +[titan] 2025-07-09 16:16:52,588 - root - INFO - step: 12110 loss: 17.2696 memory: 44.58GiB(31.99%) tps: 84,337 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 2.7145 global_avg_mtp_loss: 14.5551 +[titan] 2025-07-09 16:16:52,588 - root - INFO - lr: 2.9170e-04 gnorm: 0.88 [ 2:41:56<19:35:16] +[titan] 2025-07-09 16:16:56,525 - root - INFO - step: 12115 loss: 17.2654 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.7104 global_avg_mtp_loss: 14.5549 +[titan] 2025-07-09 16:16:56,526 - root - INFO - lr: 2.9169e-04 gnorm: 0.87 [ 2:42:00<19:35:11] +[titan] 2025-07-09 16:17:00,429 - root - INFO - step: 12120 loss: 17.2736 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.7099 global_avg_mtp_loss: 14.5637 +[titan] 2025-07-09 16:17:00,429 - root - INFO - lr: 2.9168e-04 gnorm: 0.91 [ 2:42:03<19:35:06] +[titan] 2025-07-09 16:17:04,374 - root - INFO - step: 12125 loss: 17.4284 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.7446 global_avg_mtp_loss: 14.6838 +[titan] 2025-07-09 16:17:04,374 - root - INFO - lr: 2.9167e-04 gnorm: 0.85 [ 2:42:07<19:35:02] +[titan] 2025-07-09 16:17:08,267 - root - INFO - step: 12130 loss: 17.1658 memory: 44.58GiB(31.99%) tps: 84,185 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.6917 global_avg_mtp_loss: 14.4741 +[titan] 2025-07-09 16:17:08,267 - root - INFO - lr: 2.9167e-04 gnorm: 0.98 [ 2:42:11<19:34:57] +[titan] 2025-07-09 16:17:12,160 - root - INFO - step: 12135 loss: 17.2432 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.7135 global_avg_mtp_loss: 14.5296 +[titan] 2025-07-09 16:17:12,160 - root - INFO - lr: 2.9166e-04 gnorm: 0.89 [ 2:42:15<19:34:52] +[titan] 2025-07-09 16:17:16,052 - root - INFO - step: 12140 loss: 17.4389 memory: 44.58GiB(31.99%) tps: 84,199 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.7456 global_avg_mtp_loss: 14.6933 +[titan] 2025-07-09 16:17:16,053 - root - INFO - lr: 2.9165e-04 gnorm: 0.88 [ 2:42:19<19:34:47] +[titan] 2025-07-09 16:17:19,962 - root - INFO - step: 12145 loss: 17.1954 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6838 global_avg_mtp_loss: 14.5115 +[titan] 2025-07-09 16:17:19,963 - root - INFO - lr: 2.9164e-04 gnorm: 0.88 [ 2:42:23<19:34:43] +[titan] 2025-07-09 16:17:23,076 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:17:23,861 - root - INFO - step: 12150 loss: 17.2374 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6985 global_avg_mtp_loss: 14.5389 +[titan] 2025-07-09 16:17:23,861 - root - INFO - lr: 2.9164e-04 gnorm: 0.92 [ 2:42:27<19:34:38] +[titan] 2025-07-09 16:17:27,763 - root - INFO - step: 12155 loss: 17.4567 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7434 global_avg_mtp_loss: 14.7132 +[titan] 2025-07-09 16:17:27,763 - root - INFO - lr: 2.9163e-04 gnorm: 0.89 [ 2:42:31<19:34:33] +[titan] 2025-07-09 16:17:31,679 - root - INFO - step: 12160 loss: 17.3765 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.7384 global_avg_mtp_loss: 14.6382 +[titan] 2025-07-09 16:17:31,679 - root - INFO - lr: 2.9162e-04 gnorm: 0.98 [ 2:42:35<19:34:28] +[titan] 2025-07-09 16:17:35,579 - root - INFO - step: 12165 loss: 17.4096 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.7352 global_avg_mtp_loss: 14.6744 +[titan] 2025-07-09 16:17:35,579 - root - INFO - lr: 2.9161e-04 gnorm: 0.95 [ 2:42:39<19:34:23] +[titan] 2025-07-09 16:17:39,527 - root - INFO - step: 12170 loss: 17.4587 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.7387 global_avg_mtp_loss: 14.7200 +[titan] 2025-07-09 16:17:39,528 - root - INFO - lr: 2.9161e-04 gnorm: 0.95 [ 2:42:43<19:34:19] +[titan] 2025-07-09 16:17:43,425 - root - INFO - step: 12175 loss: 17.2733 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.7082 global_avg_mtp_loss: 14.5651 +[titan] 2025-07-09 16:17:43,425 - root - INFO - lr: 2.9160e-04 gnorm: 0.93 [ 2:42:46<19:34:14] +[titan] 2025-07-09 16:17:47,356 - root - INFO - step: 12180 loss: 17.6142 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.7692 global_avg_mtp_loss: 14.8451 +[titan] 2025-07-09 16:17:47,356 - root - INFO - lr: 2.9159e-04 gnorm: 0.79 [ 2:42:50<19:34:10] +[titan] 2025-07-09 16:17:51,268 - root - INFO - step: 12185 loss: 16.8523 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.2156 +[titan] 2025-07-09 16:17:51,268 - root - INFO - lr: 2.9158e-04 gnorm: 0.85 [ 2:42:54<19:34:05] +[titan] 2025-07-09 16:17:55,176 - root - INFO - step: 12190 loss: 17.3720 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.7232 global_avg_mtp_loss: 14.6488 +[titan] 2025-07-09 16:17:55,177 - root - INFO - lr: 2.9158e-04 gnorm: 0.89 [ 2:42:58<19:34:00] +[titan] 2025-07-09 16:17:59,099 - root - INFO - step: 12195 loss: 17.6685 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.7923 global_avg_mtp_loss: 14.8762 +[titan] 2025-07-09 16:17:59,099 - root - INFO - lr: 2.9157e-04 gnorm: 1.06 [ 2:43:02<19:33:55] +[titan] 2025-07-09 16:18:02,213 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:18:03,013 - root - INFO - step: 12200 loss: 17.3499 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.7345 global_avg_mtp_loss: 14.6153 +[titan] 2025-07-09 16:18:03,013 - root - INFO - lr: 2.9156e-04 gnorm: 0.88 [ 2:43:06<19:33:51] +[titan] 2025-07-09 16:18:06,934 - root - INFO - step: 12205 loss: 17.2668 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7117 global_avg_mtp_loss: 14.5551 +[titan] 2025-07-09 16:18:06,934 - root - INFO - lr: 2.9155e-04 gnorm: 0.84 [ 2:43:10<19:33:46] +[titan] 2025-07-09 16:18:10,865 - root - INFO - step: 12210 loss: 17.2323 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.7063 global_avg_mtp_loss: 14.5260 +[titan] 2025-07-09 16:18:10,865 - root - INFO - lr: 2.9155e-04 gnorm: 0.87 [ 2:43:14<19:33:41] +[titan] 2025-07-09 16:18:14,777 - root - INFO - step: 12215 loss: 17.2423 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.7053 global_avg_mtp_loss: 14.5370 +[titan] 2025-07-09 16:18:14,777 - root - INFO - lr: 2.9154e-04 gnorm: 0.84 [ 2:43:18<19:33:37] +[titan] 2025-07-09 16:18:18,680 - root - INFO - step: 12220 loss: 17.3717 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7201 global_avg_mtp_loss: 14.6516 +[titan] 2025-07-09 16:18:18,680 - root - INFO - lr: 2.9153e-04 gnorm: 0.92 [ 2:43:22<19:33:32] +[titan] 2025-07-09 16:18:22,613 - root - INFO - step: 12225 loss: 17.2990 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.7330 global_avg_mtp_loss: 14.5660 +[titan] 2025-07-09 16:18:22,613 - root - INFO - lr: 2.9152e-04 gnorm: 0.92 [ 2:43:26<19:33:27] +[titan] 2025-07-09 16:18:26,558 - root - INFO - step: 12230 loss: 17.4619 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.7517 global_avg_mtp_loss: 14.7102 +[titan] 2025-07-09 16:18:26,558 - root - INFO - lr: 2.9152e-04 gnorm: 0.98 [ 2:43:30<19:33:23] +[titan] 2025-07-09 16:18:30,502 - root - INFO - step: 12235 loss: 17.2617 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6992 global_avg_mtp_loss: 14.5625 +[titan] 2025-07-09 16:18:30,502 - root - INFO - lr: 2.9151e-04 gnorm: 0.90 [ 2:43:34<19:33:18] +[titan] 2025-07-09 16:18:34,421 - root - INFO - step: 12240 loss: 17.1447 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.4533 +[titan] 2025-07-09 16:18:34,422 - root - INFO - lr: 2.9150e-04 gnorm: 0.94 [ 2:43:37<19:33:14] +[titan] 2025-07-09 16:18:38,309 - root - INFO - step: 12245 loss: 17.2565 memory: 44.58GiB(31.99%) tps: 84,307 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.7000 global_avg_mtp_loss: 14.5564 +[titan] 2025-07-09 16:18:38,309 - root - INFO - lr: 2.9149e-04 gnorm: 0.95 [ 2:43:41<19:33:09] +[titan] 2025-07-09 16:18:41,414 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:18:42,205 - root - INFO - step: 12250 loss: 17.1687 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6936 global_avg_mtp_loss: 14.4751 +[titan] 2025-07-09 16:18:42,206 - root - INFO - lr: 2.9149e-04 gnorm: 0.86 [ 2:43:45<19:33:04] +[titan] 2025-07-09 16:18:46,097 - root - INFO - step: 12255 loss: 17.1844 memory: 44.58GiB(31.99%) tps: 84,211 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.6920 global_avg_mtp_loss: 14.4924 +[titan] 2025-07-09 16:18:46,097 - root - INFO - lr: 2.9148e-04 gnorm: 0.92 [ 2:43:49<19:32:59] +[titan] 2025-07-09 16:18:50,003 - root - INFO - step: 12260 loss: 17.1118 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6824 global_avg_mtp_loss: 14.4295 +[titan] 2025-07-09 16:18:50,003 - root - INFO - lr: 2.9147e-04 gnorm: 0.89 [ 2:43:53<19:32:54] +[titan] 2025-07-09 16:18:53,904 - root - INFO - step: 12265 loss: 17.2896 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5774 +[titan] 2025-07-09 16:18:53,904 - root - INFO - lr: 2.9146e-04 gnorm: 0.84 [ 2:43:57<19:32:50] +[titan] 2025-07-09 16:18:57,821 - root - INFO - step: 12270 loss: 17.3951 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.7283 global_avg_mtp_loss: 14.6667 +[titan] 2025-07-09 16:18:57,821 - root - INFO - lr: 2.9146e-04 gnorm: 0.91 [ 2:44:01<19:32:45] +[titan] 2025-07-09 16:19:01,746 - root - INFO - step: 12275 loss: 17.4018 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.7372 global_avg_mtp_loss: 14.6646 +[titan] 2025-07-09 16:19:01,746 - root - INFO - lr: 2.9145e-04 gnorm: 0.85 [ 2:44:05<19:32:40] +[titan] 2025-07-09 16:19:05,641 - root - INFO - step: 12280 loss: 16.9172 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6469 global_avg_mtp_loss: 14.2703 +[titan] 2025-07-09 16:19:05,641 - root - INFO - lr: 2.9144e-04 gnorm: 0.89 [ 2:44:09<19:32:35] +[titan] 2025-07-09 16:19:09,613 - root - INFO - step: 12285 loss: 17.0490 memory: 44.58GiB(31.99%) tps: 82,519 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.6827 global_avg_mtp_loss: 14.3663 +[titan] 2025-07-09 16:19:09,613 - root - INFO - lr: 2.9143e-04 gnorm: 0.94 [ 2:44:13<19:32:31] +[titan] 2025-07-09 16:19:12,116 - root - INFO - Dumping profiler traces at step 12288 +[titan] 2025-07-09 16:19:12,148 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:19:13,731 - root - INFO - step: 12290 loss: 17.3006 memory: 44.58GiB(31.99%) tps: 79,562 tflops: 274.58 mfu: 27.76% global_avg_ntp_loss: 2.7254 global_avg_mtp_loss: 14.5752 +[titan] 2025-07-09 16:19:13,732 - root - INFO - lr: 2.9143e-04 gnorm: 1.12 [ 2:44:17<19:32:28] +[titan] 2025-07-09 16:19:17,653 - root - INFO - step: 12295 loss: 17.2836 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.7113 global_avg_mtp_loss: 14.5723 +[titan] 2025-07-09 16:19:17,653 - root - INFO - lr: 2.9142e-04 gnorm: 0.93 [ 2:44:21<19:32:23] +[titan] 2025-07-09 16:19:20,762 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:19:21,567 - root - INFO - step: 12300 loss: 17.2564 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.7042 global_avg_mtp_loss: 14.5522 +[titan] 2025-07-09 16:19:21,568 - root - INFO - lr: 2.9141e-04 gnorm: 0.90 [ 2:44:25<19:32:19] +[titan] 2025-07-09 16:19:25,496 - root - INFO - step: 12305 loss: 17.1335 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4441 +[titan] 2025-07-09 16:19:25,497 - root - INFO - lr: 2.9140e-04 gnorm: 0.95 [ 2:44:29<19:32:14] +[titan] 2025-07-09 16:19:29,435 - root - INFO - step: 12310 loss: 17.4863 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.7421 global_avg_mtp_loss: 14.7442 +[titan] 2025-07-09 16:19:29,436 - root - INFO - lr: 2.9140e-04 gnorm: 0.92 [ 2:44:32<19:32:09] +[titan] 2025-07-09 16:19:33,344 - root - INFO - step: 12315 loss: 17.1489 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6910 global_avg_mtp_loss: 14.4580 +[titan] 2025-07-09 16:19:33,344 - root - INFO - lr: 2.9139e-04 gnorm: 0.84 [ 2:44:36<19:32:05] +[titan] 2025-07-09 16:19:37,294 - root - INFO - step: 12320 loss: 17.0655 memory: 44.58GiB(31.99%) tps: 82,963 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.6740 global_avg_mtp_loss: 14.3916 +[titan] 2025-07-09 16:19:37,294 - root - INFO - lr: 2.9138e-04 gnorm: 0.99 [ 2:44:40<19:32:00] +[titan] 2025-07-09 16:19:41,185 - root - INFO - step: 12325 loss: 17.0626 memory: 44.58GiB(31.99%) tps: 84,230 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.3870 +[titan] 2025-07-09 16:19:41,185 - root - INFO - lr: 2.9137e-04 gnorm: 0.85 [ 2:44:44<19:31:55] +[titan] 2025-07-09 16:19:45,120 - root - INFO - step: 12330 loss: 17.2137 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.7102 global_avg_mtp_loss: 14.5035 +[titan] 2025-07-09 16:19:45,121 - root - INFO - lr: 2.9137e-04 gnorm: 0.92 [ 2:44:48<19:31:51] +[titan] 2025-07-09 16:19:49,026 - root - INFO - step: 12335 loss: 17.3626 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.7259 global_avg_mtp_loss: 14.6367 +[titan] 2025-07-09 16:19:49,026 - root - INFO - lr: 2.9136e-04 gnorm: 0.89 [ 2:44:52<19:31:46] +[titan] 2025-07-09 16:19:52,988 - root - INFO - step: 12340 loss: 17.2802 memory: 44.58GiB(31.99%) tps: 82,719 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.7065 global_avg_mtp_loss: 14.5736 +[titan] 2025-07-09 16:19:52,988 - root - INFO - lr: 2.9135e-04 gnorm: 0.94 [ 2:44:56<19:31:42] +[titan] 2025-07-09 16:19:56,899 - root - INFO - step: 12345 loss: 17.0527 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6759 global_avg_mtp_loss: 14.3768 +[titan] 2025-07-09 16:19:56,900 - root - INFO - lr: 2.9134e-04 gnorm: 0.89 [ 2:45:00<19:31:37] +[titan] 2025-07-09 16:20:00,029 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:20:00,819 - root - INFO - step: 12350 loss: 17.1903 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.4952 +[titan] 2025-07-09 16:20:00,819 - root - INFO - lr: 2.9134e-04 gnorm: 0.86 [ 2:45:04<19:31:32] +[titan] 2025-07-09 16:20:04,737 - root - INFO - step: 12355 loss: 17.2766 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7146 global_avg_mtp_loss: 14.5620 +[titan] 2025-07-09 16:20:04,738 - root - INFO - lr: 2.9133e-04 gnorm: 0.80 [ 2:45:08<19:31:28] +[titan] 2025-07-09 16:20:08,624 - root - INFO - step: 12360 loss: 17.1353 memory: 44.58GiB(31.99%) tps: 84,316 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.6864 global_avg_mtp_loss: 14.4489 +[titan] 2025-07-09 16:20:08,624 - root - INFO - lr: 2.9132e-04 gnorm: 0.84 [ 2:45:12<19:31:23] +[titan] 2025-07-09 16:20:12,536 - root - INFO - step: 12365 loss: 17.3976 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.7319 global_avg_mtp_loss: 14.6657 +[titan] 2025-07-09 16:20:12,536 - root - INFO - lr: 2.9131e-04 gnorm: 0.88 [ 2:45:16<19:31:18] +[titan] 2025-07-09 16:20:16,438 - root - INFO - step: 12370 loss: 17.3697 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.7282 global_avg_mtp_loss: 14.6415 +[titan] 2025-07-09 16:20:16,438 - root - INFO - lr: 2.9131e-04 gnorm: 0.92 [ 2:45:19<19:31:13] +[titan] 2025-07-09 16:20:20,353 - root - INFO - step: 12375 loss: 16.8961 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 16:20:20,353 - root - INFO - lr: 2.9130e-04 gnorm: 0.94 [ 2:45:23<19:31:09] +[titan] 2025-07-09 16:20:24,274 - root - INFO - step: 12380 loss: 17.3353 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.7318 global_avg_mtp_loss: 14.6035 +[titan] 2025-07-09 16:20:24,274 - root - INFO - lr: 2.9129e-04 gnorm: 0.87 [ 2:45:27<19:31:04] +[titan] 2025-07-09 16:20:28,173 - root - INFO - step: 12385 loss: 17.0483 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6668 global_avg_mtp_loss: 14.3816 +[titan] 2025-07-09 16:20:28,173 - root - INFO - lr: 2.9128e-04 gnorm: 0.87 [ 2:45:31<19:30:59] +[titan] 2025-07-09 16:20:32,094 - root - INFO - step: 12390 loss: 17.4479 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.7512 global_avg_mtp_loss: 14.6967 +[titan] 2025-07-09 16:20:32,094 - root - INFO - lr: 2.9128e-04 gnorm: 0.93 [ 2:45:35<19:30:55] +[titan] 2025-07-09 16:20:36,035 - root - INFO - step: 12395 loss: 17.6116 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.7804 global_avg_mtp_loss: 14.8312 +[titan] 2025-07-09 16:20:36,035 - root - INFO - lr: 2.9127e-04 gnorm: 0.88 [ 2:45:39<19:30:50] +[titan] 2025-07-09 16:20:39,167 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:20:39,963 - root - INFO - step: 12400 loss: 17.1759 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6987 global_avg_mtp_loss: 14.4772 +[titan] 2025-07-09 16:20:39,963 - root - INFO - lr: 2.9126e-04 gnorm: 0.90 [ 2:45:43<19:30:45] +[titan] 2025-07-09 16:20:43,893 - root - INFO - step: 12405 loss: 17.3190 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.7093 global_avg_mtp_loss: 14.6096 +[titan] 2025-07-09 16:20:43,893 - root - INFO - lr: 2.9125e-04 gnorm: 0.90 [ 2:45:47<19:30:41] +[titan] 2025-07-09 16:20:47,867 - root - INFO - step: 12410 loss: 17.1475 memory: 44.58GiB(31.99%) tps: 82,461 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.6778 global_avg_mtp_loss: 14.4698 +[titan] 2025-07-09 16:20:47,867 - root - INFO - lr: 2.9125e-04 gnorm: 0.87 [ 2:45:51<19:30:37] +[titan] 2025-07-09 16:20:51,776 - root - INFO - step: 12415 loss: 17.2133 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6892 global_avg_mtp_loss: 14.5242 +[titan] 2025-07-09 16:20:51,776 - root - INFO - lr: 2.9124e-04 gnorm: 0.86 [ 2:45:55<19:30:32] +[titan] 2025-07-09 16:20:55,671 - root - INFO - step: 12420 loss: 17.1177 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.4395 +[titan] 2025-07-09 16:20:55,672 - root - INFO - lr: 2.9123e-04 gnorm: 0.90 [ 2:45:59<19:30:27] +[titan] 2025-07-09 16:20:59,590 - root - INFO - step: 12425 loss: 17.0719 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6761 global_avg_mtp_loss: 14.3959 +[titan] 2025-07-09 16:20:59,590 - root - INFO - lr: 2.9122e-04 gnorm: 0.87 [ 2:46:03<19:30:22] +[titan] 2025-07-09 16:21:03,475 - root - INFO - step: 12430 loss: 17.3298 memory: 44.58GiB(31.99%) tps: 84,347 tflops: 291.10 mfu: 29.43% global_avg_ntp_loss: 2.7195 global_avg_mtp_loss: 14.6103 +[titan] 2025-07-09 16:21:03,475 - root - INFO - lr: 2.9122e-04 gnorm: 0.83 [ 2:46:07<19:30:18] +[titan] 2025-07-09 16:21:07,377 - root - INFO - step: 12435 loss: 17.0939 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6775 global_avg_mtp_loss: 14.4165 +[titan] 2025-07-09 16:21:07,377 - root - INFO - lr: 2.9121e-04 gnorm: 0.85 [ 2:46:10<19:30:13] +[titan] 2025-07-09 16:21:11,309 - root - INFO - step: 12440 loss: 17.1913 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6941 global_avg_mtp_loss: 14.4972 +[titan] 2025-07-09 16:21:11,310 - root - INFO - lr: 2.9120e-04 gnorm: 0.83 [ 2:46:14<19:30:08] +[titan] 2025-07-09 16:21:15,208 - root - INFO - step: 12445 loss: 17.4036 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.7403 global_avg_mtp_loss: 14.6633 +[titan] 2025-07-09 16:21:15,208 - root - INFO - lr: 2.9119e-04 gnorm: 1.02 [ 2:46:18<19:30:03] +[titan] 2025-07-09 16:21:18,348 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:21:19,149 - root - INFO - step: 12450 loss: 17.3579 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.7340 global_avg_mtp_loss: 14.6239 +[titan] 2025-07-09 16:21:19,149 - root - INFO - lr: 2.9118e-04 gnorm: 0.93 [ 2:46:22<19:29:59] +[titan] 2025-07-09 16:21:23,091 - root - INFO - step: 12455 loss: 17.1585 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4691 +[titan] 2025-07-09 16:21:23,092 - root - INFO - lr: 2.9118e-04 gnorm: 0.86 [ 2:46:26<19:29:54] +[titan] 2025-07-09 16:21:27,000 - root - INFO - step: 12460 loss: 17.4368 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.7637 global_avg_mtp_loss: 14.6731 +[titan] 2025-07-09 16:21:27,000 - root - INFO - lr: 2.9117e-04 gnorm: 0.91 [ 2:46:30<19:29:50] +[titan] 2025-07-09 16:21:30,900 - root - INFO - step: 12465 loss: 17.2296 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.5345 +[titan] 2025-07-09 16:21:30,900 - root - INFO - lr: 2.9116e-04 gnorm: 0.82 [ 2:46:34<19:29:45] +[titan] 2025-07-09 16:21:34,797 - root - INFO - step: 12470 loss: 17.1458 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.6864 global_avg_mtp_loss: 14.4594 +[titan] 2025-07-09 16:21:34,797 - root - INFO - lr: 2.9115e-04 gnorm: 0.81 [ 2:46:38<19:29:40] +[titan] 2025-07-09 16:21:38,726 - root - INFO - step: 12475 loss: 17.0118 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6612 global_avg_mtp_loss: 14.3506 +[titan] 2025-07-09 16:21:38,726 - root - INFO - lr: 2.9115e-04 gnorm: 0.84 [ 2:46:42<19:29:36] +[titan] 2025-07-09 16:21:42,629 - root - INFO - step: 12480 loss: 17.0105 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6669 global_avg_mtp_loss: 14.3436 +[titan] 2025-07-09 16:21:42,629 - root - INFO - lr: 2.9114e-04 gnorm: 0.83 [ 2:46:46<19:29:31] +[titan] 2025-07-09 16:21:46,554 - root - INFO - step: 12485 loss: 17.3257 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.7292 global_avg_mtp_loss: 14.5965 +[titan] 2025-07-09 16:21:46,554 - root - INFO - lr: 2.9113e-04 gnorm: 0.91 [ 2:46:50<19:29:26] +[titan] 2025-07-09 16:21:50,461 - root - INFO - step: 12490 loss: 17.2055 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6984 global_avg_mtp_loss: 14.5071 +[titan] 2025-07-09 16:21:50,461 - root - INFO - lr: 2.9112e-04 gnorm: 0.88 [ 2:46:53<19:29:22] +[titan] 2025-07-09 16:21:54,372 - root - INFO - step: 12495 loss: 16.8377 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.2010 +[titan] 2025-07-09 16:21:54,372 - root - INFO - lr: 2.9112e-04 gnorm: 0.90 [ 2:46:57<19:29:17] +[titan] 2025-07-09 16:21:57,513 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:21:58,298 - root - INFO - step: 12500 loss: 17.0038 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6657 global_avg_mtp_loss: 14.3381 +[titan] 2025-07-09 16:21:58,298 - root - INFO - lr: 2.9111e-04 gnorm: 0.88 [ 2:47:01<19:29:12] +[titan] 2025-07-09 16:22:02,204 - root - INFO - step: 12505 loss: 17.1511 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6937 global_avg_mtp_loss: 14.4574 +[titan] 2025-07-09 16:22:02,204 - root - INFO - lr: 2.9110e-04 gnorm: 0.93 [ 2:47:05<19:29:08] +[titan] 2025-07-09 16:22:06,102 - root - INFO - step: 12510 loss: 17.1754 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6902 global_avg_mtp_loss: 14.4852 +[titan] 2025-07-09 16:22:06,102 - root - INFO - lr: 2.9109e-04 gnorm: 0.91 [ 2:47:09<19:29:03] +[titan] 2025-07-09 16:22:10,027 - root - INFO - step: 12515 loss: 17.4582 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.7611 global_avg_mtp_loss: 14.6971 +[titan] 2025-07-09 16:22:10,028 - root - INFO - lr: 2.9109e-04 gnorm: 0.83 [ 2:47:13<19:28:58] +[titan] 2025-07-09 16:22:13,933 - root - INFO - step: 12520 loss: 17.2588 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5466 +[titan] 2025-07-09 16:22:13,934 - root - INFO - lr: 2.9108e-04 gnorm: 0.90 [ 2:47:17<19:28:53] +[titan] 2025-07-09 16:22:17,830 - root - INFO - step: 12525 loss: 17.2730 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.7123 global_avg_mtp_loss: 14.5608 +[titan] 2025-07-09 16:22:17,831 - root - INFO - lr: 2.9107e-04 gnorm: 0.90 [ 2:47:21<19:28:49] +[titan] 2025-07-09 16:22:21,728 - root - INFO - step: 12530 loss: 17.1514 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6836 global_avg_mtp_loss: 14.4678 +[titan] 2025-07-09 16:22:21,728 - root - INFO - lr: 2.9106e-04 gnorm: 0.95 [ 2:47:25<19:28:44] +[titan] 2025-07-09 16:22:25,668 - root - INFO - step: 12535 loss: 17.3345 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.7163 global_avg_mtp_loss: 14.6183 +[titan] 2025-07-09 16:22:25,668 - root - INFO - lr: 2.9106e-04 gnorm: 0.85 [ 2:47:29<19:28:39] +[titan] 2025-07-09 16:22:29,580 - root - INFO - step: 12540 loss: 17.0652 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6809 global_avg_mtp_loss: 14.3843 +[titan] 2025-07-09 16:22:29,580 - root - INFO - lr: 2.9105e-04 gnorm: 0.85 [ 2:47:33<19:28:35] +[titan] 2025-07-09 16:22:33,500 - root - INFO - step: 12545 loss: 17.1068 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6829 global_avg_mtp_loss: 14.4239 +[titan] 2025-07-09 16:22:33,501 - root - INFO - lr: 2.9104e-04 gnorm: 0.91 [ 2:47:37<19:28:30] +[titan] 2025-07-09 16:22:36,614 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:22:37,405 - root - INFO - step: 12550 loss: 17.0653 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6684 global_avg_mtp_loss: 14.3969 +[titan] 2025-07-09 16:22:37,405 - root - INFO - lr: 2.9103e-04 gnorm: 0.88 [ 2:47:40<19:28:25] +[titan] 2025-07-09 16:22:41,321 - root - INFO - step: 12555 loss: 17.2758 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.7208 global_avg_mtp_loss: 14.5550 +[titan] 2025-07-09 16:22:41,321 - root - INFO - lr: 2.9102e-04 gnorm: 0.88 [ 2:47:44<19:28:21] +[titan] 2025-07-09 16:22:45,221 - root - INFO - step: 12560 loss: 17.1754 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6912 global_avg_mtp_loss: 14.4842 +[titan] 2025-07-09 16:22:45,222 - root - INFO - lr: 2.9102e-04 gnorm: 0.89 [ 2:47:48<19:28:16] +[titan] 2025-07-09 16:22:49,139 - root - INFO - step: 12565 loss: 17.1280 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6880 global_avg_mtp_loss: 14.4401 +[titan] 2025-07-09 16:22:49,140 - root - INFO - lr: 2.9101e-04 gnorm: 0.97 [ 2:47:52<19:28:11] +[titan] 2025-07-09 16:22:53,048 - root - INFO - step: 12570 loss: 17.1789 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6908 global_avg_mtp_loss: 14.4881 +[titan] 2025-07-09 16:22:53,048 - root - INFO - lr: 2.9100e-04 gnorm: 1.00 [ 2:47:56<19:28:07] +[titan] 2025-07-09 16:22:56,948 - root - INFO - step: 12575 loss: 17.4609 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.7341 global_avg_mtp_loss: 14.7268 +[titan] 2025-07-09 16:22:56,948 - root - INFO - lr: 2.9099e-04 gnorm: 0.86 [ 2:48:00<19:28:02] +[titan] 2025-07-09 16:23:00,846 - root - INFO - step: 12580 loss: 16.8270 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6255 global_avg_mtp_loss: 14.2015 +[titan] 2025-07-09 16:23:00,846 - root - INFO - lr: 2.9099e-04 gnorm: 0.83 [ 2:48:04<19:27:57] +[titan] 2025-07-09 16:23:04,738 - root - INFO - step: 12585 loss: 16.9590 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.2949 +[titan] 2025-07-09 16:23:04,738 - root - INFO - lr: 2.9098e-04 gnorm: 0.86 [ 2:48:08<19:27:52] +[titan] 2025-07-09 16:23:08,622 - root - INFO - step: 12590 loss: 17.5468 memory: 44.58GiB(31.99%) tps: 84,368 tflops: 291.17 mfu: 29.44% global_avg_ntp_loss: 2.7681 global_avg_mtp_loss: 14.7787 +[titan] 2025-07-09 16:23:08,622 - root - INFO - lr: 2.9097e-04 gnorm: 0.87 [ 2:48:12<19:27:47] +[titan] 2025-07-09 16:23:12,559 - root - INFO - step: 12595 loss: 17.3071 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.7171 global_avg_mtp_loss: 14.5899 +[titan] 2025-07-09 16:23:12,559 - root - INFO - lr: 2.9096e-04 gnorm: 0.88 [ 2:48:16<19:27:43] +[titan] 2025-07-09 16:23:15,687 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:23:16,487 - root - INFO - step: 12600 loss: 17.2403 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.7094 global_avg_mtp_loss: 14.5309 +[titan] 2025-07-09 16:23:16,488 - root - INFO - lr: 2.9096e-04 gnorm: 0.88 [ 2:48:20<19:27:38] +[titan] 2025-07-09 16:23:20,399 - root - INFO - step: 12605 loss: 17.6318 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.7900 global_avg_mtp_loss: 14.8417 +[titan] 2025-07-09 16:23:20,400 - root - INFO - lr: 2.9095e-04 gnorm: 0.87 [ 2:48:23<19:27:34] +[titan] 2025-07-09 16:23:24,318 - root - INFO - step: 12610 loss: 17.2845 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7312 global_avg_mtp_loss: 14.5532 +[titan] 2025-07-09 16:23:24,318 - root - INFO - lr: 2.9094e-04 gnorm: 0.99 [ 2:48:27<19:27:29] +[titan] 2025-07-09 16:23:28,267 - root - INFO - step: 12615 loss: 17.4353 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.7483 global_avg_mtp_loss: 14.6870 +[titan] 2025-07-09 16:23:28,268 - root - INFO - lr: 2.9093e-04 gnorm: 0.90 [ 2:48:31<19:27:25] +[titan] 2025-07-09 16:23:32,177 - root - INFO - step: 12620 loss: 17.3270 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.7240 global_avg_mtp_loss: 14.6030 +[titan] 2025-07-09 16:23:32,177 - root - INFO - lr: 2.9092e-04 gnorm: 0.99 [ 2:48:35<19:27:20] +[titan] 2025-07-09 16:23:36,062 - root - INFO - step: 12625 loss: 17.7485 memory: 44.58GiB(31.99%) tps: 84,333 tflops: 291.05 mfu: 29.43% global_avg_ntp_loss: 2.8132 global_avg_mtp_loss: 14.9354 +[titan] 2025-07-09 16:23:36,063 - root - INFO - lr: 2.9092e-04 gnorm: 1.12 [ 2:48:39<19:27:15] +[titan] 2025-07-09 16:23:40,045 - root - INFO - step: 12630 loss: 17.1449 memory: 44.58GiB(31.99%) tps: 82,283 tflops: 283.97 mfu: 28.71% global_avg_ntp_loss: 2.6825 global_avg_mtp_loss: 14.4624 +[titan] 2025-07-09 16:23:40,045 - root - INFO - lr: 2.9091e-04 gnorm: 0.87 [ 2:48:43<19:27:11] +[titan] 2025-07-09 16:23:43,972 - root - INFO - step: 12635 loss: 17.3516 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.7208 global_avg_mtp_loss: 14.6308 +[titan] 2025-07-09 16:23:43,972 - root - INFO - lr: 2.9090e-04 gnorm: 0.88 [ 2:48:47<19:27:06] +[titan] 2025-07-09 16:23:47,905 - root - INFO - step: 12640 loss: 17.2564 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.6939 global_avg_mtp_loss: 14.5626 +[titan] 2025-07-09 16:23:47,905 - root - INFO - lr: 2.9089e-04 gnorm: 0.84 [ 2:48:51<19:27:02] +[titan] 2025-07-09 16:23:51,810 - root - INFO - step: 12645 loss: 17.2926 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.7128 global_avg_mtp_loss: 14.5798 +[titan] 2025-07-09 16:23:51,810 - root - INFO - lr: 2.9089e-04 gnorm: 0.89 [ 2:48:55<19:26:57] +[titan] 2025-07-09 16:23:54,951 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:23:55,737 - root - INFO - step: 12650 loss: 17.1271 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.7242 global_avg_mtp_loss: 14.4029 +[titan] 2025-07-09 16:23:55,738 - root - INFO - lr: 2.9088e-04 gnorm: 10.70 [ 2:48:59<19:26:52] +[titan] 2025-07-09 16:23:59,650 - root - INFO - step: 12655 loss: 17.1239 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.4393 +[titan] 2025-07-09 16:23:59,650 - root - INFO - lr: 2.9087e-04 gnorm: 0.99 [ 2:49:03<19:26:48] +[titan] 2025-07-09 16:24:03,565 - root - INFO - step: 12660 loss: 17.3368 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.7242 global_avg_mtp_loss: 14.6126 +[titan] 2025-07-09 16:24:03,565 - root - INFO - lr: 2.9086e-04 gnorm: 0.93 [ 2:49:07<19:26:43] +[titan] 2025-07-09 16:24:07,477 - root - INFO - step: 12665 loss: 17.5663 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7736 global_avg_mtp_loss: 14.7927 +[titan] 2025-07-09 16:24:07,478 - root - INFO - lr: 2.9085e-04 gnorm: 0.89 [ 2:49:10<19:26:38] +[titan] 2025-07-09 16:24:11,370 - root - INFO - step: 12670 loss: 17.0406 memory: 44.58GiB(31.99%) tps: 84,183 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.6585 global_avg_mtp_loss: 14.3821 +[titan] 2025-07-09 16:24:11,371 - root - INFO - lr: 2.9085e-04 gnorm: 0.88 [ 2:49:14<19:26:34] +[titan] 2025-07-09 16:24:15,274 - root - INFO - step: 12675 loss: 17.1597 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6858 global_avg_mtp_loss: 14.4739 +[titan] 2025-07-09 16:24:15,275 - root - INFO - lr: 2.9084e-04 gnorm: 0.87 [ 2:49:18<19:26:29] +[titan] 2025-07-09 16:24:19,194 - root - INFO - step: 12680 loss: 17.2890 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.7092 global_avg_mtp_loss: 14.5799 +[titan] 2025-07-09 16:24:19,195 - root - INFO - lr: 2.9083e-04 gnorm: 0.87 [ 2:49:22<19:26:24] +[titan] 2025-07-09 16:24:23,119 - root - INFO - step: 12685 loss: 17.3231 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.7264 global_avg_mtp_loss: 14.5966 +[titan] 2025-07-09 16:24:23,119 - root - INFO - lr: 2.9082e-04 gnorm: 0.84 [ 2:49:26<19:26:20] +[titan] 2025-07-09 16:24:27,036 - root - INFO - step: 12690 loss: 17.0411 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6667 global_avg_mtp_loss: 14.3745 +[titan] 2025-07-09 16:24:27,036 - root - INFO - lr: 2.9082e-04 gnorm: 0.83 [ 2:49:30<19:26:15] +[titan] 2025-07-09 16:24:30,971 - root - INFO - step: 12695 loss: 17.3199 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.7223 global_avg_mtp_loss: 14.5976 +[titan] 2025-07-09 16:24:30,971 - root - INFO - lr: 2.9081e-04 gnorm: 0.87 [ 2:49:34<19:26:11] +[titan] 2025-07-09 16:24:34,090 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:24:34,878 - root - INFO - step: 12700 loss: 17.3030 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7209 global_avg_mtp_loss: 14.5820 +[titan] 2025-07-09 16:24:34,879 - root - INFO - lr: 2.9080e-04 gnorm: 0.88 [ 2:49:38<19:26:06] +[titan] 2025-07-09 16:24:38,808 - root - INFO - step: 12705 loss: 17.2518 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.7057 global_avg_mtp_loss: 14.5461 +[titan] 2025-07-09 16:24:38,808 - root - INFO - lr: 2.9079e-04 gnorm: 0.95 [ 2:49:42<19:26:01] +[titan] 2025-07-09 16:24:42,722 - root - INFO - step: 12710 loss: 17.3327 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7251 global_avg_mtp_loss: 14.6076 +[titan] 2025-07-09 16:24:42,722 - root - INFO - lr: 2.9078e-04 gnorm: 0.88 [ 2:49:46<19:25:57] +[titan] 2025-07-09 16:24:46,625 - root - INFO - step: 12715 loss: 17.3410 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.7128 global_avg_mtp_loss: 14.6282 +[titan] 2025-07-09 16:24:46,625 - root - INFO - lr: 2.9078e-04 gnorm: 0.97 [ 2:49:50<19:25:52] +[titan] 2025-07-09 16:24:50,526 - root - INFO - step: 12720 loss: 17.0268 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6610 global_avg_mtp_loss: 14.3658 +[titan] 2025-07-09 16:24:50,526 - root - INFO - lr: 2.9077e-04 gnorm: 0.89 [ 2:49:54<19:25:47] +[titan] 2025-07-09 16:24:54,422 - root - INFO - step: 12725 loss: 17.5429 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.7660 global_avg_mtp_loss: 14.7769 +[titan] 2025-07-09 16:24:54,422 - root - INFO - lr: 2.9076e-04 gnorm: 1.05 [ 2:49:57<19:25:42] +[titan] 2025-07-09 16:24:58,348 - root - INFO - step: 12730 loss: 17.1817 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6870 global_avg_mtp_loss: 14.4946 +[titan] 2025-07-09 16:24:58,348 - root - INFO - lr: 2.9075e-04 gnorm: 0.86 [ 2:50:01<19:25:38] +[titan] 2025-07-09 16:25:02,302 - root - INFO - step: 12735 loss: 17.3875 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.7444 global_avg_mtp_loss: 14.6431 +[titan] 2025-07-09 16:25:02,302 - root - INFO - lr: 2.9075e-04 gnorm: 0.91 [ 2:50:05<19:25:34] +[titan] 2025-07-09 16:25:06,200 - root - INFO - step: 12740 loss: 17.3260 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.7103 global_avg_mtp_loss: 14.6158 +[titan] 2025-07-09 16:25:06,200 - root - INFO - lr: 2.9074e-04 gnorm: 0.96 [ 2:50:09<19:25:29] +[titan] 2025-07-09 16:25:10,097 - root - INFO - step: 12745 loss: 17.2310 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.7114 global_avg_mtp_loss: 14.5196 +[titan] 2025-07-09 16:25:10,098 - root - INFO - lr: 2.9073e-04 gnorm: 0.91 [ 2:50:13<19:25:24] +[titan] 2025-07-09 16:25:13,216 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:25:14,002 - root - INFO - step: 12750 loss: 17.2202 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6974 global_avg_mtp_loss: 14.5229 +[titan] 2025-07-09 16:25:14,002 - root - INFO - lr: 2.9072e-04 gnorm: 0.81 [ 2:50:17<19:25:19] +[titan] 2025-07-09 16:25:17,902 - root - INFO - step: 12755 loss: 17.2398 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.7048 global_avg_mtp_loss: 14.5350 +[titan] 2025-07-09 16:25:17,902 - root - INFO - lr: 2.9071e-04 gnorm: 0.88 [ 2:50:21<19:25:15] +[titan] 2025-07-09 16:25:21,850 - root - INFO - step: 12760 loss: 17.3547 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.7209 global_avg_mtp_loss: 14.6338 +[titan] 2025-07-09 16:25:21,851 - root - INFO - lr: 2.9071e-04 gnorm: 0.87 [ 2:50:25<19:25:10] +[titan] 2025-07-09 16:25:25,825 - root - INFO - step: 12765 loss: 17.2810 memory: 44.58GiB(31.99%) tps: 82,447 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.7182 global_avg_mtp_loss: 14.5628 +[titan] 2025-07-09 16:25:25,825 - root - INFO - lr: 2.9070e-04 gnorm: 0.85 [ 2:50:29<19:25:06] +[titan] 2025-07-09 16:25:29,725 - root - INFO - step: 12770 loss: 17.0433 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6646 global_avg_mtp_loss: 14.3787 +[titan] 2025-07-09 16:25:29,725 - root - INFO - lr: 2.9069e-04 gnorm: 0.94 [ 2:50:33<19:25:01] +[titan] 2025-07-09 16:25:33,611 - root - INFO - step: 12775 loss: 17.1214 memory: 44.58GiB(31.99%) tps: 84,322 tflops: 291.01 mfu: 29.42% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.4391 +[titan] 2025-07-09 16:25:33,611 - root - INFO - lr: 2.9068e-04 gnorm: 0.89 [ 2:50:37<19:24:56] +[titan] 2025-07-09 16:25:37,525 - root - INFO - step: 12780 loss: 17.2524 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6977 global_avg_mtp_loss: 14.5547 +[titan] 2025-07-09 16:25:37,526 - root - INFO - lr: 2.9068e-04 gnorm: 0.90 [ 2:50:41<19:24:52] +[titan] 2025-07-09 16:25:41,415 - root - INFO - step: 12785 loss: 17.2272 memory: 44.58GiB(31.99%) tps: 84,259 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.6924 global_avg_mtp_loss: 14.5348 +[titan] 2025-07-09 16:25:41,415 - root - INFO - lr: 2.9067e-04 gnorm: 0.89 [ 2:50:44<19:24:47] +[titan] 2025-07-09 16:25:45,345 - root - INFO - step: 12790 loss: 17.1995 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6918 global_avg_mtp_loss: 14.5078 +[titan] 2025-07-09 16:25:45,345 - root - INFO - lr: 2.9066e-04 gnorm: 0.90 [ 2:50:48<19:24:42] +[titan] 2025-07-09 16:25:49,267 - root - INFO - step: 12795 loss: 16.9420 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6528 global_avg_mtp_loss: 14.2892 +[titan] 2025-07-09 16:25:49,268 - root - INFO - lr: 2.9065e-04 gnorm: 0.96 [ 2:50:52<19:24:38] +[titan] 2025-07-09 16:25:52,436 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:25:53,237 - root - INFO - step: 12800 loss: 16.9629 memory: 44.58GiB(31.99%) tps: 82,548 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.6736 global_avg_mtp_loss: 14.2893 +[titan] 2025-07-09 16:25:53,238 - root - INFO - lr: 2.9064e-04 gnorm: 0.81 [ 2:50:56<19:24:34] +[titan] 2025-07-09 16:25:53,389 - root - INFO - Dumping profiler traces at step 12800 +[titan] 2025-07-09 16:25:53,422 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:25:57,341 - root - INFO - step: 12805 loss: 17.2073 memory: 44.58GiB(31.99%) tps: 79,867 tflops: 275.64 mfu: 27.87% global_avg_ntp_loss: 2.6930 global_avg_mtp_loss: 14.5143 +[titan] 2025-07-09 16:25:57,341 - root - INFO - lr: 2.9064e-04 gnorm: 0.94 [ 2:51:00<19:24:30] +[titan] 2025-07-09 16:26:01,268 - root - INFO - step: 12810 loss: 17.3127 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.7231 global_avg_mtp_loss: 14.5896 +[titan] 2025-07-09 16:26:01,268 - root - INFO - lr: 2.9063e-04 gnorm: 0.89 [ 2:51:04<19:24:26] +[titan] 2025-07-09 16:26:05,173 - root - INFO - step: 12815 loss: 17.4332 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.7352 global_avg_mtp_loss: 14.6980 +[titan] 2025-07-09 16:26:05,174 - root - INFO - lr: 2.9062e-04 gnorm: 0.95 [ 2:51:08<19:24:21] +[titan] 2025-07-09 16:26:09,068 - root - INFO - step: 12820 loss: 17.5251 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.7569 global_avg_mtp_loss: 14.7682 +[titan] 2025-07-09 16:26:09,069 - root - INFO - lr: 2.9061e-04 gnorm: 0.96 [ 2:51:12<19:24:16] +[titan] 2025-07-09 16:26:13,014 - root - INFO - step: 12825 loss: 17.2935 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.7118 global_avg_mtp_loss: 14.5817 +[titan] 2025-07-09 16:26:13,014 - root - INFO - lr: 2.9060e-04 gnorm: 0.96 [ 2:51:16<19:24:12] +[titan] 2025-07-09 16:26:16,933 - root - INFO - step: 12830 loss: 17.3003 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7097 global_avg_mtp_loss: 14.5906 +[titan] 2025-07-09 16:26:16,933 - root - INFO - lr: 2.9060e-04 gnorm: 0.86 [ 2:51:20<19:24:07] +[titan] 2025-07-09 16:26:20,872 - root - INFO - step: 12835 loss: 17.2879 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.7059 global_avg_mtp_loss: 14.5820 +[titan] 2025-07-09 16:26:20,872 - root - INFO - lr: 2.9059e-04 gnorm: 0.86 [ 2:51:24<19:24:03] +[titan] 2025-07-09 16:26:24,822 - root - INFO - step: 12840 loss: 17.3452 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.7235 global_avg_mtp_loss: 14.6217 +[titan] 2025-07-09 16:26:24,822 - root - INFO - lr: 2.9058e-04 gnorm: 0.90 [ 2:51:28<19:23:58] +[titan] 2025-07-09 16:26:28,735 - root - INFO - step: 12845 loss: 17.1226 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6844 global_avg_mtp_loss: 14.4382 +[titan] 2025-07-09 16:26:28,736 - root - INFO - lr: 2.9057e-04 gnorm: 0.90 [ 2:51:32<19:23:54] +[titan] 2025-07-09 16:26:31,876 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:26:32,675 - root - INFO - step: 12850 loss: 17.3611 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.7403 global_avg_mtp_loss: 14.6209 +[titan] 2025-07-09 16:26:32,675 - root - INFO - lr: 2.9057e-04 gnorm: 0.87 [ 2:51:36<19:23:49] +[titan] 2025-07-09 16:26:36,568 - root - INFO - step: 12855 loss: 17.2852 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.7058 global_avg_mtp_loss: 14.5793 +[titan] 2025-07-09 16:26:36,569 - root - INFO - lr: 2.9056e-04 gnorm: 0.85 [ 2:51:40<19:23:44] +[titan] 2025-07-09 16:26:40,479 - root - INFO - step: 12860 loss: 17.1364 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6971 global_avg_mtp_loss: 14.4394 +[titan] 2025-07-09 16:26:40,479 - root - INFO - lr: 2.9055e-04 gnorm: 0.87 [ 2:51:43<19:23:40] +[titan] 2025-07-09 16:26:44,405 - root - INFO - step: 12865 loss: 17.1919 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6928 global_avg_mtp_loss: 14.4992 +[titan] 2025-07-09 16:26:44,405 - root - INFO - lr: 2.9054e-04 gnorm: 0.99 [ 2:51:47<19:23:35] +[titan] 2025-07-09 16:26:48,320 - root - INFO - step: 12870 loss: 16.8038 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6277 global_avg_mtp_loss: 14.1762 +[titan] 2025-07-09 16:26:48,320 - root - INFO - lr: 2.9053e-04 gnorm: 0.88 [ 2:51:51<19:23:31] +[titan] 2025-07-09 16:26:52,237 - root - INFO - step: 12875 loss: 17.1641 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6786 global_avg_mtp_loss: 14.4856 +[titan] 2025-07-09 16:26:52,238 - root - INFO - lr: 2.9053e-04 gnorm: 0.91 [ 2:51:55<19:23:26] +[titan] 2025-07-09 16:26:56,144 - root - INFO - step: 12880 loss: 17.1571 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6875 global_avg_mtp_loss: 14.4696 +[titan] 2025-07-09 16:26:56,144 - root - INFO - lr: 2.9052e-04 gnorm: 0.82 [ 2:51:59<19:23:21] +[titan] 2025-07-09 16:27:00,035 - root - INFO - step: 12885 loss: 17.0551 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6686 global_avg_mtp_loss: 14.3865 +[titan] 2025-07-09 16:27:00,035 - root - INFO - lr: 2.9051e-04 gnorm: 0.92 [ 2:52:03<19:23:16] +[titan] 2025-07-09 16:27:03,939 - root - INFO - step: 12890 loss: 16.9692 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3053 +[titan] 2025-07-09 16:27:03,940 - root - INFO - lr: 2.9050e-04 gnorm: 0.90 [ 2:52:07<19:23:12] +[titan] 2025-07-09 16:27:07,852 - root - INFO - step: 12895 loss: 17.0986 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6823 global_avg_mtp_loss: 14.4164 +[titan] 2025-07-09 16:27:07,852 - root - INFO - lr: 2.9049e-04 gnorm: 0.82 [ 2:52:11<19:23:07] +[titan] 2025-07-09 16:27:11,021 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:27:11,816 - root - INFO - step: 12900 loss: 17.0985 memory: 44.58GiB(31.99%) tps: 82,662 tflops: 285.28 mfu: 28.85% global_avg_ntp_loss: 2.6802 global_avg_mtp_loss: 14.4183 +[titan] 2025-07-09 16:27:11,816 - root - INFO - lr: 2.9049e-04 gnorm: 0.94 [ 2:52:15<19:23:03] +[titan] 2025-07-09 16:27:15,723 - root - INFO - step: 12905 loss: 17.2522 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.7038 global_avg_mtp_loss: 14.5484 +[titan] 2025-07-09 16:27:15,723 - root - INFO - lr: 2.9048e-04 gnorm: 0.94 [ 2:52:19<19:22:58] +[titan] 2025-07-09 16:27:19,618 - root - INFO - step: 12910 loss: 17.0406 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6695 global_avg_mtp_loss: 14.3711 +[titan] 2025-07-09 16:27:19,618 - root - INFO - lr: 2.9047e-04 gnorm: 0.82 [ 2:52:23<19:22:53] +[titan] 2025-07-09 16:27:23,509 - root - INFO - step: 12915 loss: 17.5263 memory: 44.58GiB(31.99%) tps: 84,212 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.7635 global_avg_mtp_loss: 14.7628 +[titan] 2025-07-09 16:27:23,509 - root - INFO - lr: 2.9046e-04 gnorm: 0.82 [ 2:52:26<19:22:49] +[titan] 2025-07-09 16:27:27,419 - root - INFO - step: 12920 loss: 16.9001 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6513 global_avg_mtp_loss: 14.2488 +[titan] 2025-07-09 16:27:27,420 - root - INFO - lr: 2.9046e-04 gnorm: 0.90 [ 2:52:30<19:22:44] +[titan] 2025-07-09 16:27:31,310 - root - INFO - step: 12925 loss: 17.0326 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.3685 +[titan] 2025-07-09 16:27:31,310 - root - INFO - lr: 2.9045e-04 gnorm: 0.87 [ 2:52:34<19:22:39] +[titan] 2025-07-09 16:27:35,198 - root - INFO - step: 12930 loss: 17.2575 memory: 44.58GiB(31.99%) tps: 84,297 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.7019 global_avg_mtp_loss: 14.5556 +[titan] 2025-07-09 16:27:35,198 - root - INFO - lr: 2.9044e-04 gnorm: 0.91 [ 2:52:38<19:22:34] +[titan] 2025-07-09 16:27:39,099 - root - INFO - step: 12935 loss: 16.9731 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6545 global_avg_mtp_loss: 14.3185 +[titan] 2025-07-09 16:27:39,099 - root - INFO - lr: 2.9043e-04 gnorm: 0.86 [ 2:52:42<19:22:30] +[titan] 2025-07-09 16:27:42,983 - root - INFO - step: 12940 loss: 17.4468 memory: 44.58GiB(31.99%) tps: 84,372 tflops: 291.18 mfu: 29.44% global_avg_ntp_loss: 2.7357 global_avg_mtp_loss: 14.7110 +[titan] 2025-07-09 16:27:42,983 - root - INFO - lr: 2.9042e-04 gnorm: 0.80 [ 2:52:46<19:22:25] +[titan] 2025-07-09 16:27:46,884 - root - INFO - step: 12945 loss: 17.1031 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6826 global_avg_mtp_loss: 14.4205 +[titan] 2025-07-09 16:27:46,884 - root - INFO - lr: 2.9042e-04 gnorm: 0.88 [ 2:52:50<19:22:20] +[titan] 2025-07-09 16:27:49,999 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:27:50,787 - root - INFO - step: 12950 loss: 17.2119 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.7060 global_avg_mtp_loss: 14.5059 +[titan] 2025-07-09 16:27:50,787 - root - INFO - lr: 2.9041e-04 gnorm: 0.81 [ 2:52:54<19:22:15] +[titan] 2025-07-09 16:27:54,728 - root - INFO - step: 12955 loss: 17.0136 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6481 global_avg_mtp_loss: 14.3655 +[titan] 2025-07-09 16:27:54,728 - root - INFO - lr: 2.9040e-04 gnorm: 0.88 [ 2:52:58<19:22:11] +[titan] 2025-07-09 16:27:58,647 - root - INFO - step: 12960 loss: 17.3108 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5986 +[titan] 2025-07-09 16:27:58,647 - root - INFO - lr: 2.9039e-04 gnorm: 0.81 [ 2:53:02<19:22:06] +[titan] 2025-07-09 16:28:02,549 - root - INFO - step: 12965 loss: 17.1591 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6977 global_avg_mtp_loss: 14.4615 +[titan] 2025-07-09 16:28:02,549 - root - INFO - lr: 2.9038e-04 gnorm: 0.96 [ 2:53:06<19:22:02] +[titan] 2025-07-09 16:28:06,485 - root - INFO - step: 12970 loss: 17.1170 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.6856 global_avg_mtp_loss: 14.4314 +[titan] 2025-07-09 16:28:06,485 - root - INFO - lr: 2.9038e-04 gnorm: 0.86 [ 2:53:09<19:21:57] +[titan] 2025-07-09 16:28:10,408 - root - INFO - step: 12975 loss: 17.2122 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6994 global_avg_mtp_loss: 14.5128 +[titan] 2025-07-09 16:28:10,408 - root - INFO - lr: 2.9037e-04 gnorm: 0.86 [ 2:53:13<19:21:53] +[titan] 2025-07-09 16:28:14,335 - root - INFO - step: 12980 loss: 17.1067 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6711 global_avg_mtp_loss: 14.4356 +[titan] 2025-07-09 16:28:14,336 - root - INFO - lr: 2.9036e-04 gnorm: 0.92 [ 2:53:17<19:21:48] +[titan] 2025-07-09 16:28:18,273 - root - INFO - step: 12985 loss: 17.2198 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6980 global_avg_mtp_loss: 14.5218 +[titan] 2025-07-09 16:28:18,273 - root - INFO - lr: 2.9035e-04 gnorm: 0.83 [ 2:53:21<19:21:44] +[titan] 2025-07-09 16:28:22,180 - root - INFO - step: 12990 loss: 16.9961 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6529 global_avg_mtp_loss: 14.3432 +[titan] 2025-07-09 16:28:22,180 - root - INFO - lr: 2.9034e-04 gnorm: 0.91 [ 2:53:25<19:21:39] +[titan] 2025-07-09 16:28:26,116 - root - INFO - step: 12995 loss: 17.1019 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.6701 global_avg_mtp_loss: 14.4318 +[titan] 2025-07-09 16:28:26,116 - root - INFO - lr: 2.9034e-04 gnorm: 0.82 [ 2:53:29<19:21:35] +[titan] 2025-07-09 16:28:29,238 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:28:30,026 - root - INFO - step: 13000 loss: 17.1483 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6765 global_avg_mtp_loss: 14.4719 +[titan] 2025-07-09 16:28:30,026 - root - INFO - lr: 2.9033e-04 gnorm: 0.88 [ 2:53:33<19:21:30] +[titan] 2025-07-09 16:28:33,913 - root - INFO - step: 13005 loss: 17.3267 memory: 44.58GiB(31.99%) tps: 84,291 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.7193 global_avg_mtp_loss: 14.6074 +[titan] 2025-07-09 16:28:33,914 - root - INFO - lr: 2.9032e-04 gnorm: 0.87 [ 2:53:37<19:21:25] +[titan] 2025-07-09 16:28:37,797 - root - INFO - step: 13010 loss: 16.9727 memory: 44.58GiB(31.99%) tps: 84,386 tflops: 291.23 mfu: 29.45% global_avg_ntp_loss: 2.6635 global_avg_mtp_loss: 14.3091 +[titan] 2025-07-09 16:28:37,797 - root - INFO - lr: 2.9031e-04 gnorm: 0.86 [ 2:53:41<19:21:20] +[titan] 2025-07-09 16:28:41,687 - root - INFO - step: 13015 loss: 17.1250 memory: 44.58GiB(31.99%) tps: 84,250 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.6746 global_avg_mtp_loss: 14.4503 +[titan] 2025-07-09 16:28:41,687 - root - INFO - lr: 2.9030e-04 gnorm: 0.87 [ 2:53:45<19:21:15] +[titan] 2025-07-09 16:28:45,600 - root - INFO - step: 13020 loss: 17.2054 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.7026 global_avg_mtp_loss: 14.5028 +[titan] 2025-07-09 16:28:45,600 - root - INFO - lr: 2.9030e-04 gnorm: 0.99 [ 2:53:49<19:21:11] +[titan] 2025-07-09 16:28:49,529 - root - INFO - step: 13025 loss: 17.0910 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6810 global_avg_mtp_loss: 14.4099 +[titan] 2025-07-09 16:28:49,529 - root - INFO - lr: 2.9029e-04 gnorm: 0.89 [ 2:53:53<19:21:06] +[titan] 2025-07-09 16:28:53,419 - root - INFO - step: 13030 loss: 17.3112 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.7295 global_avg_mtp_loss: 14.5817 +[titan] 2025-07-09 16:28:53,419 - root - INFO - lr: 2.9028e-04 gnorm: 0.82 [ 2:53:56<19:21:02] +[titan] 2025-07-09 16:28:57,330 - root - INFO - step: 13035 loss: 17.2841 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.7105 global_avg_mtp_loss: 14.5736 +[titan] 2025-07-09 16:28:57,331 - root - INFO - lr: 2.9027e-04 gnorm: 0.83 [ 2:54:00<19:20:57] +[titan] 2025-07-09 16:29:01,231 - root - INFO - step: 13040 loss: 17.2463 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.6896 global_avg_mtp_loss: 14.5566 +[titan] 2025-07-09 16:29:01,231 - root - INFO - lr: 2.9026e-04 gnorm: 0.89 [ 2:54:04<19:20:52] +[titan] 2025-07-09 16:29:05,145 - root - INFO - step: 13045 loss: 17.1967 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.7014 global_avg_mtp_loss: 14.4954 +[titan] 2025-07-09 16:29:05,145 - root - INFO - lr: 2.9026e-04 gnorm: 0.82 [ 2:54:08<19:20:48] +[titan] 2025-07-09 16:29:08,271 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:29:09,055 - root - INFO - step: 13050 loss: 17.0339 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6652 global_avg_mtp_loss: 14.3687 +[titan] 2025-07-09 16:29:09,055 - root - INFO - lr: 2.9025e-04 gnorm: 0.91 [ 2:54:12<19:20:43] +[titan] 2025-07-09 16:29:12,958 - root - INFO - step: 13055 loss: 17.1353 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6825 global_avg_mtp_loss: 14.4528 +[titan] 2025-07-09 16:29:12,958 - root - INFO - lr: 2.9024e-04 gnorm: 0.79 [ 2:54:16<19:20:38] +[titan] 2025-07-09 16:29:16,856 - root - INFO - step: 13060 loss: 17.4339 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.7313 global_avg_mtp_loss: 14.7026 +[titan] 2025-07-09 16:29:16,856 - root - INFO - lr: 2.9023e-04 gnorm: 0.99 [ 2:54:20<19:20:34] +[titan] 2025-07-09 16:29:20,757 - root - INFO - step: 13065 loss: 17.2803 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6928 global_avg_mtp_loss: 14.5875 +[titan] 2025-07-09 16:29:20,757 - root - INFO - lr: 2.9022e-04 gnorm: 0.84 [ 2:54:24<19:20:29] +[titan] 2025-07-09 16:29:24,658 - root - INFO - step: 13070 loss: 17.1848 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6748 global_avg_mtp_loss: 14.5100 +[titan] 2025-07-09 16:29:24,658 - root - INFO - lr: 2.9022e-04 gnorm: 1.00 [ 2:54:28<19:20:24] +[titan] 2025-07-09 16:29:28,555 - root - INFO - step: 13075 loss: 17.3506 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7284 global_avg_mtp_loss: 14.6222 +[titan] 2025-07-09 16:29:28,555 - root - INFO - lr: 2.9021e-04 gnorm: 0.89 [ 2:54:32<19:20:19] +[titan] 2025-07-09 16:29:32,453 - root - INFO - step: 13080 loss: 17.1531 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.6859 global_avg_mtp_loss: 14.4672 +[titan] 2025-07-09 16:29:32,453 - root - INFO - lr: 2.9020e-04 gnorm: 0.91 [ 2:54:35<19:20:15] +[titan] 2025-07-09 16:29:36,349 - root - INFO - step: 13085 loss: 17.0941 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6706 global_avg_mtp_loss: 14.4234 +[titan] 2025-07-09 16:29:36,349 - root - INFO - lr: 2.9019e-04 gnorm: 0.94 [ 2:54:39<19:20:10] +[titan] 2025-07-09 16:29:40,268 - root - INFO - step: 13090 loss: 17.4091 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.7399 global_avg_mtp_loss: 14.6692 +[titan] 2025-07-09 16:29:40,268 - root - INFO - lr: 2.9018e-04 gnorm: 0.98 [ 2:54:43<19:20:05] +[titan] 2025-07-09 16:29:44,206 - root - INFO - step: 13095 loss: 17.1130 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6909 global_avg_mtp_loss: 14.4221 +[titan] 2025-07-09 16:29:44,206 - root - INFO - lr: 2.9018e-04 gnorm: 0.86 [ 2:54:47<19:20:01] +[titan] 2025-07-09 16:29:47,341 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:29:48,151 - root - INFO - step: 13100 loss: 17.1230 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.6813 global_avg_mtp_loss: 14.4417 +[titan] 2025-07-09 16:29:48,152 - root - INFO - lr: 2.9017e-04 gnorm: 0.93 [ 2:54:51<19:19:57] +[titan] 2025-07-09 16:29:52,072 - root - INFO - step: 13105 loss: 17.1747 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6882 global_avg_mtp_loss: 14.4865 +[titan] 2025-07-09 16:29:52,072 - root - INFO - lr: 2.9016e-04 gnorm: 0.83 [ 2:54:55<19:19:52] +[titan] 2025-07-09 16:29:56,006 - root - INFO - step: 13110 loss: 17.2693 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.7129 global_avg_mtp_loss: 14.5564 +[titan] 2025-07-09 16:29:56,006 - root - INFO - lr: 2.9015e-04 gnorm: 0.81 [ 2:54:59<19:19:48] +[titan] 2025-07-09 16:29:59,963 - root - INFO - step: 13115 loss: 17.3312 memory: 44.58GiB(31.99%) tps: 82,827 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.7236 global_avg_mtp_loss: 14.6076 +[titan] 2025-07-09 16:29:59,963 - root - INFO - lr: 2.9014e-04 gnorm: 0.82 [ 2:55:03<19:19:43] +[titan] 2025-07-09 16:30:03,860 - root - INFO - step: 13120 loss: 17.0727 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6665 global_avg_mtp_loss: 14.4061 +[titan] 2025-07-09 16:30:03,860 - root - INFO - lr: 2.9014e-04 gnorm: 0.93 [ 2:55:07<19:19:39] +[titan] 2025-07-09 16:30:07,811 - root - INFO - step: 13125 loss: 17.1985 memory: 44.58GiB(31.99%) tps: 82,937 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.5139 +[titan] 2025-07-09 16:30:07,812 - root - INFO - lr: 2.9013e-04 gnorm: 0.90 [ 2:55:11<19:19:34] +[titan] 2025-07-09 16:30:11,777 - root - INFO - step: 13130 loss: 17.0444 memory: 44.58GiB(31.99%) tps: 82,646 tflops: 285.23 mfu: 28.84% global_avg_ntp_loss: 2.6663 global_avg_mtp_loss: 14.3781 +[titan] 2025-07-09 16:30:11,777 - root - INFO - lr: 2.9012e-04 gnorm: 0.83 [ 2:55:15<19:19:30] +[titan] 2025-07-09 16:30:15,716 - root - INFO - step: 13135 loss: 17.0087 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.3573 +[titan] 2025-07-09 16:30:15,716 - root - INFO - lr: 2.9011e-04 gnorm: 0.82 [ 2:55:19<19:19:25] +[titan] 2025-07-09 16:30:19,641 - root - INFO - step: 13140 loss: 17.3203 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.7251 global_avg_mtp_loss: 14.5952 +[titan] 2025-07-09 16:30:19,642 - root - INFO - lr: 2.9010e-04 gnorm: 0.91 [ 2:55:23<19:19:21] +[titan] 2025-07-09 16:30:23,557 - root - INFO - step: 13145 loss: 17.2764 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.7044 global_avg_mtp_loss: 14.5720 +[titan] 2025-07-09 16:30:23,558 - root - INFO - lr: 2.9010e-04 gnorm: 0.88 [ 2:55:27<19:19:16] +[titan] 2025-07-09 16:30:26,702 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:30:27,494 - root - INFO - step: 13150 loss: 17.3225 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.7349 global_avg_mtp_loss: 14.5876 +[titan] 2025-07-09 16:30:27,494 - root - INFO - lr: 2.9009e-04 gnorm: 0.81 [ 2:55:30<19:19:12] +[titan] 2025-07-09 16:30:31,432 - root - INFO - step: 13155 loss: 17.1533 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.6843 global_avg_mtp_loss: 14.4690 +[titan] 2025-07-09 16:30:31,432 - root - INFO - lr: 2.9008e-04 gnorm: 1.06 [ 2:55:34<19:19:07] +[titan] 2025-07-09 16:30:35,362 - root - INFO - step: 13160 loss: 17.0342 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6639 global_avg_mtp_loss: 14.3703 +[titan] 2025-07-09 16:30:35,363 - root - INFO - lr: 2.9007e-04 gnorm: 0.96 [ 2:55:38<19:19:03] +[titan] 2025-07-09 16:30:39,303 - root - INFO - step: 13165 loss: 16.9556 memory: 44.58GiB(31.99%) tps: 83,171 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.6512 global_avg_mtp_loss: 14.3044 +[titan] 2025-07-09 16:30:39,303 - root - INFO - lr: 2.9006e-04 gnorm: 0.86 [ 2:55:42<19:18:59] +[titan] 2025-07-09 16:30:43,211 - root - INFO - step: 13170 loss: 17.1183 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6776 global_avg_mtp_loss: 14.4407 +[titan] 2025-07-09 16:30:43,211 - root - INFO - lr: 2.9006e-04 gnorm: 0.82 [ 2:55:46<19:18:54] +[titan] 2025-07-09 16:30:47,127 - root - INFO - step: 13175 loss: 17.2691 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.7071 global_avg_mtp_loss: 14.5620 +[titan] 2025-07-09 16:30:47,127 - root - INFO - lr: 2.9005e-04 gnorm: 0.89 [ 2:55:50<19:18:49] +[titan] 2025-07-09 16:30:51,056 - root - INFO - step: 13180 loss: 17.0775 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6767 global_avg_mtp_loss: 14.4008 +[titan] 2025-07-09 16:30:51,056 - root - INFO - lr: 2.9004e-04 gnorm: 0.94 [ 2:55:54<19:18:45] +[titan] 2025-07-09 16:30:54,979 - root - INFO - step: 13185 loss: 17.2703 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.7014 global_avg_mtp_loss: 14.5689 +[titan] 2025-07-09 16:30:54,979 - root - INFO - lr: 2.9003e-04 gnorm: 0.86 [ 2:55:58<19:18:40] +[titan] 2025-07-09 16:30:58,876 - root - INFO - step: 13190 loss: 17.3309 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7281 global_avg_mtp_loss: 14.6028 +[titan] 2025-07-09 16:30:58,876 - root - INFO - lr: 2.9002e-04 gnorm: 0.86 [ 2:56:02<19:18:36] +[titan] 2025-07-09 16:31:02,770 - root - INFO - step: 13195 loss: 17.2618 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.6952 global_avg_mtp_loss: 14.5666 +[titan] 2025-07-09 16:31:02,770 - root - INFO - lr: 2.9002e-04 gnorm: 0.87 [ 2:56:06<19:18:31] +[titan] 2025-07-09 16:31:05,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:31:06,666 - root - INFO - step: 13200 loss: 17.2375 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.7197 global_avg_mtp_loss: 14.5178 +[titan] 2025-07-09 16:31:06,666 - root - INFO - lr: 2.9001e-04 gnorm: 0.96 [ 2:56:10<19:18:26] +[titan] 2025-07-09 16:31:10,598 - root - INFO - step: 13205 loss: 17.1385 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6794 global_avg_mtp_loss: 14.4591 +[titan] 2025-07-09 16:31:10,599 - root - INFO - lr: 2.9000e-04 gnorm: 0.85 [ 2:56:14<19:18:22] +[titan] 2025-07-09 16:31:14,502 - root - INFO - step: 13210 loss: 17.1005 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6704 global_avg_mtp_loss: 14.4301 +[titan] 2025-07-09 16:31:14,502 - root - INFO - lr: 2.8999e-04 gnorm: 0.87 [ 2:56:17<19:18:17] +[titan] 2025-07-09 16:31:18,427 - root - INFO - step: 13215 loss: 17.2842 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.7072 global_avg_mtp_loss: 14.5770 +[titan] 2025-07-09 16:31:18,427 - root - INFO - lr: 2.8998e-04 gnorm: 0.87 [ 2:56:21<19:18:12] +[titan] 2025-07-09 16:31:22,337 - root - INFO - step: 13220 loss: 17.0688 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.4155 +[titan] 2025-07-09 16:31:22,337 - root - INFO - lr: 2.8997e-04 gnorm: 0.86 [ 2:56:25<19:18:08] +[titan] 2025-07-09 16:31:26,239 - root - INFO - step: 13225 loss: 17.2710 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.7196 global_avg_mtp_loss: 14.5514 +[titan] 2025-07-09 16:31:26,239 - root - INFO - lr: 2.8997e-04 gnorm: 1.00 [ 2:56:29<19:18:03] +[titan] 2025-07-09 16:31:30,180 - root - INFO - step: 13230 loss: 17.2441 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.7059 global_avg_mtp_loss: 14.5382 +[titan] 2025-07-09 16:31:30,180 - root - INFO - lr: 2.8996e-04 gnorm: 0.94 [ 2:56:33<19:17:59] +[titan] 2025-07-09 16:31:34,086 - root - INFO - step: 13235 loss: 17.2326 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.7057 global_avg_mtp_loss: 14.5269 +[titan] 2025-07-09 16:31:34,086 - root - INFO - lr: 2.8995e-04 gnorm: 0.95 [ 2:56:37<19:17:54] +[titan] 2025-07-09 16:31:38,018 - root - INFO - step: 13240 loss: 17.2712 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.7023 global_avg_mtp_loss: 14.5689 +[titan] 2025-07-09 16:31:38,018 - root - INFO - lr: 2.8994e-04 gnorm: 1.00 [ 2:56:41<19:17:50] +[titan] 2025-07-09 16:31:41,938 - root - INFO - step: 13245 loss: 17.2172 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.7090 global_avg_mtp_loss: 14.5082 +[titan] 2025-07-09 16:31:41,938 - root - INFO - lr: 2.8993e-04 gnorm: 0.90 [ 2:56:45<19:17:45] +[titan] 2025-07-09 16:31:45,077 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:31:45,870 - root - INFO - step: 13250 loss: 16.9913 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.6695 global_avg_mtp_loss: 14.3219 +[titan] 2025-07-09 16:31:45,870 - root - INFO - lr: 2.8993e-04 gnorm: 1.00 [ 2:56:49<19:17:41] +[titan] 2025-07-09 16:31:49,775 - root - INFO - step: 13255 loss: 17.3723 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.7280 global_avg_mtp_loss: 14.6443 +[titan] 2025-07-09 16:31:49,775 - root - INFO - lr: 2.8992e-04 gnorm: 0.83 [ 2:56:53<19:17:36] +[titan] 2025-07-09 16:31:53,671 - root - INFO - step: 13260 loss: 17.3221 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.7061 global_avg_mtp_loss: 14.6160 +[titan] 2025-07-09 16:31:53,671 - root - INFO - lr: 2.8991e-04 gnorm: 0.88 [ 2:56:57<19:17:31] +[titan] 2025-07-09 16:31:57,607 - root - INFO - step: 13265 loss: 17.1582 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.4737 +[titan] 2025-07-09 16:31:57,607 - root - INFO - lr: 2.8990e-04 gnorm: 0.83 [ 2:57:01<19:17:27] +[titan] 2025-07-09 16:32:01,562 - root - INFO - step: 13270 loss: 17.1604 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.6914 global_avg_mtp_loss: 14.4690 +[titan] 2025-07-09 16:32:01,562 - root - INFO - lr: 2.8989e-04 gnorm: 0.83 [ 2:57:05<19:17:22] +[titan] 2025-07-09 16:32:05,472 - root - INFO - step: 13275 loss: 17.1713 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6997 global_avg_mtp_loss: 14.4716 +[titan] 2025-07-09 16:32:05,472 - root - INFO - lr: 2.8989e-04 gnorm: 0.82 [ 2:57:08<19:17:18] +[titan] 2025-07-09 16:32:09,377 - root - INFO - step: 13280 loss: 17.1806 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6926 global_avg_mtp_loss: 14.4880 +[titan] 2025-07-09 16:32:09,377 - root - INFO - lr: 2.8988e-04 gnorm: 0.87 [ 2:57:12<19:17:13] +[titan] 2025-07-09 16:32:13,313 - root - INFO - step: 13285 loss: 17.2498 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.6934 global_avg_mtp_loss: 14.5563 +[titan] 2025-07-09 16:32:13,313 - root - INFO - lr: 2.8987e-04 gnorm: 0.83 [ 2:57:16<19:17:09] +[titan] 2025-07-09 16:32:17,300 - root - INFO - step: 13290 loss: 17.2725 memory: 44.58GiB(31.99%) tps: 82,182 tflops: 283.62 mfu: 28.68% global_avg_ntp_loss: 2.7153 global_avg_mtp_loss: 14.5571 +[titan] 2025-07-09 16:32:17,301 - root - INFO - lr: 2.8986e-04 gnorm: 0.82 [ 2:57:20<19:17:05] +[titan] 2025-07-09 16:32:21,202 - root - INFO - step: 13295 loss: 17.1949 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7059 global_avg_mtp_loss: 14.4889 +[titan] 2025-07-09 16:32:21,203 - root - INFO - lr: 2.8985e-04 gnorm: 0.90 [ 2:57:24<19:17:00] +[titan] 2025-07-09 16:32:24,311 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:32:25,102 - root - INFO - step: 13300 loss: 17.1951 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6950 global_avg_mtp_loss: 14.5001 +[titan] 2025-07-09 16:32:25,102 - root - INFO - lr: 2.8984e-04 gnorm: 0.86 [ 2:57:28<19:16:55] +[titan] 2025-07-09 16:32:28,999 - root - INFO - step: 13305 loss: 17.2902 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.7134 global_avg_mtp_loss: 14.5768 +[titan] 2025-07-09 16:32:28,999 - root - INFO - lr: 2.8984e-04 gnorm: 0.81 [ 2:57:32<19:16:51] +[titan] 2025-07-09 16:32:32,976 - root - INFO - step: 13310 loss: 17.3279 memory: 44.58GiB(31.99%) tps: 82,402 tflops: 284.38 mfu: 28.75% global_avg_ntp_loss: 2.7245 global_avg_mtp_loss: 14.6034 +[titan] 2025-07-09 16:32:32,976 - root - INFO - lr: 2.8983e-04 gnorm: 0.90 [ 2:57:36<19:16:46] +[titan] 2025-07-09 16:32:34,718 - root - INFO - Dumping profiler traces at step 13312 +[titan] 2025-07-09 16:32:34,749 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:32:37,099 - root - INFO - step: 13315 loss: 17.4013 memory: 44.58GiB(31.99%) tps: 79,481 tflops: 274.30 mfu: 27.74% global_avg_ntp_loss: 2.7576 global_avg_mtp_loss: 14.6437 +[titan] 2025-07-09 16:32:37,100 - root - INFO - lr: 2.8982e-04 gnorm: 0.88 [ 2:57:40<19:16:43] +[titan] 2025-07-09 16:32:40,985 - root - INFO - step: 13320 loss: 17.3549 memory: 44.58GiB(31.99%) tps: 84,346 tflops: 291.09 mfu: 29.43% global_avg_ntp_loss: 2.7247 global_avg_mtp_loss: 14.6302 +[titan] 2025-07-09 16:32:40,985 - root - INFO - lr: 2.8981e-04 gnorm: 0.81 [ 2:57:44<19:16:38] +[titan] 2025-07-09 16:32:44,907 - root - INFO - step: 13325 loss: 17.5480 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.7730 global_avg_mtp_loss: 14.7750 +[titan] 2025-07-09 16:32:44,907 - root - INFO - lr: 2.8980e-04 gnorm: 0.91 [ 2:57:48<19:16:34] +[titan] 2025-07-09 16:32:48,798 - root - INFO - step: 13330 loss: 17.1214 memory: 44.58GiB(31.99%) tps: 84,217 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6763 global_avg_mtp_loss: 14.4451 +[titan] 2025-07-09 16:32:48,798 - root - INFO - lr: 2.8980e-04 gnorm: 0.92 [ 2:57:52<19:16:29] +[titan] 2025-07-09 16:32:52,705 - root - INFO - step: 13335 loss: 17.4388 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.7386 global_avg_mtp_loss: 14.7002 +[titan] 2025-07-09 16:32:52,705 - root - INFO - lr: 2.8979e-04 gnorm: 0.82 [ 2:57:56<19:16:24] +[titan] 2025-07-09 16:32:56,640 - root - INFO - step: 13340 loss: 16.9708 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6584 global_avg_mtp_loss: 14.3124 +[titan] 2025-07-09 16:32:56,640 - root - INFO - lr: 2.8978e-04 gnorm: 0.89 [ 2:58:00<19:16:20] +[titan] 2025-07-09 16:33:00,543 - root - INFO - step: 13345 loss: 17.1468 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4574 +[titan] 2025-07-09 16:33:00,543 - root - INFO - lr: 2.8977e-04 gnorm: 0.93 [ 2:58:03<19:16:15] +[titan] 2025-07-09 16:33:03,663 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:33:04,452 - root - INFO - step: 13350 loss: 17.0292 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.3669 +[titan] 2025-07-09 16:33:04,452 - root - INFO - lr: 2.8976e-04 gnorm: 0.95 [ 2:58:07<19:16:11] +[titan] 2025-07-09 16:33:08,365 - root - INFO - step: 13355 loss: 17.1559 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6809 global_avg_mtp_loss: 14.4751 +[titan] 2025-07-09 16:33:08,366 - root - INFO - lr: 2.8976e-04 gnorm: 0.88 [ 2:58:11<19:16:06] +[titan] 2025-07-09 16:33:12,247 - root - INFO - step: 13360 loss: 17.1148 memory: 44.58GiB(31.99%) tps: 84,438 tflops: 291.41 mfu: 29.47% global_avg_ntp_loss: 2.6862 global_avg_mtp_loss: 14.4286 +[titan] 2025-07-09 16:33:12,247 - root - INFO - lr: 2.8975e-04 gnorm: 0.98 [ 2:58:15<19:16:01] +[titan] 2025-07-09 16:33:16,142 - root - INFO - step: 13365 loss: 17.3485 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.7330 global_avg_mtp_loss: 14.6155 +[titan] 2025-07-09 16:33:16,142 - root - INFO - lr: 2.8974e-04 gnorm: 0.95 [ 2:58:19<19:15:57] +[titan] 2025-07-09 16:33:20,041 - root - INFO - step: 13370 loss: 17.0294 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6710 global_avg_mtp_loss: 14.3583 +[titan] 2025-07-09 16:33:20,041 - root - INFO - lr: 2.8973e-04 gnorm: 0.89 [ 2:58:23<19:15:52] +[titan] 2025-07-09 16:33:23,954 - root - INFO - step: 13375 loss: 17.2284 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7034 global_avg_mtp_loss: 14.5251 +[titan] 2025-07-09 16:33:23,955 - root - INFO - lr: 2.8972e-04 gnorm: 0.87 [ 2:58:27<19:15:47] +[titan] 2025-07-09 16:33:27,874 - root - INFO - step: 13380 loss: 17.2258 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.5307 +[titan] 2025-07-09 16:33:27,874 - root - INFO - lr: 2.8971e-04 gnorm: 0.82 [ 2:58:31<19:15:43] +[titan] 2025-07-09 16:33:31,792 - root - INFO - step: 13385 loss: 17.1755 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6909 global_avg_mtp_loss: 14.4846 +[titan] 2025-07-09 16:33:31,792 - root - INFO - lr: 2.8971e-04 gnorm: 0.86 [ 2:58:35<19:15:38] +[titan] 2025-07-09 16:33:35,730 - root - INFO - step: 13390 loss: 17.0534 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.6751 global_avg_mtp_loss: 14.3783 +[titan] 2025-07-09 16:33:35,730 - root - INFO - lr: 2.8970e-04 gnorm: 0.84 [ 2:58:39<19:15:34] +[titan] 2025-07-09 16:33:39,651 - root - INFO - step: 13395 loss: 17.1110 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6775 global_avg_mtp_loss: 14.4335 +[titan] 2025-07-09 16:33:39,652 - root - INFO - lr: 2.8969e-04 gnorm: 0.84 [ 2:58:43<19:15:29] +[titan] 2025-07-09 16:33:42,797 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:33:43,603 - root - INFO - step: 13400 loss: 17.3382 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.7109 global_avg_mtp_loss: 14.6273 +[titan] 2025-07-09 16:33:43,603 - root - INFO - lr: 2.8968e-04 gnorm: 0.90 [ 2:58:47<19:15:25] +[titan] 2025-07-09 16:33:47,502 - root - INFO - step: 13405 loss: 17.2442 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.7050 global_avg_mtp_loss: 14.5392 +[titan] 2025-07-09 16:33:47,502 - root - INFO - lr: 2.8967e-04 gnorm: 0.91 [ 2:58:50<19:15:20] +[titan] 2025-07-09 16:33:51,402 - root - INFO - step: 13410 loss: 17.2910 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.7145 global_avg_mtp_loss: 14.5765 +[titan] 2025-07-09 16:33:51,402 - root - INFO - lr: 2.8966e-04 gnorm: 0.88 [ 2:58:54<19:15:16] +[titan] 2025-07-09 16:33:55,304 - root - INFO - step: 13415 loss: 17.1172 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6779 global_avg_mtp_loss: 14.4393 +[titan] 2025-07-09 16:33:55,304 - root - INFO - lr: 2.8966e-04 gnorm: 0.88 [ 2:58:58<19:15:11] +[titan] 2025-07-09 16:33:59,208 - root - INFO - step: 13420 loss: 17.2685 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.7087 global_avg_mtp_loss: 14.5597 +[titan] 2025-07-09 16:33:59,208 - root - INFO - lr: 2.8965e-04 gnorm: 0.91 [ 2:59:02<19:15:06] +[titan] 2025-07-09 16:34:03,101 - root - INFO - step: 13425 loss: 17.4634 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.7501 global_avg_mtp_loss: 14.7133 +[titan] 2025-07-09 16:34:03,101 - root - INFO - lr: 2.8964e-04 gnorm: 0.92 [ 2:59:06<19:15:02] +[titan] 2025-07-09 16:34:06,997 - root - INFO - step: 13430 loss: 17.0499 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.6741 global_avg_mtp_loss: 14.3757 +[titan] 2025-07-09 16:34:06,998 - root - INFO - lr: 2.8963e-04 gnorm: 0.85 [ 2:59:10<19:14:57] +[titan] 2025-07-09 16:34:10,919 - root - INFO - step: 13435 loss: 17.3652 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.7204 global_avg_mtp_loss: 14.6448 +[titan] 2025-07-09 16:34:10,919 - root - INFO - lr: 2.8962e-04 gnorm: 0.88 [ 2:59:14<19:14:52] +[titan] 2025-07-09 16:34:14,843 - root - INFO - step: 13440 loss: 17.1395 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6911 global_avg_mtp_loss: 14.4484 +[titan] 2025-07-09 16:34:14,843 - root - INFO - lr: 2.8962e-04 gnorm: 0.88 [ 2:59:18<19:14:48] +[titan] 2025-07-09 16:34:18,740 - root - INFO - step: 13445 loss: 17.3533 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7243 global_avg_mtp_loss: 14.6289 +[titan] 2025-07-09 16:34:18,740 - root - INFO - lr: 2.8961e-04 gnorm: 0.86 [ 2:59:22<19:14:43] +[titan] 2025-07-09 16:34:21,849 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:34:22,638 - root - INFO - step: 13450 loss: 16.9982 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6525 global_avg_mtp_loss: 14.3457 +[titan] 2025-07-09 16:34:22,638 - root - INFO - lr: 2.8960e-04 gnorm: 0.88 [ 2:59:26<19:14:39] +[titan] 2025-07-09 16:34:26,550 - root - INFO - step: 13455 loss: 17.2603 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.7024 global_avg_mtp_loss: 14.5578 +[titan] 2025-07-09 16:34:26,550 - root - INFO - lr: 2.8959e-04 gnorm: 0.86 [ 2:59:29<19:14:34] +[titan] 2025-07-09 16:34:30,463 - root - INFO - step: 13460 loss: 17.1614 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6817 global_avg_mtp_loss: 14.4797 +[titan] 2025-07-09 16:34:30,463 - root - INFO - lr: 2.8958e-04 gnorm: 0.90 [ 2:59:33<19:14:29] +[titan] 2025-07-09 16:34:34,354 - root - INFO - step: 13465 loss: 17.0749 memory: 44.58GiB(31.99%) tps: 84,233 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.6703 global_avg_mtp_loss: 14.4047 +[titan] 2025-07-09 16:34:34,354 - root - INFO - lr: 2.8957e-04 gnorm: 0.81 [ 2:59:37<19:14:25] +[titan] 2025-07-09 16:34:38,301 - root - INFO - step: 13470 loss: 17.3328 memory: 44.58GiB(31.99%) tps: 83,017 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.7207 global_avg_mtp_loss: 14.6121 +[titan] 2025-07-09 16:34:38,301 - root - INFO - lr: 2.8957e-04 gnorm: 0.85 [ 2:59:41<19:14:20] +[titan] 2025-07-09 16:34:42,242 - root - INFO - step: 13475 loss: 17.1310 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.6801 global_avg_mtp_loss: 14.4509 +[titan] 2025-07-09 16:34:42,243 - root - INFO - lr: 2.8956e-04 gnorm: 0.85 [ 2:59:45<19:14:16] +[titan] 2025-07-09 16:34:46,175 - root - INFO - step: 13480 loss: 17.2137 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.6855 global_avg_mtp_loss: 14.5281 +[titan] 2025-07-09 16:34:46,175 - root - INFO - lr: 2.8955e-04 gnorm: 0.86 [ 2:59:49<19:14:11] +[titan] 2025-07-09 16:34:50,062 - root - INFO - step: 13485 loss: 17.3914 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.7434 global_avg_mtp_loss: 14.6480 +[titan] 2025-07-09 16:34:50,062 - root - INFO - lr: 2.8954e-04 gnorm: 0.86 [ 2:59:53<19:14:07] +[titan] 2025-07-09 16:34:53,984 - root - INFO - step: 13490 loss: 17.1321 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6773 global_avg_mtp_loss: 14.4548 +[titan] 2025-07-09 16:34:53,985 - root - INFO - lr: 2.8953e-04 gnorm: 0.87 [ 2:59:57<19:14:02] +[titan] 2025-07-09 16:34:57,883 - root - INFO - step: 13495 loss: 17.4134 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.7336 global_avg_mtp_loss: 14.6798 +[titan] 2025-07-09 16:34:57,883 - root - INFO - lr: 2.8952e-04 gnorm: 0.90 [ 3:00:01<19:13:58] +[titan] 2025-07-09 16:35:01,004 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:35:01,790 - root - INFO - step: 13500 loss: 17.0084 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6544 global_avg_mtp_loss: 14.3540 +[titan] 2025-07-09 16:35:01,790 - root - INFO - lr: 2.8952e-04 gnorm: 0.87 [ 3:00:05<19:13:53] +[titan] 2025-07-09 16:35:05,724 - root - INFO - step: 13505 loss: 17.0797 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.6747 global_avg_mtp_loss: 14.4050 +[titan] 2025-07-09 16:35:05,724 - root - INFO - lr: 2.8951e-04 gnorm: 0.90 [ 3:00:09<19:13:49] +[titan] 2025-07-09 16:35:09,620 - root - INFO - step: 13510 loss: 17.0275 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.6681 global_avg_mtp_loss: 14.3594 +[titan] 2025-07-09 16:35:09,621 - root - INFO - lr: 2.8950e-04 gnorm: 0.90 [ 3:00:13<19:13:44] +[titan] 2025-07-09 16:35:13,535 - root - INFO - step: 13515 loss: 17.0083 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6587 global_avg_mtp_loss: 14.3496 +[titan] 2025-07-09 16:35:13,535 - root - INFO - lr: 2.8949e-04 gnorm: 0.90 [ 3:00:16<19:13:39] +[titan] 2025-07-09 16:35:17,451 - root - INFO - step: 13520 loss: 17.4554 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.7614 global_avg_mtp_loss: 14.6940 +[titan] 2025-07-09 16:35:17,451 - root - INFO - lr: 2.8948e-04 gnorm: 0.84 [ 3:00:20<19:13:35] +[titan] 2025-07-09 16:35:21,414 - root - INFO - step: 13525 loss: 16.9593 memory: 44.58GiB(31.99%) tps: 82,682 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.3113 +[titan] 2025-07-09 16:35:21,414 - root - INFO - lr: 2.8947e-04 gnorm: 0.90 [ 3:00:24<19:13:30] +[titan] 2025-07-09 16:35:25,340 - root - INFO - step: 13530 loss: 17.3663 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.7314 global_avg_mtp_loss: 14.6348 +[titan] 2025-07-09 16:35:25,340 - root - INFO - lr: 2.8947e-04 gnorm: 0.89 [ 3:00:28<19:13:26] +[titan] 2025-07-09 16:35:29,266 - root - INFO - step: 13535 loss: 17.2619 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.7036 global_avg_mtp_loss: 14.5583 +[titan] 2025-07-09 16:35:29,267 - root - INFO - lr: 2.8946e-04 gnorm: 0.89 [ 3:00:32<19:13:21] +[titan] 2025-07-09 16:35:33,172 - root - INFO - step: 13540 loss: 17.0925 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6708 global_avg_mtp_loss: 14.4216 +[titan] 2025-07-09 16:35:33,172 - root - INFO - lr: 2.8945e-04 gnorm: 0.94 [ 3:00:36<19:13:17] +[titan] 2025-07-09 16:35:37,116 - root - INFO - step: 13545 loss: 17.0905 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.6763 global_avg_mtp_loss: 14.4143 +[titan] 2025-07-09 16:35:37,117 - root - INFO - lr: 2.8944e-04 gnorm: 0.92 [ 3:00:40<19:13:13] +[titan] 2025-07-09 16:35:40,228 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:35:41,020 - root - INFO - step: 13550 loss: 16.9830 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6499 global_avg_mtp_loss: 14.3331 +[titan] 2025-07-09 16:35:41,021 - root - INFO - lr: 2.8943e-04 gnorm: 0.92 [ 3:00:44<19:13:08] +[titan] 2025-07-09 16:35:44,933 - root - INFO - step: 13555 loss: 17.3772 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.7228 global_avg_mtp_loss: 14.6544 +[titan] 2025-07-09 16:35:44,933 - root - INFO - lr: 2.8943e-04 gnorm: 0.87 [ 3:00:48<19:13:03] +[titan] 2025-07-09 16:35:48,862 - root - INFO - step: 13560 loss: 17.1602 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6835 global_avg_mtp_loss: 14.4767 +[titan] 2025-07-09 16:35:48,862 - root - INFO - lr: 2.8942e-04 gnorm: 0.86 [ 3:00:52<19:12:59] +[titan] 2025-07-09 16:35:52,750 - root - INFO - step: 13565 loss: 17.2422 memory: 44.58GiB(31.99%) tps: 84,285 tflops: 290.88 mfu: 29.41% global_avg_ntp_loss: 2.7063 global_avg_mtp_loss: 14.5360 +[titan] 2025-07-09 16:35:52,750 - root - INFO - lr: 2.8941e-04 gnorm: 0.84 [ 3:00:56<19:12:54] +[titan] 2025-07-09 16:35:56,670 - root - INFO - step: 13570 loss: 17.2811 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.7093 global_avg_mtp_loss: 14.5718 +[titan] 2025-07-09 16:35:56,670 - root - INFO - lr: 2.8940e-04 gnorm: 0.82 [ 3:01:00<19:12:50] +[titan] 2025-07-09 16:36:00,580 - root - INFO - step: 13575 loss: 17.2972 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7099 global_avg_mtp_loss: 14.5873 +[titan] 2025-07-09 16:36:00,580 - root - INFO - lr: 2.8939e-04 gnorm: 0.82 [ 3:01:04<19:12:45] +[titan] 2025-07-09 16:36:04,482 - root - INFO - step: 13580 loss: 17.0875 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6701 global_avg_mtp_loss: 14.4174 +[titan] 2025-07-09 16:36:04,483 - root - INFO - lr: 2.8938e-04 gnorm: 0.93 [ 3:01:07<19:12:40] +[titan] 2025-07-09 16:36:08,396 - root - INFO - step: 13585 loss: 17.1495 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6918 global_avg_mtp_loss: 14.4576 +[titan] 2025-07-09 16:36:08,396 - root - INFO - lr: 2.8938e-04 gnorm: 0.83 [ 3:01:11<19:12:36] +[titan] 2025-07-09 16:36:12,331 - root - INFO - step: 13590 loss: 16.9899 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6482 global_avg_mtp_loss: 14.3417 +[titan] 2025-07-09 16:36:12,332 - root - INFO - lr: 2.8937e-04 gnorm: 0.84 [ 3:01:15<19:12:31] +[titan] 2025-07-09 16:36:16,277 - root - INFO - step: 13595 loss: 17.2422 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.7014 global_avg_mtp_loss: 14.5408 +[titan] 2025-07-09 16:36:16,277 - root - INFO - lr: 2.8936e-04 gnorm: 0.83 [ 3:01:19<19:12:27] +[titan] 2025-07-09 16:36:19,382 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:36:20,168 - root - INFO - step: 13600 loss: 17.2305 memory: 44.58GiB(31.99%) tps: 84,217 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.7051 global_avg_mtp_loss: 14.5254 +[titan] 2025-07-09 16:36:20,168 - root - INFO - lr: 2.8935e-04 gnorm: 0.82 [ 3:01:23<19:12:22] +[titan] 2025-07-09 16:36:24,055 - root - INFO - step: 13605 loss: 17.1826 memory: 44.58GiB(31.99%) tps: 84,322 tflops: 291.01 mfu: 29.42% global_avg_ntp_loss: 2.6910 global_avg_mtp_loss: 14.4917 +[titan] 2025-07-09 16:36:24,055 - root - INFO - lr: 2.8934e-04 gnorm: 0.79 [ 3:01:27<19:12:18] +[titan] 2025-07-09 16:36:27,978 - root - INFO - step: 13610 loss: 17.1595 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6789 global_avg_mtp_loss: 14.4806 +[titan] 2025-07-09 16:36:27,979 - root - INFO - lr: 2.8933e-04 gnorm: 0.92 [ 3:01:31<19:12:13] +[titan] 2025-07-09 16:36:31,907 - root - INFO - step: 13615 loss: 17.0588 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6680 global_avg_mtp_loss: 14.3908 +[titan] 2025-07-09 16:36:31,907 - root - INFO - lr: 2.8933e-04 gnorm: 0.90 [ 3:01:35<19:12:09] +[titan] 2025-07-09 16:36:35,829 - root - INFO - step: 13620 loss: 17.1943 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6957 global_avg_mtp_loss: 14.4986 +[titan] 2025-07-09 16:36:35,829 - root - INFO - lr: 2.8932e-04 gnorm: 0.84 [ 3:01:39<19:12:04] +[titan] 2025-07-09 16:36:39,780 - root - INFO - step: 13625 loss: 16.7804 memory: 44.58GiB(31.99%) tps: 82,937 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.1595 +[titan] 2025-07-09 16:36:39,781 - root - INFO - lr: 2.8931e-04 gnorm: 0.93 [ 3:01:43<19:12:00] +[titan] 2025-07-09 16:36:43,718 - root - INFO - step: 13630 loss: 17.3794 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.7445 global_avg_mtp_loss: 14.6349 +[titan] 2025-07-09 16:36:43,718 - root - INFO - lr: 2.8930e-04 gnorm: 0.94 [ 3:01:47<19:11:55] +[titan] 2025-07-09 16:36:47,612 - root - INFO - step: 13635 loss: 17.1128 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6876 global_avg_mtp_loss: 14.4252 +[titan] 2025-07-09 16:36:47,612 - root - INFO - lr: 2.8929e-04 gnorm: 0.90 [ 3:01:51<19:11:51] +[titan] 2025-07-09 16:36:51,535 - root - INFO - step: 13640 loss: 17.1153 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6796 global_avg_mtp_loss: 14.4357 +[titan] 2025-07-09 16:36:51,535 - root - INFO - lr: 2.8928e-04 gnorm: 0.80 [ 3:01:54<19:11:46] +[titan] 2025-07-09 16:36:55,461 - root - INFO - step: 13645 loss: 17.2500 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.7076 global_avg_mtp_loss: 14.5424 +[titan] 2025-07-09 16:36:55,461 - root - INFO - lr: 2.8928e-04 gnorm: 0.83 [ 3:01:58<19:11:42] +[titan] 2025-07-09 16:36:58,585 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:36:59,384 - root - INFO - step: 13650 loss: 17.2427 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.7096 global_avg_mtp_loss: 14.5331 +[titan] 2025-07-09 16:36:59,384 - root - INFO - lr: 2.8927e-04 gnorm: 0.89 [ 3:02:02<19:11:37] +[titan] 2025-07-09 16:37:03,306 - root - INFO - step: 13655 loss: 17.0844 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6827 global_avg_mtp_loss: 14.4018 +[titan] 2025-07-09 16:37:03,306 - root - INFO - lr: 2.8926e-04 gnorm: 0.85 [ 3:02:06<19:11:33] +[titan] 2025-07-09 16:37:07,220 - root - INFO - step: 13660 loss: 16.8690 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2379 +[titan] 2025-07-09 16:37:07,221 - root - INFO - lr: 2.8925e-04 gnorm: 0.82 [ 3:02:10<19:11:28] +[titan] 2025-07-09 16:37:11,108 - root - INFO - step: 13665 loss: 17.1108 memory: 44.58GiB(31.99%) tps: 84,295 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.4466 +[titan] 2025-07-09 16:37:11,108 - root - INFO - lr: 2.8924e-04 gnorm: 0.82 [ 3:02:14<19:11:23] +[titan] 2025-07-09 16:37:15,055 - root - INFO - step: 13670 loss: 17.1125 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.6871 global_avg_mtp_loss: 14.4253 +[titan] 2025-07-09 16:37:15,056 - root - INFO - lr: 2.8923e-04 gnorm: 0.81 [ 3:02:18<19:11:19] +[titan] 2025-07-09 16:37:18,973 - root - INFO - step: 13675 loss: 17.1228 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6695 global_avg_mtp_loss: 14.4533 +[titan] 2025-07-09 16:37:18,973 - root - INFO - lr: 2.8922e-04 gnorm: 0.91 [ 3:02:22<19:11:15] +[titan] 2025-07-09 16:37:22,897 - root - INFO - step: 13680 loss: 17.2078 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6910 global_avg_mtp_loss: 14.5168 +[titan] 2025-07-09 16:37:22,897 - root - INFO - lr: 2.8922e-04 gnorm: 0.80 [ 3:02:26<19:11:10] +[titan] 2025-07-09 16:37:26,801 - root - INFO - step: 13685 loss: 17.0536 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6786 global_avg_mtp_loss: 14.3750 +[titan] 2025-07-09 16:37:26,801 - root - INFO - lr: 2.8921e-04 gnorm: 0.96 [ 3:02:30<19:11:05] +[titan] 2025-07-09 16:37:30,765 - root - INFO - step: 13690 loss: 17.0558 memory: 44.58GiB(31.99%) tps: 82,683 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.6548 global_avg_mtp_loss: 14.4010 +[titan] 2025-07-09 16:37:30,765 - root - INFO - lr: 2.8920e-04 gnorm: 0.90 [ 3:02:34<19:11:01] +[titan] 2025-07-09 16:37:34,671 - root - INFO - step: 13695 loss: 17.3041 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.7120 global_avg_mtp_loss: 14.5920 +[titan] 2025-07-09 16:37:34,671 - root - INFO - lr: 2.8919e-04 gnorm: 0.99 [ 3:02:38<19:10:57] +[titan] 2025-07-09 16:37:37,789 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:37:38,577 - root - INFO - step: 13700 loss: 17.3757 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.7286 global_avg_mtp_loss: 14.6471 +[titan] 2025-07-09 16:37:38,577 - root - INFO - lr: 2.8918e-04 gnorm: 0.92 [ 3:02:41<19:10:52] +[titan] 2025-07-09 16:37:42,497 - root - INFO - step: 13705 loss: 17.3218 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.7165 global_avg_mtp_loss: 14.6054 +[titan] 2025-07-09 16:37:42,498 - root - INFO - lr: 2.8917e-04 gnorm: 0.83 [ 3:02:45<19:10:48] +[titan] 2025-07-09 16:37:46,413 - root - INFO - step: 13710 loss: 17.3583 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7393 global_avg_mtp_loss: 14.6190 +[titan] 2025-07-09 16:37:46,413 - root - INFO - lr: 2.8917e-04 gnorm: 0.88 [ 3:02:49<19:10:43] +[titan] 2025-07-09 16:37:50,353 - root - INFO - step: 13715 loss: 17.2037 memory: 44.58GiB(31.99%) tps: 83,156 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.7073 global_avg_mtp_loss: 14.4964 +[titan] 2025-07-09 16:37:50,354 - root - INFO - lr: 2.8916e-04 gnorm: 0.84 [ 3:02:53<19:10:39] +[titan] 2025-07-09 16:37:54,268 - root - INFO - step: 13720 loss: 17.1186 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.4236 +[titan] 2025-07-09 16:37:54,269 - root - INFO - lr: 2.8915e-04 gnorm: 0.88 [ 3:02:57<19:10:34] +[titan] 2025-07-09 16:37:58,196 - root - INFO - step: 13725 loss: 17.3427 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.7446 global_avg_mtp_loss: 14.5981 +[titan] 2025-07-09 16:37:58,196 - root - INFO - lr: 2.8914e-04 gnorm: 0.93 [ 3:03:01<19:10:30] +[titan] 2025-07-09 16:38:02,101 - root - INFO - step: 13730 loss: 17.0823 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6615 global_avg_mtp_loss: 14.4208 +[titan] 2025-07-09 16:38:02,102 - root - INFO - lr: 2.8913e-04 gnorm: 0.86 [ 3:03:05<19:10:25] +[titan] 2025-07-09 16:38:06,025 - root - INFO - step: 13735 loss: 17.4285 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.7351 global_avg_mtp_loss: 14.6934 +[titan] 2025-07-09 16:38:06,025 - root - INFO - lr: 2.8912e-04 gnorm: 0.86 [ 3:03:09<19:10:21] +[titan] 2025-07-09 16:38:09,925 - root - INFO - step: 13740 loss: 17.1589 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6971 global_avg_mtp_loss: 14.4618 +[titan] 2025-07-09 16:38:09,925 - root - INFO - lr: 2.8912e-04 gnorm: 0.86 [ 3:03:13<19:10:16] +[titan] 2025-07-09 16:38:13,820 - root - INFO - step: 13745 loss: 17.0520 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.3764 +[titan] 2025-07-09 16:38:13,820 - root - INFO - lr: 2.8911e-04 gnorm: 0.89 [ 3:03:17<19:10:11] +[titan] 2025-07-09 16:38:16,946 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:38:17,730 - root - INFO - step: 13750 loss: 17.0481 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.3885 +[titan] 2025-07-09 16:38:17,730 - root - INFO - lr: 2.8910e-04 gnorm: 0.86 [ 3:03:21<19:10:07] +[titan] 2025-07-09 16:38:21,637 - root - INFO - step: 13755 loss: 17.0302 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6725 global_avg_mtp_loss: 14.3576 +[titan] 2025-07-09 16:38:21,637 - root - INFO - lr: 2.8909e-04 gnorm: 0.97 [ 3:03:25<19:10:02] +[titan] 2025-07-09 16:38:25,542 - root - INFO - step: 13760 loss: 17.1793 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6869 global_avg_mtp_loss: 14.4924 +[titan] 2025-07-09 16:38:25,542 - root - INFO - lr: 2.8908e-04 gnorm: 0.89 [ 3:03:28<19:09:57] +[titan] 2025-07-09 16:38:29,445 - root - INFO - step: 13765 loss: 16.9930 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6449 global_avg_mtp_loss: 14.3482 +[titan] 2025-07-09 16:38:29,445 - root - INFO - lr: 2.8907e-04 gnorm: 0.85 [ 3:03:32<19:09:53] +[titan] 2025-07-09 16:38:33,397 - root - INFO - step: 13770 loss: 17.0052 memory: 44.58GiB(31.99%) tps: 82,921 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.6633 global_avg_mtp_loss: 14.3418 +[titan] 2025-07-09 16:38:33,397 - root - INFO - lr: 2.8906e-04 gnorm: 0.87 [ 3:03:36<19:09:49] +[titan] 2025-07-09 16:38:37,295 - root - INFO - step: 13775 loss: 17.2727 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.7104 global_avg_mtp_loss: 14.5623 +[titan] 2025-07-09 16:38:37,295 - root - INFO - lr: 2.8906e-04 gnorm: 0.79 [ 3:03:40<19:09:44] +[titan] 2025-07-09 16:38:41,209 - root - INFO - step: 13780 loss: 17.2555 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.7166 global_avg_mtp_loss: 14.5389 +[titan] 2025-07-09 16:38:41,210 - root - INFO - lr: 2.8905e-04 gnorm: 0.85 [ 3:03:44<19:09:39] +[titan] 2025-07-09 16:38:45,138 - root - INFO - step: 13785 loss: 17.1334 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6910 global_avg_mtp_loss: 14.4424 +[titan] 2025-07-09 16:38:45,139 - root - INFO - lr: 2.8904e-04 gnorm: 0.87 [ 3:03:48<19:09:35] +[titan] 2025-07-09 16:38:49,085 - root - INFO - step: 13790 loss: 17.2721 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.7108 global_avg_mtp_loss: 14.5613 +[titan] 2025-07-09 16:38:49,086 - root - INFO - lr: 2.8903e-04 gnorm: 0.82 [ 3:03:52<19:09:31] +[titan] 2025-07-09 16:38:53,009 - root - INFO - step: 13795 loss: 17.3566 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.7195 global_avg_mtp_loss: 14.6371 +[titan] 2025-07-09 16:38:53,010 - root - INFO - lr: 2.8902e-04 gnorm: 0.88 [ 3:03:56<19:09:26] +[titan] 2025-07-09 16:38:56,135 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:38:56,963 - root - INFO - step: 13800 loss: 17.0815 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.6762 global_avg_mtp_loss: 14.4053 +[titan] 2025-07-09 16:38:56,963 - root - INFO - lr: 2.8901e-04 gnorm: 0.92 [ 3:04:00<19:09:22] +[titan] 2025-07-09 16:39:00,878 - root - INFO - step: 13805 loss: 17.3776 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7280 global_avg_mtp_loss: 14.6496 +[titan] 2025-07-09 16:39:00,878 - root - INFO - lr: 2.8901e-04 gnorm: 0.94 [ 3:04:04<19:09:17] +[titan] 2025-07-09 16:39:04,781 - root - INFO - step: 13810 loss: 17.1577 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6884 global_avg_mtp_loss: 14.4693 +[titan] 2025-07-09 16:39:04,781 - root - INFO - lr: 2.8900e-04 gnorm: 0.95 [ 3:04:08<19:09:13] +[titan] 2025-07-09 16:39:08,680 - root - INFO - step: 13815 loss: 17.0093 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6553 global_avg_mtp_loss: 14.3540 +[titan] 2025-07-09 16:39:08,680 - root - INFO - lr: 2.8899e-04 gnorm: 0.88 [ 3:04:12<19:09:08] +[titan] 2025-07-09 16:39:12,623 - root - INFO - step: 13820 loss: 17.1085 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.6849 global_avg_mtp_loss: 14.4236 +[titan] 2025-07-09 16:39:12,623 - root - INFO - lr: 2.8898e-04 gnorm: 6.41 [ 3:04:16<19:09:04] +[titan] 2025-07-09 16:39:15,969 - root - INFO - Dumping profiler traces at step 13824 +[titan] 2025-07-09 16:39:16,000 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:39:16,798 - root - INFO - step: 13825 loss: 17.0022 memory: 44.58GiB(31.99%) tps: 78,485 tflops: 270.86 mfu: 27.39% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.3324 +[titan] 2025-07-09 16:39:16,798 - root - INFO - lr: 2.8897e-04 gnorm: 0.88 [ 3:04:20<19:09:01] +[titan] 2025-07-09 16:39:20,707 - root - INFO - step: 13830 loss: 17.1573 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6708 global_avg_mtp_loss: 14.4865 +[titan] 2025-07-09 16:39:20,707 - root - INFO - lr: 2.8896e-04 gnorm: 0.94 [ 3:04:24<19:08:56] +[titan] 2025-07-09 16:39:24,605 - root - INFO - step: 13835 loss: 17.0433 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.3832 +[titan] 2025-07-09 16:39:24,605 - root - INFO - lr: 2.8895e-04 gnorm: 0.86 [ 3:04:28<19:08:52] +[titan] 2025-07-09 16:39:28,506 - root - INFO - step: 13840 loss: 16.9963 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.3405 +[titan] 2025-07-09 16:39:28,507 - root - INFO - lr: 2.8895e-04 gnorm: 0.89 [ 3:04:31<19:08:47] +[titan] 2025-07-09 16:39:32,401 - root - INFO - step: 13845 loss: 17.1074 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6851 global_avg_mtp_loss: 14.4223 +[titan] 2025-07-09 16:39:32,402 - root - INFO - lr: 2.8894e-04 gnorm: 0.87 [ 3:04:35<19:08:42] +[titan] 2025-07-09 16:39:35,508 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:39:36,308 - root - INFO - step: 13850 loss: 16.7912 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6251 global_avg_mtp_loss: 14.1662 +[titan] 2025-07-09 16:39:36,308 - root - INFO - lr: 2.8893e-04 gnorm: 0.79 [ 3:04:39<19:08:38] +[titan] 2025-07-09 16:39:40,243 - root - INFO - step: 13855 loss: 17.0526 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.3992 +[titan] 2025-07-09 16:39:40,243 - root - INFO - lr: 2.8892e-04 gnorm: 0.87 [ 3:04:43<19:08:33] +[titan] 2025-07-09 16:39:44,158 - root - INFO - step: 13860 loss: 17.0199 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6637 global_avg_mtp_loss: 14.3562 +[titan] 2025-07-09 16:39:44,158 - root - INFO - lr: 2.8891e-04 gnorm: 0.95 [ 3:04:47<19:08:29] +[titan] 2025-07-09 16:39:48,073 - root - INFO - step: 13865 loss: 16.9826 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6585 global_avg_mtp_loss: 14.3242 +[titan] 2025-07-09 16:39:48,073 - root - INFO - lr: 2.8890e-04 gnorm: 0.88 [ 3:04:51<19:08:24] +[titan] 2025-07-09 16:39:52,001 - root - INFO - step: 13870 loss: 17.1347 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6748 global_avg_mtp_loss: 14.4600 +[titan] 2025-07-09 16:39:52,001 - root - INFO - lr: 2.8890e-04 gnorm: 0.86 [ 3:04:55<19:08:20] +[titan] 2025-07-09 16:39:55,899 - root - INFO - step: 13875 loss: 17.2420 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.7054 global_avg_mtp_loss: 14.5366 +[titan] 2025-07-09 16:39:55,899 - root - INFO - lr: 2.8889e-04 gnorm: 0.91 [ 3:04:59<19:08:15] +[titan] 2025-07-09 16:39:59,817 - root - INFO - step: 13880 loss: 17.0586 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6830 global_avg_mtp_loss: 14.3756 +[titan] 2025-07-09 16:39:59,818 - root - INFO - lr: 2.8888e-04 gnorm: 0.79 [ 3:05:03<19:08:11] +[titan] 2025-07-09 16:40:03,741 - root - INFO - step: 13885 loss: 17.0379 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6620 global_avg_mtp_loss: 14.3759 +[titan] 2025-07-09 16:40:03,741 - root - INFO - lr: 2.8887e-04 gnorm: 0.80 [ 3:05:07<19:08:06] +[titan] 2025-07-09 16:40:07,635 - root - INFO - step: 13890 loss: 16.8672 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.2258 +[titan] 2025-07-09 16:40:07,635 - root - INFO - lr: 2.8886e-04 gnorm: 0.81 [ 3:05:11<19:08:01] +[titan] 2025-07-09 16:40:11,546 - root - INFO - step: 13895 loss: 17.0815 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6839 global_avg_mtp_loss: 14.3976 +[titan] 2025-07-09 16:40:11,547 - root - INFO - lr: 2.8885e-04 gnorm: 0.84 [ 3:05:14<19:07:57] +[titan] 2025-07-09 16:40:14,662 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:40:15,446 - root - INFO - step: 13900 loss: 17.1650 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6783 global_avg_mtp_loss: 14.4867 +[titan] 2025-07-09 16:40:15,446 - root - INFO - lr: 2.8884e-04 gnorm: 0.79 [ 3:05:18<19:07:52] +[titan] 2025-07-09 16:40:19,347 - root - INFO - step: 13905 loss: 17.2515 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.7075 global_avg_mtp_loss: 14.5441 +[titan] 2025-07-09 16:40:19,348 - root - INFO - lr: 2.8884e-04 gnorm: 0.94 [ 3:05:22<19:07:48] +[titan] 2025-07-09 16:40:23,240 - root - INFO - step: 13910 loss: 17.3180 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.7282 global_avg_mtp_loss: 14.5898 +[titan] 2025-07-09 16:40:23,241 - root - INFO - lr: 2.8883e-04 gnorm: 1.01 [ 3:05:26<19:07:43] +[titan] 2025-07-09 16:40:27,139 - root - INFO - step: 13915 loss: 17.1136 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4379 +[titan] 2025-07-09 16:40:27,140 - root - INFO - lr: 2.8882e-04 gnorm: 0.87 [ 3:05:30<19:07:38] +[titan] 2025-07-09 16:40:31,078 - root - INFO - step: 13920 loss: 17.1969 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.6901 global_avg_mtp_loss: 14.5068 +[titan] 2025-07-09 16:40:31,079 - root - INFO - lr: 2.8881e-04 gnorm: 0.88 [ 3:05:34<19:07:34] +[titan] 2025-07-09 16:40:35,000 - root - INFO - step: 13925 loss: 16.8432 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6312 global_avg_mtp_loss: 14.2120 +[titan] 2025-07-09 16:40:35,000 - root - INFO - lr: 2.8880e-04 gnorm: 0.90 [ 3:05:38<19:07:30] +[titan] 2025-07-09 16:40:38,914 - root - INFO - step: 13930 loss: 16.9324 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6561 global_avg_mtp_loss: 14.2763 +[titan] 2025-07-09 16:40:38,914 - root - INFO - lr: 2.8879e-04 gnorm: 0.86 [ 3:05:42<19:07:25] +[titan] 2025-07-09 16:40:42,863 - root - INFO - step: 13935 loss: 17.2410 memory: 44.58GiB(31.99%) tps: 82,996 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6972 global_avg_mtp_loss: 14.5438 +[titan] 2025-07-09 16:40:42,863 - root - INFO - lr: 2.8878e-04 gnorm: 0.83 [ 3:05:46<19:07:21] +[titan] 2025-07-09 16:40:46,810 - root - INFO - step: 13940 loss: 17.3296 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.7192 global_avg_mtp_loss: 14.6104 +[titan] 2025-07-09 16:40:46,810 - root - INFO - lr: 2.8878e-04 gnorm: 0.88 [ 3:05:50<19:07:16] +[titan] 2025-07-09 16:40:50,712 - root - INFO - step: 13945 loss: 17.1253 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6777 global_avg_mtp_loss: 14.4476 +[titan] 2025-07-09 16:40:50,713 - root - INFO - lr: 2.8877e-04 gnorm: 0.95 [ 3:05:54<19:07:12] +[titan] 2025-07-09 16:40:53,840 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:40:54,647 - root - INFO - step: 13950 loss: 17.1305 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.4354 +[titan] 2025-07-09 16:40:54,647 - root - INFO - lr: 2.8876e-04 gnorm: 0.87 [ 3:05:58<19:07:07] +[titan] 2025-07-09 16:40:58,588 - root - INFO - step: 13955 loss: 17.3113 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.7121 global_avg_mtp_loss: 14.5992 +[titan] 2025-07-09 16:40:58,588 - root - INFO - lr: 2.8875e-04 gnorm: 0.86 [ 3:06:01<19:07:03] +[titan] 2025-07-09 16:41:02,487 - root - INFO - step: 13960 loss: 17.1925 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.5103 +[titan] 2025-07-09 16:41:02,488 - root - INFO - lr: 2.8874e-04 gnorm: 0.88 [ 3:06:05<19:06:58] +[titan] 2025-07-09 16:41:06,420 - root - INFO - step: 13965 loss: 17.2396 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.7045 global_avg_mtp_loss: 14.5351 +[titan] 2025-07-09 16:41:06,420 - root - INFO - lr: 2.8873e-04 gnorm: 0.91 [ 3:06:09<19:06:54] +[titan] 2025-07-09 16:41:10,368 - root - INFO - step: 13970 loss: 17.2827 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.7123 global_avg_mtp_loss: 14.5704 +[titan] 2025-07-09 16:41:10,368 - root - INFO - lr: 2.8872e-04 gnorm: 1.00 [ 3:06:13<19:06:50] +[titan] 2025-07-09 16:41:14,312 - root - INFO - step: 13975 loss: 17.0871 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6768 global_avg_mtp_loss: 14.4103 +[titan] 2025-07-09 16:41:14,312 - root - INFO - lr: 2.8872e-04 gnorm: 0.90 [ 3:06:17<19:06:45] +[titan] 2025-07-09 16:41:18,214 - root - INFO - step: 13980 loss: 16.8668 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6369 global_avg_mtp_loss: 14.2299 +[titan] 2025-07-09 16:41:18,214 - root - INFO - lr: 2.8871e-04 gnorm: 0.86 [ 3:06:21<19:06:41] +[titan] 2025-07-09 16:41:22,113 - root - INFO - step: 13985 loss: 17.2138 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6967 global_avg_mtp_loss: 14.5171 +[titan] 2025-07-09 16:41:22,114 - root - INFO - lr: 2.8870e-04 gnorm: 0.88 [ 3:06:25<19:06:36] +[titan] 2025-07-09 16:41:26,022 - root - INFO - step: 13990 loss: 17.3650 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.7384 global_avg_mtp_loss: 14.6266 +[titan] 2025-07-09 16:41:26,022 - root - INFO - lr: 2.8869e-04 gnorm: 0.87 [ 3:06:29<19:06:32] +[titan] 2025-07-09 16:41:29,928 - root - INFO - step: 13995 loss: 17.0145 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6613 global_avg_mtp_loss: 14.3532 +[titan] 2025-07-09 16:41:29,928 - root - INFO - lr: 2.8868e-04 gnorm: 0.84 [ 3:06:33<19:06:27] +[titan] 2025-07-09 16:41:33,029 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:41:33,814 - root - INFO - step: 14000 loss: 17.1454 memory: 44.58GiB(31.99%) tps: 84,340 tflops: 291.07 mfu: 29.43% global_avg_ntp_loss: 2.6831 global_avg_mtp_loss: 14.4624 +[titan] 2025-07-09 16:41:33,814 - root - INFO - lr: 2.8867e-04 gnorm: 0.86 [ 3:06:37<19:06:22] +[titan] 2025-07-09 16:41:37,750 - root - INFO - step: 14005 loss: 17.0800 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6592 global_avg_mtp_loss: 14.4208 +[titan] 2025-07-09 16:41:37,750 - root - INFO - lr: 2.8866e-04 gnorm: 0.86 [ 3:06:41<19:06:18] +[titan] 2025-07-09 16:41:41,693 - root - INFO - step: 14010 loss: 17.0093 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.3452 +[titan] 2025-07-09 16:41:41,693 - root - INFO - lr: 2.8866e-04 gnorm: 0.85 [ 3:06:45<19:06:14] +[titan] 2025-07-09 16:41:45,599 - root - INFO - step: 14015 loss: 17.2724 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.7082 global_avg_mtp_loss: 14.5642 +[titan] 2025-07-09 16:41:45,599 - root - INFO - lr: 2.8865e-04 gnorm: 0.91 [ 3:06:48<19:06:09] +[titan] 2025-07-09 16:41:49,515 - root - INFO - step: 14020 loss: 16.9475 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.6394 global_avg_mtp_loss: 14.3081 +[titan] 2025-07-09 16:41:49,515 - root - INFO - lr: 2.8864e-04 gnorm: 0.87 [ 3:06:52<19:06:04] +[titan] 2025-07-09 16:41:53,407 - root - INFO - step: 14025 loss: 17.1586 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.6832 global_avg_mtp_loss: 14.4754 +[titan] 2025-07-09 16:41:53,407 - root - INFO - lr: 2.8863e-04 gnorm: 0.82 [ 3:06:56<19:06:00] +[titan] 2025-07-09 16:41:57,315 - root - INFO - step: 14030 loss: 17.3751 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7268 global_avg_mtp_loss: 14.6483 +[titan] 2025-07-09 16:41:57,315 - root - INFO - lr: 2.8862e-04 gnorm: 0.87 [ 3:07:00<19:05:55] +[titan] 2025-07-09 16:42:01,226 - root - INFO - step: 14035 loss: 17.3636 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.7358 global_avg_mtp_loss: 14.6278 +[titan] 2025-07-09 16:42:01,226 - root - INFO - lr: 2.8861e-04 gnorm: 0.96 [ 3:07:04<19:05:51] +[titan] 2025-07-09 16:42:05,164 - root - INFO - step: 14040 loss: 17.2834 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.5846 +[titan] 2025-07-09 16:42:05,164 - root - INFO - lr: 2.8860e-04 gnorm: 0.82 [ 3:07:08<19:05:46] +[titan] 2025-07-09 16:42:09,087 - root - INFO - step: 14045 loss: 17.1138 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6764 global_avg_mtp_loss: 14.4373 +[titan] 2025-07-09 16:42:09,087 - root - INFO - lr: 2.8860e-04 gnorm: 0.82 [ 3:07:12<19:05:42] +[titan] 2025-07-09 16:42:12,206 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:42:12,998 - root - INFO - step: 14050 loss: 16.8714 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6347 global_avg_mtp_loss: 14.2367 +[titan] 2025-07-09 16:42:12,998 - root - INFO - lr: 2.8859e-04 gnorm: 0.82 [ 3:07:16<19:05:37] +[titan] 2025-07-09 16:42:16,913 - root - INFO - step: 14055 loss: 17.2040 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6968 global_avg_mtp_loss: 14.5073 +[titan] 2025-07-09 16:42:16,913 - root - INFO - lr: 2.8858e-04 gnorm: 0.83 [ 3:07:20<19:05:33] +[titan] 2025-07-09 16:42:20,851 - root - INFO - step: 14060 loss: 17.1751 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.6852 global_avg_mtp_loss: 14.4898 +[titan] 2025-07-09 16:42:20,851 - root - INFO - lr: 2.8857e-04 gnorm: 0.84 [ 3:07:24<19:05:28] +[titan] 2025-07-09 16:42:24,760 - root - INFO - step: 14065 loss: 16.9928 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.3408 +[titan] 2025-07-09 16:42:24,761 - root - INFO - lr: 2.8856e-04 gnorm: 0.85 [ 3:07:28<19:05:24] +[titan] 2025-07-09 16:42:28,711 - root - INFO - step: 14070 loss: 17.4019 memory: 44.58GiB(31.99%) tps: 82,947 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.7285 global_avg_mtp_loss: 14.6734 +[titan] 2025-07-09 16:42:28,712 - root - INFO - lr: 2.8855e-04 gnorm: 0.89 [ 3:07:32<19:05:20] +[titan] 2025-07-09 16:42:32,668 - root - INFO - step: 14075 loss: 16.8408 memory: 44.58GiB(31.99%) tps: 82,820 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.6298 global_avg_mtp_loss: 14.2111 +[titan] 2025-07-09 16:42:32,668 - root - INFO - lr: 2.8854e-04 gnorm: 0.86 [ 3:07:36<19:05:15] +[titan] 2025-07-09 16:42:36,566 - root - INFO - step: 14080 loss: 17.0334 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6655 global_avg_mtp_loss: 14.3679 +[titan] 2025-07-09 16:42:36,566 - root - INFO - lr: 2.8854e-04 gnorm: 0.82 [ 3:07:39<19:05:11] +[titan] 2025-07-09 16:42:40,452 - root - INFO - step: 14085 loss: 17.1103 memory: 44.58GiB(31.99%) tps: 84,328 tflops: 291.03 mfu: 29.43% global_avg_ntp_loss: 2.6739 global_avg_mtp_loss: 14.4364 +[titan] 2025-07-09 16:42:40,452 - root - INFO - lr: 2.8853e-04 gnorm: 0.88 [ 3:07:43<19:05:06] +[titan] 2025-07-09 16:42:44,363 - root - INFO - step: 14090 loss: 17.3639 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.7316 global_avg_mtp_loss: 14.6323 +[titan] 2025-07-09 16:42:44,363 - root - INFO - lr: 2.8852e-04 gnorm: 0.84 [ 3:07:47<19:05:02] +[titan] 2025-07-09 16:42:48,281 - root - INFO - step: 14095 loss: 17.2750 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.7127 global_avg_mtp_loss: 14.5622 +[titan] 2025-07-09 16:42:48,281 - root - INFO - lr: 2.8851e-04 gnorm: 1.00 [ 3:07:51<19:04:57] +[titan] 2025-07-09 16:42:51,450 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:42:52,242 - root - INFO - step: 14100 loss: 16.9388 memory: 44.58GiB(31.99%) tps: 82,736 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.6522 global_avg_mtp_loss: 14.2866 +[titan] 2025-07-09 16:42:52,242 - root - INFO - lr: 2.8850e-04 gnorm: 0.84 [ 3:07:55<19:04:53] +[titan] 2025-07-09 16:42:56,176 - root - INFO - step: 14105 loss: 17.1444 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6937 global_avg_mtp_loss: 14.4507 +[titan] 2025-07-09 16:42:56,177 - root - INFO - lr: 2.8849e-04 gnorm: 0.97 [ 3:07:59<19:04:48] +[titan] 2025-07-09 16:43:00,123 - root - INFO - step: 14110 loss: 17.2067 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.6844 global_avg_mtp_loss: 14.5223 +[titan] 2025-07-09 16:43:00,123 - root - INFO - lr: 2.8848e-04 gnorm: 0.96 [ 3:08:03<19:04:44] +[titan] 2025-07-09 16:43:04,043 - root - INFO - step: 14115 loss: 17.0294 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3656 +[titan] 2025-07-09 16:43:04,043 - root - INFO - lr: 2.8847e-04 gnorm: 0.86 [ 3:08:07<19:04:40] +[titan] 2025-07-09 16:43:08,011 - root - INFO - step: 14120 loss: 17.1781 memory: 44.58GiB(31.99%) tps: 82,594 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.7042 global_avg_mtp_loss: 14.4739 +[titan] 2025-07-09 16:43:08,011 - root - INFO - lr: 2.8847e-04 gnorm: 0.91 [ 3:08:11<19:04:35] +[titan] 2025-07-09 16:43:11,947 - root - INFO - step: 14125 loss: 17.1933 memory: 44.58GiB(31.99%) tps: 83,247 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.7012 global_avg_mtp_loss: 14.4922 +[titan] 2025-07-09 16:43:11,947 - root - INFO - lr: 2.8846e-04 gnorm: 0.89 [ 3:08:15<19:04:31] +[titan] 2025-07-09 16:43:15,853 - root - INFO - step: 14130 loss: 16.9019 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.2539 +[titan] 2025-07-09 16:43:15,853 - root - INFO - lr: 2.8845e-04 gnorm: 0.86 [ 3:08:19<19:04:26] +[titan] 2025-07-09 16:43:19,777 - root - INFO - step: 14135 loss: 17.0749 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6637 global_avg_mtp_loss: 14.4112 +[titan] 2025-07-09 16:43:19,777 - root - INFO - lr: 2.8844e-04 gnorm: 0.88 [ 3:08:23<19:04:22] +[titan] 2025-07-09 16:43:23,696 - root - INFO - step: 14140 loss: 16.8464 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.2093 +[titan] 2025-07-09 16:43:23,697 - root - INFO - lr: 2.8843e-04 gnorm: 0.88 [ 3:08:27<19:04:18] +[titan] 2025-07-09 16:43:27,619 - root - INFO - step: 14145 loss: 17.0788 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6706 global_avg_mtp_loss: 14.4082 +[titan] 2025-07-09 16:43:27,619 - root - INFO - lr: 2.8842e-04 gnorm: 0.90 [ 3:08:30<19:04:13] +[titan] 2025-07-09 16:43:30,755 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:43:31,545 - root - INFO - step: 14150 loss: 17.4150 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.7466 global_avg_mtp_loss: 14.6684 +[titan] 2025-07-09 16:43:31,545 - root - INFO - lr: 2.8841e-04 gnorm: 0.82 [ 3:08:34<19:04:09] +[titan] 2025-07-09 16:43:35,508 - root - INFO - step: 14155 loss: 17.2261 memory: 44.58GiB(31.99%) tps: 82,691 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.7078 global_avg_mtp_loss: 14.5184 +[titan] 2025-07-09 16:43:35,508 - root - INFO - lr: 2.8841e-04 gnorm: 0.84 [ 3:08:38<19:04:04] +[titan] 2025-07-09 16:43:39,420 - root - INFO - step: 14160 loss: 17.2004 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6991 global_avg_mtp_loss: 14.5013 +[titan] 2025-07-09 16:43:39,420 - root - INFO - lr: 2.8840e-04 gnorm: 0.84 [ 3:08:42<19:04:00] +[titan] 2025-07-09 16:43:43,336 - root - INFO - step: 14165 loss: 16.9543 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6418 global_avg_mtp_loss: 14.3125 +[titan] 2025-07-09 16:43:43,336 - root - INFO - lr: 2.8839e-04 gnorm: 0.87 [ 3:08:46<19:03:55] +[titan] 2025-07-09 16:43:47,262 - root - INFO - step: 14170 loss: 16.5162 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5783 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-09 16:43:47,263 - root - INFO - lr: 2.8838e-04 gnorm: 0.97 [ 3:08:50<19:03:51] +[titan] 2025-07-09 16:43:51,157 - root - INFO - step: 14175 loss: 17.0356 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.6573 global_avg_mtp_loss: 14.3783 +[titan] 2025-07-09 16:43:51,157 - root - INFO - lr: 2.8837e-04 gnorm: 0.79 [ 3:08:54<19:03:46] +[titan] 2025-07-09 16:43:55,126 - root - INFO - step: 14180 loss: 16.9651 memory: 44.58GiB(31.99%) tps: 82,569 tflops: 284.96 mfu: 28.81% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3133 +[titan] 2025-07-09 16:43:55,126 - root - INFO - lr: 2.8836e-04 gnorm: 0.80 [ 3:08:58<19:03:42] +[titan] 2025-07-09 16:43:59,031 - root - INFO - step: 14185 loss: 17.1488 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6731 global_avg_mtp_loss: 14.4756 +[titan] 2025-07-09 16:43:59,031 - root - INFO - lr: 2.8835e-04 gnorm: 0.87 [ 3:09:02<19:03:38] +[titan] 2025-07-09 16:44:02,927 - root - INFO - step: 14190 loss: 17.1276 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6804 global_avg_mtp_loss: 14.4473 +[titan] 2025-07-09 16:44:02,927 - root - INFO - lr: 2.8834e-04 gnorm: 0.83 [ 3:09:06<19:03:33] +[titan] 2025-07-09 16:44:06,856 - root - INFO - step: 14195 loss: 17.2899 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.7160 global_avg_mtp_loss: 14.5739 +[titan] 2025-07-09 16:44:06,857 - root - INFO - lr: 2.8834e-04 gnorm: 0.83 [ 3:09:10<19:03:29] +[titan] 2025-07-09 16:44:10,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:44:10,825 - root - INFO - step: 14200 loss: 16.9983 memory: 44.58GiB(31.99%) tps: 82,573 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.6347 global_avg_mtp_loss: 14.3636 +[titan] 2025-07-09 16:44:10,825 - root - INFO - lr: 2.8833e-04 gnorm: 0.85 [ 3:09:14<19:03:24] +[titan] 2025-07-09 16:44:14,735 - root - INFO - step: 14205 loss: 17.3700 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7140 global_avg_mtp_loss: 14.6560 +[titan] 2025-07-09 16:44:14,735 - root - INFO - lr: 2.8832e-04 gnorm: 0.82 [ 3:09:18<19:03:20] +[titan] 2025-07-09 16:44:18,648 - root - INFO - step: 14210 loss: 17.3576 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.7361 global_avg_mtp_loss: 14.6215 +[titan] 2025-07-09 16:44:18,648 - root - INFO - lr: 2.8831e-04 gnorm: 0.91 [ 3:09:22<19:03:15] +[titan] 2025-07-09 16:44:22,550 - root - INFO - step: 14215 loss: 17.2551 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.7085 global_avg_mtp_loss: 14.5466 +[titan] 2025-07-09 16:44:22,550 - root - INFO - lr: 2.8830e-04 gnorm: 0.87 [ 3:09:25<19:03:11] +[titan] 2025-07-09 16:44:26,470 - root - INFO - step: 14220 loss: 17.0530 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.3903 +[titan] 2025-07-09 16:44:26,470 - root - INFO - lr: 2.8829e-04 gnorm: 0.84 [ 3:09:29<19:03:06] +[titan] 2025-07-09 16:44:30,393 - root - INFO - step: 14225 loss: 17.0763 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6740 global_avg_mtp_loss: 14.4023 +[titan] 2025-07-09 16:44:30,393 - root - INFO - lr: 2.8828e-04 gnorm: 0.80 [ 3:09:33<19:03:02] +[titan] 2025-07-09 16:44:34,286 - root - INFO - step: 14230 loss: 17.2147 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6901 global_avg_mtp_loss: 14.5247 +[titan] 2025-07-09 16:44:34,287 - root - INFO - lr: 2.8827e-04 gnorm: 0.93 [ 3:09:37<19:02:57] +[titan] 2025-07-09 16:44:38,181 - root - INFO - step: 14235 loss: 17.3218 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.7178 global_avg_mtp_loss: 14.6040 +[titan] 2025-07-09 16:44:38,181 - root - INFO - lr: 2.8827e-04 gnorm: 0.89 [ 3:09:41<19:02:53] +[titan] 2025-07-09 16:44:42,105 - root - INFO - step: 14240 loss: 17.1053 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6713 global_avg_mtp_loss: 14.4341 +[titan] 2025-07-09 16:44:42,105 - root - INFO - lr: 2.8826e-04 gnorm: 0.89 [ 3:09:45<19:02:48] +[titan] 2025-07-09 16:44:46,012 - root - INFO - step: 14245 loss: 17.0658 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.3836 +[titan] 2025-07-09 16:44:46,012 - root - INFO - lr: 2.8825e-04 gnorm: 0.83 [ 3:09:49<19:02:44] +[titan] 2025-07-09 16:44:49,165 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:44:49,951 - root - INFO - step: 14250 loss: 17.0947 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.6776 global_avg_mtp_loss: 14.4171 +[titan] 2025-07-09 16:44:49,951 - root - INFO - lr: 2.8824e-04 gnorm: 0.87 [ 3:09:53<19:02:39] +[titan] 2025-07-09 16:44:53,849 - root - INFO - step: 14255 loss: 16.9858 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.3230 +[titan] 2025-07-09 16:44:53,849 - root - INFO - lr: 2.8823e-04 gnorm: 0.80 [ 3:09:57<19:02:35] +[titan] 2025-07-09 16:44:57,747 - root - INFO - step: 14260 loss: 16.9371 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.2949 +[titan] 2025-07-09 16:44:57,747 - root - INFO - lr: 2.8822e-04 gnorm: 0.83 [ 3:10:01<19:02:30] +[titan] 2025-07-09 16:45:01,701 - root - INFO - step: 14265 loss: 16.9619 memory: 44.58GiB(31.99%) tps: 82,872 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.6708 global_avg_mtp_loss: 14.2911 +[titan] 2025-07-09 16:45:01,702 - root - INFO - lr: 2.8821e-04 gnorm: 0.81 [ 3:10:05<19:02:26] +[titan] 2025-07-09 16:45:05,642 - root - INFO - step: 14270 loss: 17.3321 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.7320 global_avg_mtp_loss: 14.6001 +[titan] 2025-07-09 16:45:05,642 - root - INFO - lr: 2.8820e-04 gnorm: 0.91 [ 3:10:09<19:02:21] +[titan] 2025-07-09 16:45:09,556 - root - INFO - step: 14275 loss: 17.0591 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.6531 global_avg_mtp_loss: 14.4060 +[titan] 2025-07-09 16:45:09,556 - root - INFO - lr: 2.8820e-04 gnorm: 0.80 [ 3:10:12<19:02:17] +[titan] 2025-07-09 16:45:13,467 - root - INFO - step: 14280 loss: 17.3142 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.7186 global_avg_mtp_loss: 14.5956 +[titan] 2025-07-09 16:45:13,468 - root - INFO - lr: 2.8819e-04 gnorm: 0.88 [ 3:10:16<19:02:12] +[titan] 2025-07-09 16:45:17,384 - root - INFO - step: 14285 loss: 17.1564 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6682 global_avg_mtp_loss: 14.4882 +[titan] 2025-07-09 16:45:17,384 - root - INFO - lr: 2.8818e-04 gnorm: 0.90 [ 3:10:20<19:02:08] +[titan] 2025-07-09 16:45:21,316 - root - INFO - step: 14290 loss: 17.2617 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.6959 global_avg_mtp_loss: 14.5658 +[titan] 2025-07-09 16:45:21,316 - root - INFO - lr: 2.8817e-04 gnorm: 0.83 [ 3:10:24<19:02:04] +[titan] 2025-07-09 16:45:25,234 - root - INFO - step: 14295 loss: 17.2566 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6973 global_avg_mtp_loss: 14.5593 +[titan] 2025-07-09 16:45:25,235 - root - INFO - lr: 2.8816e-04 gnorm: 0.90 [ 3:10:28<19:01:59] +[titan] 2025-07-09 16:45:28,349 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:45:29,151 - root - INFO - step: 14300 loss: 17.3022 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.7164 global_avg_mtp_loss: 14.5858 +[titan] 2025-07-09 16:45:29,151 - root - INFO - lr: 2.8815e-04 gnorm: 0.90 [ 3:10:32<19:01:55] +[titan] 2025-07-09 16:45:33,053 - root - INFO - step: 14305 loss: 17.0769 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6689 global_avg_mtp_loss: 14.4080 +[titan] 2025-07-09 16:45:33,053 - root - INFO - lr: 2.8814e-04 gnorm: 0.94 [ 3:10:36<19:01:50] +[titan] 2025-07-09 16:45:36,971 - root - INFO - step: 14310 loss: 17.1215 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6674 global_avg_mtp_loss: 14.4541 +[titan] 2025-07-09 16:45:36,972 - root - INFO - lr: 2.8813e-04 gnorm: 0.91 [ 3:10:40<19:01:46] +[titan] 2025-07-09 16:45:40,871 - root - INFO - step: 14315 loss: 16.9473 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6386 global_avg_mtp_loss: 14.3087 +[titan] 2025-07-09 16:45:40,872 - root - INFO - lr: 2.8813e-04 gnorm: 0.81 [ 3:10:44<19:01:41] +[titan] 2025-07-09 16:45:44,802 - root - INFO - step: 14320 loss: 17.1160 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6769 global_avg_mtp_loss: 14.4391 +[titan] 2025-07-09 16:45:44,802 - root - INFO - lr: 2.8812e-04 gnorm: 0.87 [ 3:10:48<19:01:37] +[titan] 2025-07-09 16:45:48,711 - root - INFO - step: 14325 loss: 17.2082 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6864 global_avg_mtp_loss: 14.5218 +[titan] 2025-07-09 16:45:48,712 - root - INFO - lr: 2.8811e-04 gnorm: 0.95 [ 3:10:52<19:01:32] +[titan] 2025-07-09 16:45:52,629 - root - INFO - step: 14330 loss: 17.3678 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.7389 global_avg_mtp_loss: 14.6289 +[titan] 2025-07-09 16:45:52,629 - root - INFO - lr: 2.8810e-04 gnorm: 0.92 [ 3:10:55<19:01:28] +[titan] 2025-07-09 16:45:56,603 - root - INFO - step: 14335 loss: 16.9867 memory: 44.58GiB(31.99%) tps: 82,449 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.6468 global_avg_mtp_loss: 14.3399 +[titan] 2025-07-09 16:45:56,604 - root - INFO - lr: 2.8809e-04 gnorm: 0.86 [ 3:10:59<19:01:23] +[titan] 2025-07-09 16:45:57,544 - root - INFO - Dumping profiler traces at step 14336 +[titan] 2025-07-09 16:45:57,574 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:46:00,691 - root - INFO - step: 14340 loss: 17.0981 memory: 44.58GiB(31.99%) tps: 80,180 tflops: 276.72 mfu: 27.98% global_avg_ntp_loss: 2.6748 global_avg_mtp_loss: 14.4233 +[titan] 2025-07-09 16:46:00,691 - root - INFO - lr: 2.8808e-04 gnorm: 0.85 [ 3:11:04<19:01:20] +[titan] 2025-07-09 16:46:04,599 - root - INFO - step: 14345 loss: 17.2911 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.7110 global_avg_mtp_loss: 14.5801 +[titan] 2025-07-09 16:46:04,599 - root - INFO - lr: 2.8807e-04 gnorm: 0.83 [ 3:11:07<19:01:15] +[titan] 2025-07-09 16:46:07,739 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:46:08,529 - root - INFO - step: 14350 loss: 16.8615 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.2298 +[titan] 2025-07-09 16:46:08,529 - root - INFO - lr: 2.8806e-04 gnorm: 0.86 [ 3:11:11<19:01:11] +[titan] 2025-07-09 16:46:12,472 - root - INFO - step: 14355 loss: 17.0549 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.6690 global_avg_mtp_loss: 14.3859 +[titan] 2025-07-09 16:46:12,472 - root - INFO - lr: 2.8806e-04 gnorm: 0.88 [ 3:11:15<19:01:07] +[titan] 2025-07-09 16:46:16,363 - root - INFO - step: 14360 loss: 17.2609 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.7027 global_avg_mtp_loss: 14.5583 +[titan] 2025-07-09 16:46:16,363 - root - INFO - lr: 2.8805e-04 gnorm: 0.87 [ 3:11:19<19:01:02] +[titan] 2025-07-09 16:46:20,266 - root - INFO - step: 14365 loss: 16.9633 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6504 global_avg_mtp_loss: 14.3129 +[titan] 2025-07-09 16:46:20,266 - root - INFO - lr: 2.8804e-04 gnorm: 0.85 [ 3:11:23<19:00:58] +[titan] 2025-07-09 16:46:24,176 - root - INFO - step: 14370 loss: 17.2207 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6959 global_avg_mtp_loss: 14.5248 +[titan] 2025-07-09 16:46:24,176 - root - INFO - lr: 2.8803e-04 gnorm: 0.91 [ 3:11:27<19:00:53] +[titan] 2025-07-09 16:46:28,093 - root - INFO - step: 14375 loss: 17.2649 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.7019 global_avg_mtp_loss: 14.5629 +[titan] 2025-07-09 16:46:28,093 - root - INFO - lr: 2.8802e-04 gnorm: 0.79 [ 3:11:31<19:00:49] +[titan] 2025-07-09 16:46:32,011 - root - INFO - step: 14380 loss: 17.0358 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6621 global_avg_mtp_loss: 14.3736 +[titan] 2025-07-09 16:46:32,011 - root - INFO - lr: 2.8801e-04 gnorm: 0.80 [ 3:11:35<19:00:44] +[titan] 2025-07-09 16:46:35,924 - root - INFO - step: 14385 loss: 17.0639 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.3940 +[titan] 2025-07-09 16:46:35,924 - root - INFO - lr: 2.8800e-04 gnorm: 0.85 [ 3:11:39<19:00:40] +[titan] 2025-07-09 16:46:39,859 - root - INFO - step: 14390 loss: 17.0995 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6735 global_avg_mtp_loss: 14.4260 +[titan] 2025-07-09 16:46:39,859 - root - INFO - lr: 2.8799e-04 gnorm: 0.80 [ 3:11:43<19:00:35] +[titan] 2025-07-09 16:46:43,759 - root - INFO - step: 14395 loss: 17.2756 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.7129 global_avg_mtp_loss: 14.5627 +[titan] 2025-07-09 16:46:43,759 - root - INFO - lr: 2.8798e-04 gnorm: 0.83 [ 3:11:47<19:00:31] +[titan] 2025-07-09 16:46:46,870 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:46:47,661 - root - INFO - step: 14400 loss: 17.1983 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6893 global_avg_mtp_loss: 14.5091 +[titan] 2025-07-09 16:46:47,661 - root - INFO - lr: 2.8798e-04 gnorm: 0.85 [ 3:11:51<19:00:26] +[titan] 2025-07-09 16:46:51,581 - root - INFO - step: 14405 loss: 17.1356 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.4646 +[titan] 2025-07-09 16:46:51,581 - root - INFO - lr: 2.8797e-04 gnorm: 0.89 [ 3:11:54<19:00:22] +[titan] 2025-07-09 16:46:55,462 - root - INFO - step: 14410 loss: 16.8948 memory: 44.58GiB(31.99%) tps: 84,438 tflops: 291.41 mfu: 29.47% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 16:46:55,462 - root - INFO - lr: 2.8796e-04 gnorm: 0.86 [ 3:11:58<19:00:17] +[titan] 2025-07-09 16:46:59,413 - root - INFO - step: 14415 loss: 17.0782 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.6722 global_avg_mtp_loss: 14.4060 +[titan] 2025-07-09 16:46:59,413 - root - INFO - lr: 2.8795e-04 gnorm: 0.91 [ 3:12:02<19:00:13] +[titan] 2025-07-09 16:47:03,350 - root - INFO - step: 14420 loss: 17.3317 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.7208 global_avg_mtp_loss: 14.6109 +[titan] 2025-07-09 16:47:03,350 - root - INFO - lr: 2.8794e-04 gnorm: 0.88 [ 3:12:06<19:00:08] +[titan] 2025-07-09 16:47:07,273 - root - INFO - step: 14425 loss: 17.0629 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6775 global_avg_mtp_loss: 14.3854 +[titan] 2025-07-09 16:47:07,274 - root - INFO - lr: 2.8793e-04 gnorm: 0.94 [ 3:12:10<19:00:04] +[titan] 2025-07-09 16:47:11,183 - root - INFO - step: 14430 loss: 17.4134 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.7259 global_avg_mtp_loss: 14.6875 +[titan] 2025-07-09 16:47:11,184 - root - INFO - lr: 2.8792e-04 gnorm: 0.86 [ 3:12:14<18:59:59] +[titan] 2025-07-09 16:47:15,097 - root - INFO - step: 14435 loss: 17.2492 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.7071 global_avg_mtp_loss: 14.5422 +[titan] 2025-07-09 16:47:15,097 - root - INFO - lr: 2.8791e-04 gnorm: 0.81 [ 3:12:18<18:59:55] +[titan] 2025-07-09 16:47:19,028 - root - INFO - step: 14440 loss: 16.5670 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5808 global_avg_mtp_loss: 13.9862 +[titan] 2025-07-09 16:47:19,028 - root - INFO - lr: 2.8791e-04 gnorm: 0.84 [ 3:12:22<18:59:50] +[titan] 2025-07-09 16:47:22,932 - root - INFO - step: 14445 loss: 17.1029 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6759 global_avg_mtp_loss: 14.4270 +[titan] 2025-07-09 16:47:22,933 - root - INFO - lr: 2.8790e-04 gnorm: 0.86 [ 3:12:26<18:59:46] +[titan] 2025-07-09 16:47:26,060 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:47:26,849 - root - INFO - step: 14450 loss: 17.1442 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6911 global_avg_mtp_loss: 14.4531 +[titan] 2025-07-09 16:47:26,850 - root - INFO - lr: 2.8789e-04 gnorm: 0.88 [ 3:12:30<18:59:41] +[titan] 2025-07-09 16:47:30,746 - root - INFO - step: 14455 loss: 17.3016 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.7230 global_avg_mtp_loss: 14.5786 +[titan] 2025-07-09 16:47:30,746 - root - INFO - lr: 2.8788e-04 gnorm: 0.87 [ 3:12:34<18:59:37] +[titan] 2025-07-09 16:47:34,684 - root - INFO - step: 14460 loss: 16.8697 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6288 global_avg_mtp_loss: 14.2409 +[titan] 2025-07-09 16:47:34,684 - root - INFO - lr: 2.8787e-04 gnorm: 0.84 [ 3:12:38<18:59:33] +[titan] 2025-07-09 16:47:38,583 - root - INFO - step: 14465 loss: 17.0667 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6707 global_avg_mtp_loss: 14.3960 +[titan] 2025-07-09 16:47:38,583 - root - INFO - lr: 2.8786e-04 gnorm: 0.77 [ 3:12:41<18:59:28] +[titan] 2025-07-09 16:47:42,487 - root - INFO - step: 14470 loss: 17.1447 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6854 global_avg_mtp_loss: 14.4592 +[titan] 2025-07-09 16:47:42,487 - root - INFO - lr: 2.8785e-04 gnorm: 0.87 [ 3:12:45<18:59:23] +[titan] 2025-07-09 16:47:46,438 - root - INFO - step: 14475 loss: 17.2848 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.7023 global_avg_mtp_loss: 14.5825 +[titan] 2025-07-09 16:47:46,438 - root - INFO - lr: 2.8784e-04 gnorm: 0.86 [ 3:12:49<18:59:19] +[titan] 2025-07-09 16:47:50,367 - root - INFO - step: 14480 loss: 16.9264 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6439 global_avg_mtp_loss: 14.2824 +[titan] 2025-07-09 16:47:50,368 - root - INFO - lr: 2.8783e-04 gnorm: 0.90 [ 3:12:53<18:59:15] +[titan] 2025-07-09 16:47:54,299 - root - INFO - step: 14485 loss: 17.0390 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6633 global_avg_mtp_loss: 14.3758 +[titan] 2025-07-09 16:47:54,299 - root - INFO - lr: 2.8783e-04 gnorm: 0.82 [ 3:12:57<18:59:10] +[titan] 2025-07-09 16:47:58,219 - root - INFO - step: 14490 loss: 17.1913 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6971 global_avg_mtp_loss: 14.4943 +[titan] 2025-07-09 16:47:58,219 - root - INFO - lr: 2.8782e-04 gnorm: 0.80 [ 3:13:01<18:59:06] +[titan] 2025-07-09 16:48:02,123 - root - INFO - step: 14495 loss: 17.1638 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6950 global_avg_mtp_loss: 14.4688 +[titan] 2025-07-09 16:48:02,124 - root - INFO - lr: 2.8781e-04 gnorm: 0.92 [ 3:13:05<18:59:01] +[titan] 2025-07-09 16:48:05,273 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:48:06,065 - root - INFO - step: 14500 loss: 16.8682 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6513 global_avg_mtp_loss: 14.2169 +[titan] 2025-07-09 16:48:06,065 - root - INFO - lr: 2.8780e-04 gnorm: 1.09 [ 3:13:09<18:58:57] +[titan] 2025-07-09 16:48:09,986 - root - INFO - step: 14505 loss: 17.2152 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6881 global_avg_mtp_loss: 14.5271 +[titan] 2025-07-09 16:48:09,987 - root - INFO - lr: 2.8779e-04 gnorm: 0.89 [ 3:13:13<18:58:53] +[titan] 2025-07-09 16:48:13,926 - root - INFO - step: 14510 loss: 16.8445 memory: 44.58GiB(31.99%) tps: 83,174 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.6257 global_avg_mtp_loss: 14.2188 +[titan] 2025-07-09 16:48:13,927 - root - INFO - lr: 2.8778e-04 gnorm: 0.91 [ 3:13:17<18:58:48] +[titan] 2025-07-09 16:48:17,865 - root - INFO - step: 14515 loss: 17.3001 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.7231 global_avg_mtp_loss: 14.5771 +[titan] 2025-07-09 16:48:17,865 - root - INFO - lr: 2.8777e-04 gnorm: 0.91 [ 3:13:21<18:58:44] +[titan] 2025-07-09 16:48:21,794 - root - INFO - step: 14520 loss: 17.1714 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6830 global_avg_mtp_loss: 14.4884 +[titan] 2025-07-09 16:48:21,794 - root - INFO - lr: 2.8776e-04 gnorm: 0.97 [ 3:13:25<18:58:40] +[titan] 2025-07-09 16:48:25,698 - root - INFO - step: 14525 loss: 17.0608 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6758 global_avg_mtp_loss: 14.3850 +[titan] 2025-07-09 16:48:25,698 - root - INFO - lr: 2.8775e-04 gnorm: 0.86 [ 3:13:29<18:58:35] +[titan] 2025-07-09 16:48:29,638 - root - INFO - step: 14530 loss: 17.3271 memory: 44.58GiB(31.99%) tps: 83,174 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.7275 global_avg_mtp_loss: 14.5997 +[titan] 2025-07-09 16:48:29,638 - root - INFO - lr: 2.8775e-04 gnorm: 0.80 [ 3:13:32<18:58:31] +[titan] 2025-07-09 16:48:33,532 - root - INFO - step: 14535 loss: 17.1688 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6792 global_avg_mtp_loss: 14.4897 +[titan] 2025-07-09 16:48:33,532 - root - INFO - lr: 2.8774e-04 gnorm: 0.86 [ 3:13:36<18:58:26] +[titan] 2025-07-09 16:48:37,450 - root - INFO - step: 14540 loss: 17.0246 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6646 global_avg_mtp_loss: 14.3601 +[titan] 2025-07-09 16:48:37,451 - root - INFO - lr: 2.8773e-04 gnorm: 0.85 [ 3:13:40<18:58:22] +[titan] 2025-07-09 16:48:41,355 - root - INFO - step: 14545 loss: 17.1456 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6945 global_avg_mtp_loss: 14.4512 +[titan] 2025-07-09 16:48:41,355 - root - INFO - lr: 2.8772e-04 gnorm: 0.78 [ 3:13:44<18:58:17] +[titan] 2025-07-09 16:48:44,476 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:48:45,259 - root - INFO - step: 14550 loss: 17.2096 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.7042 global_avg_mtp_loss: 14.5053 +[titan] 2025-07-09 16:48:45,260 - root - INFO - lr: 2.8771e-04 gnorm: 0.86 [ 3:13:48<18:58:13] +[titan] 2025-07-09 16:48:49,156 - root - INFO - step: 14555 loss: 17.2530 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.7036 global_avg_mtp_loss: 14.5494 +[titan] 2025-07-09 16:48:49,156 - root - INFO - lr: 2.8770e-04 gnorm: 0.96 [ 3:13:52<18:58:08] +[titan] 2025-07-09 16:48:53,051 - root - INFO - step: 14560 loss: 17.2736 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6965 global_avg_mtp_loss: 14.5771 +[titan] 2025-07-09 16:48:53,051 - root - INFO - lr: 2.8769e-04 gnorm: 0.88 [ 3:13:56<18:58:03] +[titan] 2025-07-09 16:48:56,949 - root - INFO - step: 14565 loss: 16.9826 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6600 global_avg_mtp_loss: 14.3226 +[titan] 2025-07-09 16:48:56,950 - root - INFO - lr: 2.8768e-04 gnorm: 0.89 [ 3:14:00<18:57:59] +[titan] 2025-07-09 16:49:00,855 - root - INFO - step: 14570 loss: 17.3241 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.7235 global_avg_mtp_loss: 14.6006 +[titan] 2025-07-09 16:49:00,856 - root - INFO - lr: 2.8767e-04 gnorm: 0.85 [ 3:14:04<18:57:54] +[titan] 2025-07-09 16:49:04,762 - root - INFO - step: 14575 loss: 17.6183 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.7882 global_avg_mtp_loss: 14.8301 +[titan] 2025-07-09 16:49:04,763 - root - INFO - lr: 2.8766e-04 gnorm: 0.91 [ 3:14:08<18:57:50] +[titan] 2025-07-09 16:49:08,658 - root - INFO - step: 14580 loss: 16.9960 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6507 global_avg_mtp_loss: 14.3453 +[titan] 2025-07-09 16:49:08,659 - root - INFO - lr: 2.8766e-04 gnorm: 0.87 [ 3:14:11<18:57:45] +[titan] 2025-07-09 16:49:12,555 - root - INFO - step: 14585 loss: 17.4277 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.7412 global_avg_mtp_loss: 14.6865 +[titan] 2025-07-09 16:49:12,555 - root - INFO - lr: 2.8765e-04 gnorm: 0.80 [ 3:14:15<18:57:41] +[titan] 2025-07-09 16:49:16,456 - root - INFO - step: 14590 loss: 16.9301 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6489 global_avg_mtp_loss: 14.2812 +[titan] 2025-07-09 16:49:16,456 - root - INFO - lr: 2.8764e-04 gnorm: 0.85 [ 3:14:19<18:57:36] +[titan] 2025-07-09 16:49:20,392 - root - INFO - step: 14595 loss: 17.0721 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.4057 +[titan] 2025-07-09 16:49:20,392 - root - INFO - lr: 2.8763e-04 gnorm: 0.87 [ 3:14:23<18:57:32] +[titan] 2025-07-09 16:49:23,510 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:49:24,298 - root - INFO - step: 14600 loss: 17.1051 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6753 global_avg_mtp_loss: 14.4298 +[titan] 2025-07-09 16:49:24,299 - root - INFO - lr: 2.8762e-04 gnorm: 0.87 [ 3:14:27<18:57:27] +[titan] 2025-07-09 16:49:28,199 - root - INFO - step: 14605 loss: 17.0025 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6637 global_avg_mtp_loss: 14.3387 +[titan] 2025-07-09 16:49:28,199 - root - INFO - lr: 2.8761e-04 gnorm: 0.85 [ 3:14:31<18:57:23] +[titan] 2025-07-09 16:49:32,107 - root - INFO - step: 14610 loss: 17.3106 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7153 global_avg_mtp_loss: 14.5952 +[titan] 2025-07-09 16:49:32,107 - root - INFO - lr: 2.8760e-04 gnorm: 0.88 [ 3:14:35<18:57:18] +[titan] 2025-07-09 16:49:36,014 - root - INFO - step: 14615 loss: 17.2101 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7041 global_avg_mtp_loss: 14.5059 +[titan] 2025-07-09 16:49:36,014 - root - INFO - lr: 2.8759e-04 gnorm: 0.87 [ 3:14:39<18:57:14] +[titan] 2025-07-09 16:49:39,920 - root - INFO - step: 14620 loss: 17.0879 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6631 global_avg_mtp_loss: 14.4248 +[titan] 2025-07-09 16:49:39,921 - root - INFO - lr: 2.8758e-04 gnorm: 0.86 [ 3:14:43<18:57:09] +[titan] 2025-07-09 16:49:43,807 - root - INFO - step: 14625 loss: 17.2235 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.7027 global_avg_mtp_loss: 14.5208 +[titan] 2025-07-09 16:49:43,808 - root - INFO - lr: 2.8758e-04 gnorm: 0.88 [ 3:14:47<18:57:04] +[titan] 2025-07-09 16:49:47,752 - root - INFO - step: 14630 loss: 17.1577 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.6773 global_avg_mtp_loss: 14.4804 +[titan] 2025-07-09 16:49:47,753 - root - INFO - lr: 2.8757e-04 gnorm: 0.87 [ 3:14:51<18:57:00] +[titan] 2025-07-09 16:49:51,665 - root - INFO - step: 14635 loss: 16.9828 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.6581 global_avg_mtp_loss: 14.3247 +[titan] 2025-07-09 16:49:51,665 - root - INFO - lr: 2.8756e-04 gnorm: 0.80 [ 3:14:54<18:56:56] +[titan] 2025-07-09 16:49:55,568 - root - INFO - step: 14640 loss: 17.3239 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.7191 global_avg_mtp_loss: 14.6048 +[titan] 2025-07-09 16:49:55,568 - root - INFO - lr: 2.8755e-04 gnorm: 0.87 [ 3:14:58<18:56:51] +[titan] 2025-07-09 16:49:59,482 - root - INFO - step: 14645 loss: 17.3636 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.7354 global_avg_mtp_loss: 14.6282 +[titan] 2025-07-09 16:49:59,482 - root - INFO - lr: 2.8754e-04 gnorm: 1.05 [ 3:15:02<18:56:47] +[titan] 2025-07-09 16:50:02,597 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:50:03,395 - root - INFO - step: 14650 loss: 17.2632 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.7165 global_avg_mtp_loss: 14.5467 +[titan] 2025-07-09 16:50:03,396 - root - INFO - lr: 2.8753e-04 gnorm: 0.80 [ 3:15:06<18:56:42] +[titan] 2025-07-09 16:50:07,285 - root - INFO - step: 14655 loss: 16.9618 memory: 44.58GiB(31.99%) tps: 84,250 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.6506 global_avg_mtp_loss: 14.3113 +[titan] 2025-07-09 16:50:07,285 - root - INFO - lr: 2.8752e-04 gnorm: 0.77 [ 3:15:10<18:56:38] +[titan] 2025-07-09 16:50:11,214 - root - INFO - step: 14660 loss: 17.1564 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6844 global_avg_mtp_loss: 14.4719 +[titan] 2025-07-09 16:50:11,214 - root - INFO - lr: 2.8751e-04 gnorm: 0.80 [ 3:15:14<18:56:33] +[titan] 2025-07-09 16:50:15,126 - root - INFO - step: 14665 loss: 17.0654 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6609 global_avg_mtp_loss: 14.4045 +[titan] 2025-07-09 16:50:15,126 - root - INFO - lr: 2.8750e-04 gnorm: 0.89 [ 3:15:18<18:56:29] +[titan] 2025-07-09 16:50:19,024 - root - INFO - step: 14670 loss: 17.1269 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6674 global_avg_mtp_loss: 14.4595 +[titan] 2025-07-09 16:50:19,025 - root - INFO - lr: 2.8749e-04 gnorm: 0.92 [ 3:15:22<18:56:24] +[titan] 2025-07-09 16:50:22,939 - root - INFO - step: 14675 loss: 17.4373 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.7502 global_avg_mtp_loss: 14.6871 +[titan] 2025-07-09 16:50:22,939 - root - INFO - lr: 2.8749e-04 gnorm: 0.88 [ 3:15:26<18:56:20] +[titan] 2025-07-09 16:50:26,846 - root - INFO - step: 14680 loss: 17.3156 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7274 global_avg_mtp_loss: 14.5882 +[titan] 2025-07-09 16:50:26,847 - root - INFO - lr: 2.8748e-04 gnorm: 0.90 [ 3:15:30<18:56:15] +[titan] 2025-07-09 16:50:30,770 - root - INFO - step: 14685 loss: 17.2310 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.7066 global_avg_mtp_loss: 14.5244 +[titan] 2025-07-09 16:50:30,770 - root - INFO - lr: 2.8747e-04 gnorm: 0.89 [ 3:15:34<18:56:11] +[titan] 2025-07-09 16:50:34,661 - root - INFO - step: 14690 loss: 16.8727 memory: 44.58GiB(31.99%) tps: 84,218 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.2444 +[titan] 2025-07-09 16:50:34,661 - root - INFO - lr: 2.8746e-04 gnorm: 0.86 [ 3:15:37<18:56:06] +[titan] 2025-07-09 16:50:38,566 - root - INFO - step: 14695 loss: 17.2114 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6971 global_avg_mtp_loss: 14.5143 +[titan] 2025-07-09 16:50:38,567 - root - INFO - lr: 2.8745e-04 gnorm: 0.86 [ 3:15:41<18:56:02] +[titan] 2025-07-09 16:50:41,720 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:50:42,529 - root - INFO - step: 14700 loss: 17.1966 memory: 44.58GiB(31.99%) tps: 82,707 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.6945 global_avg_mtp_loss: 14.5022 +[titan] 2025-07-09 16:50:42,529 - root - INFO - lr: 2.8744e-04 gnorm: 0.81 [ 3:15:45<18:55:57] +[titan] 2025-07-09 16:50:46,456 - root - INFO - step: 14705 loss: 16.9714 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6401 global_avg_mtp_loss: 14.3313 +[titan] 2025-07-09 16:50:46,457 - root - INFO - lr: 2.8743e-04 gnorm: 0.79 [ 3:15:49<18:55:53] +[titan] 2025-07-09 16:50:50,365 - root - INFO - step: 14710 loss: 17.2842 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.7178 global_avg_mtp_loss: 14.5664 +[titan] 2025-07-09 16:50:50,365 - root - INFO - lr: 2.8742e-04 gnorm: 0.95 [ 3:15:53<18:55:49] +[titan] 2025-07-09 16:50:54,300 - root - INFO - step: 14715 loss: 17.0464 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6766 global_avg_mtp_loss: 14.3698 +[titan] 2025-07-09 16:50:54,301 - root - INFO - lr: 2.8741e-04 gnorm: 0.88 [ 3:15:57<18:55:44] +[titan] 2025-07-09 16:50:58,220 - root - INFO - step: 14720 loss: 17.0241 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6552 global_avg_mtp_loss: 14.3689 +[titan] 2025-07-09 16:50:58,220 - root - INFO - lr: 2.8740e-04 gnorm: 0.84 [ 3:16:01<18:55:40] +[titan] 2025-07-09 16:51:02,141 - root - INFO - step: 14725 loss: 17.2207 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.7003 global_avg_mtp_loss: 14.5204 +[titan] 2025-07-09 16:51:02,141 - root - INFO - lr: 2.8740e-04 gnorm: 0.87 [ 3:16:05<18:55:35] +[titan] 2025-07-09 16:51:06,036 - root - INFO - step: 14730 loss: 17.2411 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.6917 global_avg_mtp_loss: 14.5494 +[titan] 2025-07-09 16:51:06,036 - root - INFO - lr: 2.8739e-04 gnorm: 0.80 [ 3:16:09<18:55:31] +[titan] 2025-07-09 16:51:09,940 - root - INFO - step: 14735 loss: 17.0717 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6649 global_avg_mtp_loss: 14.4068 +[titan] 2025-07-09 16:51:09,940 - root - INFO - lr: 2.8738e-04 gnorm: 0.82 [ 3:16:13<18:55:26] +[titan] 2025-07-09 16:51:13,848 - root - INFO - step: 14740 loss: 17.2595 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.7065 global_avg_mtp_loss: 14.5530 +[titan] 2025-07-09 16:51:13,848 - root - INFO - lr: 2.8737e-04 gnorm: 0.82 [ 3:16:17<18:55:22] +[titan] 2025-07-09 16:51:17,793 - root - INFO - step: 14745 loss: 17.1414 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.6749 global_avg_mtp_loss: 14.4665 +[titan] 2025-07-09 16:51:17,793 - root - INFO - lr: 2.8736e-04 gnorm: 0.85 [ 3:16:21<18:55:17] +[titan] 2025-07-09 16:51:20,903 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:51:21,688 - root - INFO - step: 14750 loss: 17.1258 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6735 global_avg_mtp_loss: 14.4523 +[titan] 2025-07-09 16:51:21,689 - root - INFO - lr: 2.8735e-04 gnorm: 0.75 [ 3:16:25<18:55:13] +[titan] 2025-07-09 16:51:25,591 - root - INFO - step: 14755 loss: 16.9517 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.3135 +[titan] 2025-07-09 16:51:25,591 - root - INFO - lr: 2.8734e-04 gnorm: 0.84 [ 3:16:28<18:55:08] +[titan] 2025-07-09 16:51:29,502 - root - INFO - step: 14760 loss: 17.0622 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6697 global_avg_mtp_loss: 14.3925 +[titan] 2025-07-09 16:51:29,502 - root - INFO - lr: 2.8733e-04 gnorm: 0.85 [ 3:16:32<18:55:04] +[titan] 2025-07-09 16:51:33,406 - root - INFO - step: 14765 loss: 17.4134 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7350 global_avg_mtp_loss: 14.6785 +[titan] 2025-07-09 16:51:33,407 - root - INFO - lr: 2.8732e-04 gnorm: 0.85 [ 3:16:36<18:54:59] +[titan] 2025-07-09 16:51:37,371 - root - INFO - step: 14770 loss: 16.9061 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.6355 global_avg_mtp_loss: 14.2707 +[titan] 2025-07-09 16:51:37,371 - root - INFO - lr: 2.8731e-04 gnorm: 0.83 [ 3:16:40<18:54:55] +[titan] 2025-07-09 16:51:41,330 - root - INFO - step: 14775 loss: 17.1133 memory: 44.58GiB(31.99%) tps: 82,768 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.6844 global_avg_mtp_loss: 14.4290 +[titan] 2025-07-09 16:51:41,330 - root - INFO - lr: 2.8730e-04 gnorm: 0.80 [ 3:16:44<18:54:51] +[titan] 2025-07-09 16:51:45,229 - root - INFO - step: 14780 loss: 16.9891 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6475 global_avg_mtp_loss: 14.3417 +[titan] 2025-07-09 16:51:45,230 - root - INFO - lr: 2.8730e-04 gnorm: 0.96 [ 3:16:48<18:54:46] +[titan] 2025-07-09 16:51:49,127 - root - INFO - step: 14785 loss: 17.2196 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.7016 global_avg_mtp_loss: 14.5179 +[titan] 2025-07-09 16:51:49,127 - root - INFO - lr: 2.8729e-04 gnorm: 0.89 [ 3:16:52<18:54:42] +[titan] 2025-07-09 16:51:53,024 - root - INFO - step: 14790 loss: 17.1558 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.6870 global_avg_mtp_loss: 14.4687 +[titan] 2025-07-09 16:51:53,024 - root - INFO - lr: 2.8728e-04 gnorm: 0.96 [ 3:16:56<18:54:37] +[titan] 2025-07-09 16:51:56,941 - root - INFO - step: 14795 loss: 17.0769 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6802 global_avg_mtp_loss: 14.3966 +[titan] 2025-07-09 16:51:56,941 - root - INFO - lr: 2.8727e-04 gnorm: 0.98 [ 3:17:00<18:54:33] +[titan] 2025-07-09 16:52:00,048 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:52:00,843 - root - INFO - step: 14800 loss: 17.4154 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.7340 global_avg_mtp_loss: 14.6814 +[titan] 2025-07-09 16:52:00,843 - root - INFO - lr: 2.8726e-04 gnorm: 0.88 [ 3:17:04<18:54:28] +[titan] 2025-07-09 16:52:04,743 - root - INFO - step: 14805 loss: 17.0934 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6800 global_avg_mtp_loss: 14.4134 +[titan] 2025-07-09 16:52:04,744 - root - INFO - lr: 2.8725e-04 gnorm: 0.84 [ 3:17:08<18:54:24] +[titan] 2025-07-09 16:52:08,648 - root - INFO - step: 14810 loss: 17.2782 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.7125 global_avg_mtp_loss: 14.5657 +[titan] 2025-07-09 16:52:08,648 - root - INFO - lr: 2.8724e-04 gnorm: 0.83 [ 3:17:11<18:54:19] +[titan] 2025-07-09 16:52:12,566 - root - INFO - step: 14815 loss: 17.1055 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6806 global_avg_mtp_loss: 14.4249 +[titan] 2025-07-09 16:52:12,566 - root - INFO - lr: 2.8723e-04 gnorm: 0.87 [ 3:17:15<18:54:15] +[titan] 2025-07-09 16:52:16,486 - root - INFO - step: 14820 loss: 17.3192 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.7120 global_avg_mtp_loss: 14.6072 +[titan] 2025-07-09 16:52:16,486 - root - INFO - lr: 2.8722e-04 gnorm: 0.80 [ 3:17:19<18:54:10] +[titan] 2025-07-09 16:52:20,419 - root - INFO - step: 14825 loss: 17.2287 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.7008 global_avg_mtp_loss: 14.5279 +[titan] 2025-07-09 16:52:20,420 - root - INFO - lr: 2.8721e-04 gnorm: 0.77 [ 3:17:23<18:54:06] +[titan] 2025-07-09 16:52:24,318 - root - INFO - step: 14830 loss: 17.0887 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4131 +[titan] 2025-07-09 16:52:24,318 - root - INFO - lr: 2.8720e-04 gnorm: 0.90 [ 3:17:27<18:54:01] +[titan] 2025-07-09 16:52:28,258 - root - INFO - step: 14835 loss: 17.3199 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.7264 global_avg_mtp_loss: 14.5934 +[titan] 2025-07-09 16:52:28,258 - root - INFO - lr: 2.8720e-04 gnorm: 0.84 [ 3:17:31<18:53:57] +[titan] 2025-07-09 16:52:32,160 - root - INFO - step: 14840 loss: 17.2971 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.7185 global_avg_mtp_loss: 14.5787 +[titan] 2025-07-09 16:52:32,160 - root - INFO - lr: 2.8719e-04 gnorm: 0.83 [ 3:17:35<18:53:53] +[titan] 2025-07-09 16:52:36,125 - root - INFO - step: 14845 loss: 17.2066 memory: 44.58GiB(31.99%) tps: 82,661 tflops: 285.28 mfu: 28.84% global_avg_ntp_loss: 2.7043 global_avg_mtp_loss: 14.5023 +[titan] 2025-07-09 16:52:36,125 - root - INFO - lr: 2.8718e-04 gnorm: 0.92 [ 3:17:39<18:53:48] +[titan] 2025-07-09 16:52:38,610 - root - INFO - Dumping profiler traces at step 14848 +[titan] 2025-07-09 16:52:38,640 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:52:39,427 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:52:40,218 - root - INFO - step: 14850 loss: 17.4040 memory: 44.58GiB(31.99%) tps: 80,061 tflops: 276.30 mfu: 27.94% global_avg_ntp_loss: 2.7238 global_avg_mtp_loss: 14.6801 +[titan] 2025-07-09 16:52:40,218 - root - INFO - lr: 2.8717e-04 gnorm: 0.85 [ 3:17:43<18:53:45] +[titan] 2025-07-09 16:52:44,125 - root - INFO - step: 14855 loss: 17.3268 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.7186 global_avg_mtp_loss: 14.6082 +[titan] 2025-07-09 16:52:44,125 - root - INFO - lr: 2.8716e-04 gnorm: 0.87 [ 3:17:47<18:53:41] +[titan] 2025-07-09 16:52:48,046 - root - INFO - step: 14860 loss: 17.0604 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6577 global_avg_mtp_loss: 14.4027 +[titan] 2025-07-09 16:52:48,046 - root - INFO - lr: 2.8715e-04 gnorm: 0.80 [ 3:17:51<18:53:36] +[titan] 2025-07-09 16:52:51,981 - root - INFO - step: 14865 loss: 16.8727 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.2476 +[titan] 2025-07-09 16:52:51,981 - root - INFO - lr: 2.8714e-04 gnorm: 0.79 [ 3:17:55<18:53:32] +[titan] 2025-07-09 16:52:55,874 - root - INFO - step: 14870 loss: 17.1044 memory: 44.58GiB(31.99%) tps: 84,169 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.6678 global_avg_mtp_loss: 14.4366 +[titan] 2025-07-09 16:52:55,874 - root - INFO - lr: 2.8713e-04 gnorm: 0.81 [ 3:17:59<18:53:27] +[titan] 2025-07-09 16:52:59,822 - root - INFO - step: 14875 loss: 17.1797 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.6856 global_avg_mtp_loss: 14.4941 +[titan] 2025-07-09 16:52:59,823 - root - INFO - lr: 2.8712e-04 gnorm: 0.88 [ 3:18:03<18:53:23] +[titan] 2025-07-09 16:53:03,785 - root - INFO - step: 14880 loss: 17.0529 memory: 44.58GiB(31.99%) tps: 82,711 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.6634 global_avg_mtp_loss: 14.3895 +[titan] 2025-07-09 16:53:03,785 - root - INFO - lr: 2.8711e-04 gnorm: 0.92 [ 3:18:07<18:53:19] +[titan] 2025-07-09 16:53:07,746 - root - INFO - step: 14885 loss: 16.8981 memory: 44.58GiB(31.99%) tps: 82,717 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.6464 global_avg_mtp_loss: 14.2517 +[titan] 2025-07-09 16:53:07,747 - root - INFO - lr: 2.8710e-04 gnorm: 0.81 [ 3:18:11<18:53:15] +[titan] 2025-07-09 16:53:11,649 - root - INFO - step: 14890 loss: 17.1714 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6793 global_avg_mtp_loss: 14.4922 +[titan] 2025-07-09 16:53:11,649 - root - INFO - lr: 2.8710e-04 gnorm: 0.83 [ 3:18:14<18:53:10] +[titan] 2025-07-09 16:53:15,576 - root - INFO - step: 14895 loss: 17.1765 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6901 global_avg_mtp_loss: 14.4864 +[titan] 2025-07-09 16:53:15,576 - root - INFO - lr: 2.8709e-04 gnorm: 0.85 [ 3:18:18<18:53:06] +[titan] 2025-07-09 16:53:18,719 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:53:19,517 - root - INFO - step: 14900 loss: 17.0869 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.6643 global_avg_mtp_loss: 14.4225 +[titan] 2025-07-09 16:53:19,517 - root - INFO - lr: 2.8708e-04 gnorm: 0.86 [ 3:18:22<18:53:01] +[titan] 2025-07-09 16:53:23,431 - root - INFO - step: 14905 loss: 17.0028 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3472 +[titan] 2025-07-09 16:53:23,431 - root - INFO - lr: 2.8707e-04 gnorm: 0.82 [ 3:18:26<18:52:57] +[titan] 2025-07-09 16:53:27,345 - root - INFO - step: 14910 loss: 17.2447 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6997 global_avg_mtp_loss: 14.5451 +[titan] 2025-07-09 16:53:27,345 - root - INFO - lr: 2.8706e-04 gnorm: 0.76 [ 3:18:30<18:52:52] +[titan] 2025-07-09 16:53:31,243 - root - INFO - step: 14915 loss: 17.0901 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6653 global_avg_mtp_loss: 14.4249 +[titan] 2025-07-09 16:53:31,244 - root - INFO - lr: 2.8705e-04 gnorm: 0.82 [ 3:18:34<18:52:48] +[titan] 2025-07-09 16:53:35,137 - root - INFO - step: 14920 loss: 17.1860 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6830 global_avg_mtp_loss: 14.5030 +[titan] 2025-07-09 16:53:35,138 - root - INFO - lr: 2.8704e-04 gnorm: 0.90 [ 3:18:38<18:52:43] +[titan] 2025-07-09 16:53:39,050 - root - INFO - step: 14925 loss: 16.7379 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.1337 +[titan] 2025-07-09 16:53:39,051 - root - INFO - lr: 2.8703e-04 gnorm: 0.79 [ 3:18:42<18:52:39] +[titan] 2025-07-09 16:53:42,972 - root - INFO - step: 14930 loss: 16.9505 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3014 +[titan] 2025-07-09 16:53:42,973 - root - INFO - lr: 2.8702e-04 gnorm: 0.84 [ 3:18:46<18:52:34] +[titan] 2025-07-09 16:53:46,880 - root - INFO - step: 14935 loss: 17.2154 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7033 global_avg_mtp_loss: 14.5121 +[titan] 2025-07-09 16:53:46,880 - root - INFO - lr: 2.8701e-04 gnorm: 0.80 [ 3:18:50<18:52:30] +[titan] 2025-07-09 16:53:50,793 - root - INFO - step: 14940 loss: 17.0967 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6754 global_avg_mtp_loss: 14.4213 +[titan] 2025-07-09 16:53:50,794 - root - INFO - lr: 2.8700e-04 gnorm: 0.89 [ 3:18:54<18:52:26] +[titan] 2025-07-09 16:53:54,729 - root - INFO - step: 14945 loss: 17.1620 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.6907 global_avg_mtp_loss: 14.4713 +[titan] 2025-07-09 16:53:54,729 - root - INFO - lr: 2.8699e-04 gnorm: 0.83 [ 3:18:58<18:52:21] +[titan] 2025-07-09 16:53:57,899 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:53:58,696 - root - INFO - step: 14950 loss: 17.2412 memory: 44.58GiB(31.99%) tps: 82,617 tflops: 285.12 mfu: 28.83% global_avg_ntp_loss: 2.6982 global_avg_mtp_loss: 14.5430 +[titan] 2025-07-09 16:53:58,696 - root - INFO - lr: 2.8699e-04 gnorm: 0.79 [ 3:19:01<18:52:17] +[titan] 2025-07-09 16:54:02,609 - root - INFO - step: 14955 loss: 17.2315 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6983 global_avg_mtp_loss: 14.5332 +[titan] 2025-07-09 16:54:02,609 - root - INFO - lr: 2.8698e-04 gnorm: 0.99 [ 3:19:05<18:52:13] +[titan] 2025-07-09 16:54:06,496 - root - INFO - step: 14960 loss: 16.8973 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.2610 +[titan] 2025-07-09 16:54:06,496 - root - INFO - lr: 2.8697e-04 gnorm: 0.81 [ 3:19:09<18:52:08] +[titan] 2025-07-09 16:54:10,394 - root - INFO - step: 14965 loss: 17.1187 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6835 global_avg_mtp_loss: 14.4353 +[titan] 2025-07-09 16:54:10,394 - root - INFO - lr: 2.8696e-04 gnorm: 0.87 [ 3:19:13<18:52:03] +[titan] 2025-07-09 16:54:14,304 - root - INFO - step: 14970 loss: 17.2925 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6979 global_avg_mtp_loss: 14.5947 +[titan] 2025-07-09 16:54:14,304 - root - INFO - lr: 2.8695e-04 gnorm: 0.82 [ 3:19:17<18:51:59] +[titan] 2025-07-09 16:54:18,221 - root - INFO - step: 14975 loss: 17.1554 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6928 global_avg_mtp_loss: 14.4626 +[titan] 2025-07-09 16:54:18,221 - root - INFO - lr: 2.8694e-04 gnorm: 0.81 [ 3:19:21<18:51:55] +[titan] 2025-07-09 16:54:22,120 - root - INFO - step: 14980 loss: 17.2929 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6982 global_avg_mtp_loss: 14.5947 +[titan] 2025-07-09 16:54:22,120 - root - INFO - lr: 2.8693e-04 gnorm: 0.81 [ 3:19:25<18:51:50] +[titan] 2025-07-09 16:54:26,033 - root - INFO - step: 14985 loss: 17.0223 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6567 global_avg_mtp_loss: 14.3657 +[titan] 2025-07-09 16:54:26,034 - root - INFO - lr: 2.8692e-04 gnorm: 0.83 [ 3:19:29<18:51:46] +[titan] 2025-07-09 16:54:29,923 - root - INFO - step: 14990 loss: 16.9673 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.6485 global_avg_mtp_loss: 14.3188 +[titan] 2025-07-09 16:54:29,924 - root - INFO - lr: 2.8691e-04 gnorm: 0.81 [ 3:19:33<18:51:41] +[titan] 2025-07-09 16:54:33,827 - root - INFO - step: 14995 loss: 17.3605 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.7298 global_avg_mtp_loss: 14.6308 +[titan] 2025-07-09 16:54:33,827 - root - INFO - lr: 2.8690e-04 gnorm: 0.95 [ 3:19:37<18:51:36] +[titan] 2025-07-09 16:54:36,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:54:37,760 - root - INFO - step: 15000 loss: 17.1585 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6794 global_avg_mtp_loss: 14.4791 +[titan] 2025-07-09 16:54:37,760 - root - INFO - lr: 2.8689e-04 gnorm: 0.78 [ 3:19:41<18:51:32] +[titan] 2025-07-09 16:54:41,645 - root - INFO - step: 15005 loss: 17.3014 memory: 44.58GiB(31.99%) tps: 84,354 tflops: 291.12 mfu: 29.44% global_avg_ntp_loss: 2.7113 global_avg_mtp_loss: 14.5901 +[titan] 2025-07-09 16:54:41,645 - root - INFO - lr: 2.8688e-04 gnorm: 0.85 [ 3:19:44<18:51:28] +[titan] 2025-07-09 16:54:45,548 - root - INFO - step: 15010 loss: 16.8130 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.1859 +[titan] 2025-07-09 16:54:45,548 - root - INFO - lr: 2.8688e-04 gnorm: 0.84 [ 3:19:48<18:51:23] +[titan] 2025-07-09 16:54:49,441 - root - INFO - step: 15015 loss: 17.4609 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.7423 global_avg_mtp_loss: 14.7186 +[titan] 2025-07-09 16:54:49,441 - root - INFO - lr: 2.8687e-04 gnorm: 0.83 [ 3:19:52<18:51:18] +[titan] 2025-07-09 16:54:53,338 - root - INFO - step: 15020 loss: 17.2553 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7018 global_avg_mtp_loss: 14.5535 +[titan] 2025-07-09 16:54:53,338 - root - INFO - lr: 2.8686e-04 gnorm: 0.92 [ 3:19:56<18:51:14] +[titan] 2025-07-09 16:54:57,239 - root - INFO - step: 15025 loss: 17.1024 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.4402 +[titan] 2025-07-09 16:54:57,240 - root - INFO - lr: 2.8685e-04 gnorm: 0.88 [ 3:20:00<18:51:09] +[titan] 2025-07-09 16:55:01,126 - root - INFO - step: 15030 loss: 17.2177 memory: 44.58GiB(31.99%) tps: 84,324 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 2.6903 global_avg_mtp_loss: 14.5273 +[titan] 2025-07-09 16:55:01,126 - root - INFO - lr: 2.8684e-04 gnorm: 0.92 [ 3:20:04<18:51:05] +[titan] 2025-07-09 16:55:05,051 - root - INFO - step: 15035 loss: 17.2030 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6850 global_avg_mtp_loss: 14.5180 +[titan] 2025-07-09 16:55:05,051 - root - INFO - lr: 2.8683e-04 gnorm: 0.83 [ 3:20:08<18:51:00] +[titan] 2025-07-09 16:55:08,984 - root - INFO - step: 15040 loss: 17.1151 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.6780 global_avg_mtp_loss: 14.4371 +[titan] 2025-07-09 16:55:08,984 - root - INFO - lr: 2.8682e-04 gnorm: 0.91 [ 3:20:12<18:50:56] +[titan] 2025-07-09 16:55:12,921 - root - INFO - step: 15045 loss: 17.3036 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.7238 global_avg_mtp_loss: 14.5798 +[titan] 2025-07-09 16:55:12,922 - root - INFO - lr: 2.8681e-04 gnorm: 0.87 [ 3:20:16<18:50:52] +[titan] 2025-07-09 16:55:16,036 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:55:16,825 - root - INFO - step: 15050 loss: 16.8718 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6349 global_avg_mtp_loss: 14.2368 +[titan] 2025-07-09 16:55:16,825 - root - INFO - lr: 2.8680e-04 gnorm: 0.83 [ 3:20:20<18:50:47] +[titan] 2025-07-09 16:55:20,731 - root - INFO - step: 15055 loss: 16.8002 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6232 global_avg_mtp_loss: 14.1770 +[titan] 2025-07-09 16:55:20,732 - root - INFO - lr: 2.8679e-04 gnorm: 0.81 [ 3:20:24<18:50:43] +[titan] 2025-07-09 16:55:24,644 - root - INFO - step: 15060 loss: 17.0076 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6431 global_avg_mtp_loss: 14.3645 +[titan] 2025-07-09 16:55:24,645 - root - INFO - lr: 2.8678e-04 gnorm: 0.84 [ 3:20:27<18:50:38] +[titan] 2025-07-09 16:55:28,549 - root - INFO - step: 15065 loss: 17.2503 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.7045 global_avg_mtp_loss: 14.5458 +[titan] 2025-07-09 16:55:28,550 - root - INFO - lr: 2.8677e-04 gnorm: 0.87 [ 3:20:31<18:50:34] +[titan] 2025-07-09 16:55:32,440 - root - INFO - step: 15070 loss: 17.0835 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6830 global_avg_mtp_loss: 14.4005 +[titan] 2025-07-09 16:55:32,441 - root - INFO - lr: 2.8676e-04 gnorm: 0.87 [ 3:20:35<18:50:29] +[titan] 2025-07-09 16:55:36,331 - root - INFO - step: 15075 loss: 17.0010 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.6500 global_avg_mtp_loss: 14.3510 +[titan] 2025-07-09 16:55:36,332 - root - INFO - lr: 2.8676e-04 gnorm: 0.85 [ 3:20:39<18:50:25] +[titan] 2025-07-09 16:55:40,222 - root - INFO - step: 15080 loss: 17.0636 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6676 global_avg_mtp_loss: 14.3960 +[titan] 2025-07-09 16:55:40,222 - root - INFO - lr: 2.8675e-04 gnorm: 0.87 [ 3:20:43<18:50:20] +[titan] 2025-07-09 16:55:44,148 - root - INFO - step: 15085 loss: 16.8851 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.2642 +[titan] 2025-07-09 16:55:44,148 - root - INFO - lr: 2.8674e-04 gnorm: 0.86 [ 3:20:47<18:50:16] +[titan] 2025-07-09 16:55:48,047 - root - INFO - step: 15090 loss: 16.8037 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6161 global_avg_mtp_loss: 14.1876 +[titan] 2025-07-09 16:55:48,048 - root - INFO - lr: 2.8673e-04 gnorm: 0.87 [ 3:20:51<18:50:11] +[titan] 2025-07-09 16:55:51,970 - root - INFO - step: 15095 loss: 17.0916 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6834 global_avg_mtp_loss: 14.4082 +[titan] 2025-07-09 16:55:51,970 - root - INFO - lr: 2.8672e-04 gnorm: 0.80 [ 3:20:55<18:50:07] +[titan] 2025-07-09 16:55:55,090 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:55:55,884 - root - INFO - step: 15100 loss: 17.1668 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6962 global_avg_mtp_loss: 14.4706 +[titan] 2025-07-09 16:55:55,885 - root - INFO - lr: 2.8671e-04 gnorm: 0.83 [ 3:20:59<18:50:02] +[titan] 2025-07-09 16:55:59,832 - root - INFO - step: 15105 loss: 17.2842 memory: 44.58GiB(31.99%) tps: 83,010 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.7106 global_avg_mtp_loss: 14.5736 +[titan] 2025-07-09 16:55:59,833 - root - INFO - lr: 2.8670e-04 gnorm: 0.87 [ 3:21:03<18:49:58] +[titan] 2025-07-09 16:56:03,766 - root - INFO - step: 15110 loss: 17.0191 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.6558 global_avg_mtp_loss: 14.3634 +[titan] 2025-07-09 16:56:03,766 - root - INFO - lr: 2.8669e-04 gnorm: 0.84 [ 3:21:07<18:49:54] +[titan] 2025-07-09 16:56:07,683 - root - INFO - step: 15115 loss: 17.1252 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6748 global_avg_mtp_loss: 14.4505 +[titan] 2025-07-09 16:56:07,683 - root - INFO - lr: 2.8668e-04 gnorm: 0.80 [ 3:21:10<18:49:49] +[titan] 2025-07-09 16:56:11,584 - root - INFO - step: 15120 loss: 17.4359 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.7415 global_avg_mtp_loss: 14.6944 +[titan] 2025-07-09 16:56:11,585 - root - INFO - lr: 2.8667e-04 gnorm: 0.83 [ 3:21:14<18:49:45] +[titan] 2025-07-09 16:56:15,492 - root - INFO - step: 15125 loss: 17.2115 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6998 global_avg_mtp_loss: 14.5117 +[titan] 2025-07-09 16:56:15,492 - root - INFO - lr: 2.8666e-04 gnorm: 0.91 [ 3:21:18<18:49:40] +[titan] 2025-07-09 16:56:19,395 - root - INFO - step: 15130 loss: 17.2545 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.7122 global_avg_mtp_loss: 14.5423 +[titan] 2025-07-09 16:56:19,396 - root - INFO - lr: 2.8665e-04 gnorm: 0.88 [ 3:21:22<18:49:36] +[titan] 2025-07-09 16:56:23,326 - root - INFO - step: 15135 loss: 17.0247 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.6659 global_avg_mtp_loss: 14.3588 +[titan] 2025-07-09 16:56:23,327 - root - INFO - lr: 2.8664e-04 gnorm: 0.86 [ 3:21:26<18:49:32] +[titan] 2025-07-09 16:56:27,247 - root - INFO - step: 15140 loss: 17.2309 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.5320 +[titan] 2025-07-09 16:56:27,248 - root - INFO - lr: 2.8663e-04 gnorm: 0.84 [ 3:21:30<18:49:27] +[titan] 2025-07-09 16:56:31,147 - root - INFO - step: 15145 loss: 17.0504 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6640 global_avg_mtp_loss: 14.3863 +[titan] 2025-07-09 16:56:31,147 - root - INFO - lr: 2.8663e-04 gnorm: 0.81 [ 3:21:34<18:49:23] +[titan] 2025-07-09 16:56:34,253 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:56:35,049 - root - INFO - step: 15150 loss: 17.2656 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.7048 global_avg_mtp_loss: 14.5607 +[titan] 2025-07-09 16:56:35,050 - root - INFO - lr: 2.8662e-04 gnorm: 0.81 [ 3:21:38<18:49:18] +[titan] 2025-07-09 16:56:38,955 - root - INFO - step: 15155 loss: 17.3041 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.7180 global_avg_mtp_loss: 14.5861 +[titan] 2025-07-09 16:56:38,955 - root - INFO - lr: 2.8661e-04 gnorm: 0.81 [ 3:21:42<18:49:14] +[titan] 2025-07-09 16:56:42,918 - root - INFO - step: 15160 loss: 17.1729 memory: 44.58GiB(31.99%) tps: 82,699 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.6846 global_avg_mtp_loss: 14.4883 +[titan] 2025-07-09 16:56:42,918 - root - INFO - lr: 2.8660e-04 gnorm: 0.83 [ 3:21:46<18:49:09] +[titan] 2025-07-09 16:56:46,837 - root - INFO - step: 15165 loss: 17.0418 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.3720 +[titan] 2025-07-09 16:56:46,837 - root - INFO - lr: 2.8659e-04 gnorm: 0.83 [ 3:21:50<18:49:05] +[titan] 2025-07-09 16:56:50,737 - root - INFO - step: 15170 loss: 17.5093 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.7590 global_avg_mtp_loss: 14.7503 +[titan] 2025-07-09 16:56:50,737 - root - INFO - lr: 2.8658e-04 gnorm: 0.79 [ 3:21:54<18:49:01] +[titan] 2025-07-09 16:56:54,643 - root - INFO - step: 15175 loss: 17.3619 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.7079 global_avg_mtp_loss: 14.6540 +[titan] 2025-07-09 16:56:54,643 - root - INFO - lr: 2.8657e-04 gnorm: 0.86 [ 3:21:57<18:48:56] +[titan] 2025-07-09 16:56:58,564 - root - INFO - step: 15180 loss: 17.3506 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.7287 global_avg_mtp_loss: 14.6219 +[titan] 2025-07-09 16:56:58,565 - root - INFO - lr: 2.8656e-04 gnorm: 0.85 [ 3:22:01<18:48:52] +[titan] 2025-07-09 16:57:02,474 - root - INFO - step: 15185 loss: 17.2440 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.7007 global_avg_mtp_loss: 14.5432 +[titan] 2025-07-09 16:57:02,474 - root - INFO - lr: 2.8655e-04 gnorm: 0.81 [ 3:22:05<18:48:47] +[titan] 2025-07-09 16:57:06,394 - root - INFO - step: 15190 loss: 17.0888 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6666 global_avg_mtp_loss: 14.4222 +[titan] 2025-07-09 16:57:06,394 - root - INFO - lr: 2.8654e-04 gnorm: 0.81 [ 3:22:09<18:48:43] +[titan] 2025-07-09 16:57:10,303 - root - INFO - step: 15195 loss: 16.9908 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6507 global_avg_mtp_loss: 14.3401 +[titan] 2025-07-09 16:57:10,304 - root - INFO - lr: 2.8653e-04 gnorm: 0.83 [ 3:22:13<18:48:38] +[titan] 2025-07-09 16:57:13,449 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:57:14,236 - root - INFO - step: 15200 loss: 17.1539 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6890 global_avg_mtp_loss: 14.4649 +[titan] 2025-07-09 16:57:14,236 - root - INFO - lr: 2.8652e-04 gnorm: 0.85 [ 3:22:17<18:48:34] +[titan] 2025-07-09 16:57:18,153 - root - INFO - step: 15205 loss: 17.3012 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.7117 global_avg_mtp_loss: 14.5895 +[titan] 2025-07-09 16:57:18,153 - root - INFO - lr: 2.8651e-04 gnorm: 0.83 [ 3:22:21<18:48:30] +[titan] 2025-07-09 16:57:22,082 - root - INFO - step: 15210 loss: 17.1601 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.6836 global_avg_mtp_loss: 14.4765 +[titan] 2025-07-09 16:57:22,082 - root - INFO - lr: 2.8650e-04 gnorm: 0.97 [ 3:22:25<18:48:25] +[titan] 2025-07-09 16:57:26,005 - root - INFO - step: 15215 loss: 17.3825 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.7268 global_avg_mtp_loss: 14.6556 +[titan] 2025-07-09 16:57:26,005 - root - INFO - lr: 2.8649e-04 gnorm: 0.92 [ 3:22:29<18:48:21] +[titan] 2025-07-09 16:57:29,948 - root - INFO - step: 15220 loss: 17.1604 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6874 global_avg_mtp_loss: 14.4730 +[titan] 2025-07-09 16:57:29,949 - root - INFO - lr: 2.8649e-04 gnorm: 0.85 [ 3:22:33<18:48:17] +[titan] 2025-07-09 16:57:33,858 - root - INFO - step: 15225 loss: 17.0500 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6676 global_avg_mtp_loss: 14.3824 +[titan] 2025-07-09 16:57:33,859 - root - INFO - lr: 2.8648e-04 gnorm: 0.80 [ 3:22:37<18:48:12] +[titan] 2025-07-09 16:57:37,792 - root - INFO - step: 15230 loss: 17.0071 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.6688 global_avg_mtp_loss: 14.3383 +[titan] 2025-07-09 16:57:37,792 - root - INFO - lr: 2.8647e-04 gnorm: 0.88 [ 3:22:41<18:48:08] +[titan] 2025-07-09 16:57:41,710 - root - INFO - step: 15235 loss: 16.8257 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.1966 +[titan] 2025-07-09 16:57:41,710 - root - INFO - lr: 2.8646e-04 gnorm: 0.81 [ 3:22:44<18:48:03] +[titan] 2025-07-09 16:57:45,663 - root - INFO - step: 15240 loss: 17.0049 memory: 44.58GiB(31.99%) tps: 82,905 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3557 +[titan] 2025-07-09 16:57:45,663 - root - INFO - lr: 2.8645e-04 gnorm: 0.83 [ 3:22:48<18:47:59] +[titan] 2025-07-09 16:57:49,578 - root - INFO - step: 15245 loss: 17.2538 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.7139 global_avg_mtp_loss: 14.5399 +[titan] 2025-07-09 16:57:49,578 - root - INFO - lr: 2.8644e-04 gnorm: 0.87 [ 3:22:52<18:47:55] +[titan] 2025-07-09 16:57:52,686 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:57:53,472 - root - INFO - step: 15250 loss: 17.0572 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6562 global_avg_mtp_loss: 14.4009 +[titan] 2025-07-09 16:57:53,472 - root - INFO - lr: 2.8643e-04 gnorm: 0.84 [ 3:22:56<18:47:50] +[titan] 2025-07-09 16:57:57,378 - root - INFO - step: 15255 loss: 16.9321 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.2957 +[titan] 2025-07-09 16:57:57,378 - root - INFO - lr: 2.8642e-04 gnorm: 0.95 [ 3:23:00<18:47:46] +[titan] 2025-07-09 16:58:01,288 - root - INFO - step: 15260 loss: 17.3426 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.7293 global_avg_mtp_loss: 14.6133 +[titan] 2025-07-09 16:58:01,288 - root - INFO - lr: 2.8641e-04 gnorm: 0.77 [ 3:23:04<18:47:41] +[titan] 2025-07-09 16:58:05,212 - root - INFO - step: 15265 loss: 17.2917 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.7190 global_avg_mtp_loss: 14.5727 +[titan] 2025-07-09 16:58:05,213 - root - INFO - lr: 2.8640e-04 gnorm: 0.78 [ 3:23:08<18:47:37] +[titan] 2025-07-09 16:58:09,113 - root - INFO - step: 15270 loss: 16.8550 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6323 global_avg_mtp_loss: 14.2227 +[titan] 2025-07-09 16:58:09,113 - root - INFO - lr: 2.8639e-04 gnorm: 0.78 [ 3:23:12<18:47:32] +[titan] 2025-07-09 16:58:13,025 - root - INFO - step: 15275 loss: 17.1129 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6507 global_avg_mtp_loss: 14.4622 +[titan] 2025-07-09 16:58:13,025 - root - INFO - lr: 2.8638e-04 gnorm: 0.83 [ 3:23:16<18:47:28] +[titan] 2025-07-09 16:58:16,926 - root - INFO - step: 15280 loss: 16.9929 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.3449 +[titan] 2025-07-09 16:58:16,926 - root - INFO - lr: 2.8637e-04 gnorm: 0.88 [ 3:23:20<18:47:24] +[titan] 2025-07-09 16:58:20,839 - root - INFO - step: 15285 loss: 16.8858 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.6442 global_avg_mtp_loss: 14.2416 +[titan] 2025-07-09 16:58:20,840 - root - INFO - lr: 2.8636e-04 gnorm: 0.89 [ 3:23:24<18:47:19] +[titan] 2025-07-09 16:58:24,766 - root - INFO - step: 15290 loss: 17.4330 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.7246 global_avg_mtp_loss: 14.7085 +[titan] 2025-07-09 16:58:24,766 - root - INFO - lr: 2.8635e-04 gnorm: 0.86 [ 3:23:28<18:47:15] +[titan] 2025-07-09 16:58:28,720 - root - INFO - step: 15295 loss: 16.8346 memory: 44.58GiB(31.99%) tps: 82,869 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.6272 global_avg_mtp_loss: 14.2074 +[titan] 2025-07-09 16:58:28,721 - root - INFO - lr: 2.8634e-04 gnorm: 0.79 [ 3:23:31<18:47:11] +[titan] 2025-07-09 16:58:31,876 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:58:32,662 - root - INFO - step: 15300 loss: 17.4790 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.7540 global_avg_mtp_loss: 14.7249 +[titan] 2025-07-09 16:58:32,663 - root - INFO - lr: 2.8634e-04 gnorm: 0.83 [ 3:23:35<18:47:06] +[titan] 2025-07-09 16:58:36,559 - root - INFO - step: 15305 loss: 17.1798 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.6840 global_avg_mtp_loss: 14.4958 +[titan] 2025-07-09 16:58:36,559 - root - INFO - lr: 2.8633e-04 gnorm: 0.83 [ 3:23:39<18:47:02] +[titan] 2025-07-09 16:58:40,465 - root - INFO - step: 15310 loss: 17.1124 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.6777 global_avg_mtp_loss: 14.4347 +[titan] 2025-07-09 16:58:40,465 - root - INFO - lr: 2.8632e-04 gnorm: 0.89 [ 3:23:43<18:46:57] +[titan] 2025-07-09 16:58:44,364 - root - INFO - step: 15315 loss: 17.2432 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.12 mfu: 29.34% global_avg_ntp_loss: 2.7204 global_avg_mtp_loss: 14.5228 +[titan] 2025-07-09 16:58:44,364 - root - INFO - lr: 2.8631e-04 gnorm: 0.87 [ 3:23:47<18:46:53] +[titan] 2025-07-09 16:58:48,289 - root - INFO - step: 15320 loss: 17.1806 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6827 global_avg_mtp_loss: 14.4980 +[titan] 2025-07-09 16:58:48,290 - root - INFO - lr: 2.8630e-04 gnorm: 0.86 [ 3:23:51<18:46:48] +[titan] 2025-07-09 16:58:52,199 - root - INFO - step: 15325 loss: 17.2366 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6942 global_avg_mtp_loss: 14.5424 +[titan] 2025-07-09 16:58:52,200 - root - INFO - lr: 2.8629e-04 gnorm: 0.86 [ 3:23:55<18:46:44] +[titan] 2025-07-09 16:58:56,090 - root - INFO - step: 15330 loss: 17.4697 memory: 44.58GiB(31.99%) tps: 84,243 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.7415 global_avg_mtp_loss: 14.7282 +[titan] 2025-07-09 16:58:56,090 - root - INFO - lr: 2.8628e-04 gnorm: 0.90 [ 3:23:59<18:46:39] +[titan] 2025-07-09 16:59:00,011 - root - INFO - step: 15335 loss: 17.2115 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6949 global_avg_mtp_loss: 14.5166 +[titan] 2025-07-09 16:59:00,012 - root - INFO - lr: 2.8627e-04 gnorm: 0.84 [ 3:24:03<18:46:35] +[titan] 2025-07-09 16:59:03,908 - root - INFO - step: 15340 loss: 17.2546 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6995 global_avg_mtp_loss: 14.5552 +[titan] 2025-07-09 16:59:03,909 - root - INFO - lr: 2.8626e-04 gnorm: 0.89 [ 3:24:07<18:46:30] +[titan] 2025-07-09 16:59:07,807 - root - INFO - step: 15345 loss: 17.0255 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6632 global_avg_mtp_loss: 14.3623 +[titan] 2025-07-09 16:59:07,808 - root - INFO - lr: 2.8625e-04 gnorm: 0.91 [ 3:24:11<18:46:26] +[titan] 2025-07-09 16:59:10,940 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:59:11,730 - root - INFO - step: 15350 loss: 17.2028 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6762 global_avg_mtp_loss: 14.5266 +[titan] 2025-07-09 16:59:11,730 - root - INFO - lr: 2.8624e-04 gnorm: 0.81 [ 3:24:14<18:46:22] +[titan] 2025-07-09 16:59:15,668 - root - INFO - step: 15355 loss: 17.2170 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.6963 global_avg_mtp_loss: 14.5207 +[titan] 2025-07-09 16:59:15,668 - root - INFO - lr: 2.8623e-04 gnorm: 0.82 [ 3:24:18<18:46:17] +[titan] 2025-07-09 16:59:19,645 - root - INFO - step: 15360 loss: 16.8902 memory: 44.58GiB(31.99%) tps: 82,387 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.6253 global_avg_mtp_loss: 14.2649 +[titan] 2025-07-09 16:59:19,646 - root - INFO - lr: 2.8622e-04 gnorm: 0.80 [ 3:24:22<18:46:13] +[titan] 2025-07-09 16:59:19,798 - root - INFO - Dumping profiler traces at step 15360 +[titan] 2025-07-09 16:59:19,830 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 16:59:23,731 - root - INFO - step: 15365 loss: 17.0229 memory: 44.58GiB(31.99%) tps: 80,218 tflops: 276.85 mfu: 27.99% global_avg_ntp_loss: 2.6591 global_avg_mtp_loss: 14.3638 +[titan] 2025-07-09 16:59:23,731 - root - INFO - lr: 2.8621e-04 gnorm: 0.84 [ 3:24:26<18:46:10] +[titan] 2025-07-09 16:59:27,645 - root - INFO - step: 15370 loss: 17.2065 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6979 global_avg_mtp_loss: 14.5086 +[titan] 2025-07-09 16:59:27,646 - root - INFO - lr: 2.8620e-04 gnorm: 0.92 [ 3:24:30<18:46:05] +[titan] 2025-07-09 16:59:31,532 - root - INFO - step: 15375 loss: 16.9895 memory: 44.58GiB(31.99%) tps: 84,328 tflops: 291.03 mfu: 29.43% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3339 +[titan] 2025-07-09 16:59:31,532 - root - INFO - lr: 2.8619e-04 gnorm: 0.83 [ 3:24:34<18:46:01] +[titan] 2025-07-09 16:59:35,447 - root - INFO - step: 15380 loss: 17.2435 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.7063 global_avg_mtp_loss: 14.5372 +[titan] 2025-07-09 16:59:35,447 - root - INFO - lr: 2.8618e-04 gnorm: 0.78 [ 3:24:38<18:45:56] +[titan] 2025-07-09 16:59:39,336 - root - INFO - step: 15385 loss: 17.0137 memory: 44.58GiB(31.99%) tps: 84,258 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.6579 global_avg_mtp_loss: 14.3557 +[titan] 2025-07-09 16:59:39,337 - root - INFO - lr: 2.8618e-04 gnorm: 0.89 [ 3:24:42<18:45:52] +[titan] 2025-07-09 16:59:43,258 - root - INFO - step: 15390 loss: 17.1716 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4822 +[titan] 2025-07-09 16:59:43,259 - root - INFO - lr: 2.8617e-04 gnorm: 0.79 [ 3:24:46<18:45:47] +[titan] 2025-07-09 16:59:47,211 - root - INFO - step: 15395 loss: 17.2267 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.7011 global_avg_mtp_loss: 14.5255 +[titan] 2025-07-09 16:59:47,211 - root - INFO - lr: 2.8616e-04 gnorm: 0.81 [ 3:24:50<18:45:43] +[titan] 2025-07-09 16:59:50,330 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 16:59:51,117 - root - INFO - step: 15400 loss: 16.8742 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.2411 +[titan] 2025-07-09 16:59:51,117 - root - INFO - lr: 2.8615e-04 gnorm: 0.87 [ 3:24:54<18:45:39] +[titan] 2025-07-09 16:59:55,035 - root - INFO - step: 15405 loss: 17.3182 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.7182 global_avg_mtp_loss: 14.5999 +[titan] 2025-07-09 16:59:55,036 - root - INFO - lr: 2.8614e-04 gnorm: 0.88 [ 3:24:58<18:45:34] +[titan] 2025-07-09 16:59:58,962 - root - INFO - step: 15410 loss: 17.1737 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6863 global_avg_mtp_loss: 14.4874 +[titan] 2025-07-09 16:59:58,963 - root - INFO - lr: 2.8613e-04 gnorm: 0.85 [ 3:25:02<18:45:30] +[titan] 2025-07-09 17:00:02,872 - root - INFO - step: 15415 loss: 17.1854 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.5009 +[titan] 2025-07-09 17:00:02,873 - root - INFO - lr: 2.8612e-04 gnorm: 0.93 [ 3:25:06<18:45:26] +[titan] 2025-07-09 17:00:06,792 - root - INFO - step: 15420 loss: 17.3329 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.7286 global_avg_mtp_loss: 14.6043 +[titan] 2025-07-09 17:00:06,792 - root - INFO - lr: 2.8611e-04 gnorm: 1.00 [ 3:25:10<18:45:21] +[titan] 2025-07-09 17:00:10,719 - root - INFO - step: 15425 loss: 17.1159 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6726 global_avg_mtp_loss: 14.4433 +[titan] 2025-07-09 17:00:10,719 - root - INFO - lr: 2.8610e-04 gnorm: 0.88 [ 3:25:13<18:45:17] +[titan] 2025-07-09 17:00:14,676 - root - INFO - step: 15430 loss: 17.0413 memory: 44.58GiB(31.99%) tps: 82,817 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.6607 global_avg_mtp_loss: 14.3806 +[titan] 2025-07-09 17:00:14,676 - root - INFO - lr: 2.8609e-04 gnorm: 0.83 [ 3:25:17<18:45:13] +[titan] 2025-07-09 17:00:18,583 - root - INFO - step: 15435 loss: 17.3094 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.7162 global_avg_mtp_loss: 14.5931 +[titan] 2025-07-09 17:00:18,584 - root - INFO - lr: 2.8608e-04 gnorm: 0.79 [ 3:25:21<18:45:08] +[titan] 2025-07-09 17:00:22,507 - root - INFO - step: 15440 loss: 16.7678 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.1609 +[titan] 2025-07-09 17:00:22,507 - root - INFO - lr: 2.8607e-04 gnorm: 0.87 [ 3:25:25<18:45:04] +[titan] 2025-07-09 17:00:26,413 - root - INFO - step: 15445 loss: 17.1444 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6914 global_avg_mtp_loss: 14.4530 +[titan] 2025-07-09 17:00:26,413 - root - INFO - lr: 2.8606e-04 gnorm: 0.90 [ 3:25:29<18:44:59] +[titan] 2025-07-09 17:00:29,544 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:00:30,327 - root - INFO - step: 15450 loss: 17.0705 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6678 global_avg_mtp_loss: 14.4027 +[titan] 2025-07-09 17:00:30,328 - root - INFO - lr: 2.8605e-04 gnorm: 0.82 [ 3:25:33<18:44:55] +[titan] 2025-07-09 17:00:34,214 - root - INFO - step: 15455 loss: 16.9821 memory: 44.58GiB(31.99%) tps: 84,325 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3362 +[titan] 2025-07-09 17:00:34,214 - root - INFO - lr: 2.8604e-04 gnorm: 0.81 [ 3:25:37<18:44:50] +[titan] 2025-07-09 17:00:38,129 - root - INFO - step: 15460 loss: 17.0779 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6577 global_avg_mtp_loss: 14.4202 +[titan] 2025-07-09 17:00:38,129 - root - INFO - lr: 2.8603e-04 gnorm: 0.84 [ 3:25:41<18:44:46] +[titan] 2025-07-09 17:00:42,023 - root - INFO - step: 15465 loss: 17.0774 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6706 global_avg_mtp_loss: 14.4068 +[titan] 2025-07-09 17:00:42,023 - root - INFO - lr: 2.8602e-04 gnorm: 0.87 [ 3:25:45<18:44:41] +[titan] 2025-07-09 17:00:45,911 - root - INFO - step: 15470 loss: 17.0085 memory: 44.58GiB(31.99%) tps: 84,278 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.6496 global_avg_mtp_loss: 14.3589 +[titan] 2025-07-09 17:00:45,911 - root - INFO - lr: 2.8601e-04 gnorm: 0.83 [ 3:25:49<18:44:37] +[titan] 2025-07-09 17:00:49,845 - root - INFO - step: 15475 loss: 17.2766 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.7302 global_avg_mtp_loss: 14.5464 +[titan] 2025-07-09 17:00:49,845 - root - INFO - lr: 2.8601e-04 gnorm: 0.87 [ 3:25:53<18:44:33] +[titan] 2025-07-09 17:00:53,763 - root - INFO - step: 15480 loss: 17.1279 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4574 +[titan] 2025-07-09 17:00:53,763 - root - INFO - lr: 2.8600e-04 gnorm: 0.78 [ 3:25:57<18:44:28] +[titan] 2025-07-09 17:00:57,674 - root - INFO - step: 15485 loss: 17.2357 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6944 global_avg_mtp_loss: 14.5413 +[titan] 2025-07-09 17:00:57,674 - root - INFO - lr: 2.8599e-04 gnorm: 0.92 [ 3:26:00<18:44:24] +[titan] 2025-07-09 17:01:01,585 - root - INFO - step: 15490 loss: 17.2878 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.7132 global_avg_mtp_loss: 14.5746 +[titan] 2025-07-09 17:01:01,586 - root - INFO - lr: 2.8598e-04 gnorm: 0.90 [ 3:26:04<18:44:19] +[titan] 2025-07-09 17:01:05,508 - root - INFO - step: 15495 loss: 16.7742 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.1582 +[titan] 2025-07-09 17:01:05,508 - root - INFO - lr: 2.8597e-04 gnorm: 0.81 [ 3:26:08<18:44:15] +[titan] 2025-07-09 17:01:08,646 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:01:09,438 - root - INFO - step: 15500 loss: 16.9497 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6566 global_avg_mtp_loss: 14.2931 +[titan] 2025-07-09 17:01:09,439 - root - INFO - lr: 2.8596e-04 gnorm: 0.92 [ 3:26:12<18:44:11] +[titan] 2025-07-09 17:01:13,410 - root - INFO - step: 15505 loss: 17.2159 memory: 44.58GiB(31.99%) tps: 82,520 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.6987 global_avg_mtp_loss: 14.5171 +[titan] 2025-07-09 17:01:13,410 - root - INFO - lr: 2.8595e-04 gnorm: 0.88 [ 3:26:16<18:44:06] +[titan] 2025-07-09 17:01:17,301 - root - INFO - step: 15510 loss: 17.3142 memory: 44.58GiB(31.99%) tps: 84,207 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.7247 global_avg_mtp_loss: 14.5895 +[titan] 2025-07-09 17:01:17,302 - root - INFO - lr: 2.8594e-04 gnorm: 0.88 [ 3:26:20<18:44:02] +[titan] 2025-07-09 17:01:21,194 - root - INFO - step: 15515 loss: 16.7103 memory: 44.58GiB(31.99%) tps: 84,185 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.6033 global_avg_mtp_loss: 14.1070 +[titan] 2025-07-09 17:01:21,194 - root - INFO - lr: 2.8593e-04 gnorm: 0.89 [ 3:26:24<18:43:57] +[titan] 2025-07-09 17:01:25,145 - root - INFO - step: 15520 loss: 17.3220 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.7183 global_avg_mtp_loss: 14.6037 +[titan] 2025-07-09 17:01:25,145 - root - INFO - lr: 2.8592e-04 gnorm: 0.82 [ 3:26:28<18:43:53] +[titan] 2025-07-09 17:01:29,070 - root - INFO - step: 15525 loss: 16.9340 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.2873 +[titan] 2025-07-09 17:01:29,070 - root - INFO - lr: 2.8591e-04 gnorm: 0.79 [ 3:26:32<18:43:49] +[titan] 2025-07-09 17:01:33,010 - root - INFO - step: 15530 loss: 16.6720 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.0827 +[titan] 2025-07-09 17:01:33,010 - root - INFO - lr: 2.8590e-04 gnorm: 0.89 [ 3:26:36<18:43:45] +[titan] 2025-07-09 17:01:36,944 - root - INFO - step: 15535 loss: 17.1004 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6720 global_avg_mtp_loss: 14.4284 +[titan] 2025-07-09 17:01:36,944 - root - INFO - lr: 2.8589e-04 gnorm: 0.85 [ 3:26:40<18:43:40] +[titan] 2025-07-09 17:01:40,843 - root - INFO - step: 15540 loss: 16.9782 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3292 +[titan] 2025-07-09 17:01:40,844 - root - INFO - lr: 2.8588e-04 gnorm: 0.80 [ 3:26:44<18:43:36] +[titan] 2025-07-09 17:01:44,773 - root - INFO - step: 15545 loss: 17.0926 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6704 global_avg_mtp_loss: 14.4222 +[titan] 2025-07-09 17:01:44,773 - root - INFO - lr: 2.8587e-04 gnorm: 0.76 [ 3:26:48<18:43:31] +[titan] 2025-07-09 17:01:47,901 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:01:48,689 - root - INFO - step: 15550 loss: 17.2293 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6981 global_avg_mtp_loss: 14.5312 +[titan] 2025-07-09 17:01:48,689 - root - INFO - lr: 2.8586e-04 gnorm: 0.87 [ 3:26:51<18:43:27] +[titan] 2025-07-09 17:01:52,617 - root - INFO - step: 15555 loss: 16.9646 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6656 global_avg_mtp_loss: 14.2990 +[titan] 2025-07-09 17:01:52,617 - root - INFO - lr: 2.8585e-04 gnorm: 0.89 [ 3:26:55<18:43:23] +[titan] 2025-07-09 17:01:56,528 - root - INFO - step: 15560 loss: 17.0947 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6708 global_avg_mtp_loss: 14.4239 +[titan] 2025-07-09 17:01:56,528 - root - INFO - lr: 2.8584e-04 gnorm: 0.84 [ 3:26:59<18:43:18] +[titan] 2025-07-09 17:02:00,425 - root - INFO - step: 15565 loss: 16.9137 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6490 global_avg_mtp_loss: 14.2647 +[titan] 2025-07-09 17:02:00,425 - root - INFO - lr: 2.8583e-04 gnorm: 0.90 [ 3:27:03<18:43:14] +[titan] 2025-07-09 17:02:04,317 - root - INFO - step: 15570 loss: 16.9975 memory: 44.58GiB(31.99%) tps: 84,205 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.6610 global_avg_mtp_loss: 14.3365 +[titan] 2025-07-09 17:02:04,317 - root - INFO - lr: 2.8582e-04 gnorm: 0.86 [ 3:27:07<18:43:09] +[titan] 2025-07-09 17:02:08,218 - root - INFO - step: 15575 loss: 17.0605 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6858 global_avg_mtp_loss: 14.3747 +[titan] 2025-07-09 17:02:08,218 - root - INFO - lr: 2.8581e-04 gnorm: 0.85 [ 3:27:11<18:43:05] +[titan] 2025-07-09 17:02:12,125 - root - INFO - step: 15580 loss: 16.9977 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6595 global_avg_mtp_loss: 14.3382 +[titan] 2025-07-09 17:02:12,125 - root - INFO - lr: 2.8580e-04 gnorm: 0.83 [ 3:27:15<18:43:00] +[titan] 2025-07-09 17:02:16,032 - root - INFO - step: 15585 loss: 17.0873 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6757 global_avg_mtp_loss: 14.4116 +[titan] 2025-07-09 17:02:16,032 - root - INFO - lr: 2.8580e-04 gnorm: 0.84 [ 3:27:19<18:42:56] +[titan] 2025-07-09 17:02:19,996 - root - INFO - step: 15590 loss: 17.2205 memory: 44.58GiB(31.99%) tps: 82,682 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.6930 global_avg_mtp_loss: 14.5275 +[titan] 2025-07-09 17:02:19,996 - root - INFO - lr: 2.8579e-04 gnorm: 0.92 [ 3:27:23<18:42:52] +[titan] 2025-07-09 17:02:23,910 - root - INFO - step: 15595 loss: 16.9467 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6481 global_avg_mtp_loss: 14.2986 +[titan] 2025-07-09 17:02:23,910 - root - INFO - lr: 2.8578e-04 gnorm: 0.86 [ 3:27:27<18:42:47] +[titan] 2025-07-09 17:02:27,026 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:02:27,812 - root - INFO - step: 15600 loss: 17.0143 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6604 global_avg_mtp_loss: 14.3539 +[titan] 2025-07-09 17:02:27,812 - root - INFO - lr: 2.8577e-04 gnorm: 0.88 [ 3:27:31<18:42:43] +[titan] 2025-07-09 17:02:31,750 - root - INFO - step: 15605 loss: 17.2484 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.7028 global_avg_mtp_loss: 14.5455 +[titan] 2025-07-09 17:02:31,751 - root - INFO - lr: 2.8576e-04 gnorm: 0.89 [ 3:27:34<18:42:39] +[titan] 2025-07-09 17:02:35,675 - root - INFO - step: 15610 loss: 17.2783 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.7179 global_avg_mtp_loss: 14.5603 +[titan] 2025-07-09 17:02:35,675 - root - INFO - lr: 2.8575e-04 gnorm: 0.83 [ 3:27:38<18:42:34] +[titan] 2025-07-09 17:02:39,562 - root - INFO - step: 15615 loss: 17.0756 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6716 global_avg_mtp_loss: 14.4040 +[titan] 2025-07-09 17:02:39,562 - root - INFO - lr: 2.8574e-04 gnorm: 0.81 [ 3:27:42<18:42:30] +[titan] 2025-07-09 17:02:43,598 - root - INFO - step: 15620 loss: 17.0483 memory: 44.58GiB(31.99%) tps: 81,185 tflops: 280.18 mfu: 28.33% global_avg_ntp_loss: 2.6635 global_avg_mtp_loss: 14.3849 +[titan] 2025-07-09 17:02:43,599 - root - INFO - lr: 2.8573e-04 gnorm: 0.86 [ 3:27:46<18:42:26] +[titan] 2025-07-09 17:02:47,519 - root - INFO - step: 15625 loss: 17.0262 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6608 global_avg_mtp_loss: 14.3654 +[titan] 2025-07-09 17:02:47,520 - root - INFO - lr: 2.8572e-04 gnorm: 0.87 [ 3:27:50<18:42:22] +[titan] 2025-07-09 17:02:51,425 - root - INFO - step: 15630 loss: 17.1944 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6884 global_avg_mtp_loss: 14.5060 +[titan] 2025-07-09 17:02:51,425 - root - INFO - lr: 2.8571e-04 gnorm: 0.88 [ 3:27:54<18:42:17] +[titan] 2025-07-09 17:02:55,344 - root - INFO - step: 15635 loss: 17.0499 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.6599 global_avg_mtp_loss: 14.3900 +[titan] 2025-07-09 17:02:55,345 - root - INFO - lr: 2.8570e-04 gnorm: 0.90 [ 3:27:58<18:42:13] +[titan] 2025-07-09 17:02:59,275 - root - INFO - step: 15640 loss: 16.9565 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.6573 global_avg_mtp_loss: 14.2992 +[titan] 2025-07-09 17:02:59,276 - root - INFO - lr: 2.8569e-04 gnorm: 0.80 [ 3:28:02<18:42:08] +[titan] 2025-07-09 17:03:03,202 - root - INFO - step: 15645 loss: 17.0155 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6626 global_avg_mtp_loss: 14.3530 +[titan] 2025-07-09 17:03:03,202 - root - INFO - lr: 2.8568e-04 gnorm: 0.83 [ 3:28:06<18:42:04] +[titan] 2025-07-09 17:03:06,324 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:03:07,132 - root - INFO - step: 15650 loss: 16.9961 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6575 global_avg_mtp_loss: 14.3386 +[titan] 2025-07-09 17:03:07,132 - root - INFO - lr: 2.8567e-04 gnorm: 0.85 [ 3:28:10<18:42:00] +[titan] 2025-07-09 17:03:11,030 - root - INFO - step: 15655 loss: 17.0255 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6603 global_avg_mtp_loss: 14.3652 +[titan] 2025-07-09 17:03:11,031 - root - INFO - lr: 2.8566e-04 gnorm: 0.79 [ 3:28:14<18:41:55] +[titan] 2025-07-09 17:03:14,928 - root - INFO - step: 15660 loss: 17.2037 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6857 global_avg_mtp_loss: 14.5180 +[titan] 2025-07-09 17:03:14,929 - root - INFO - lr: 2.8565e-04 gnorm: 0.80 [ 3:28:18<18:41:51] +[titan] 2025-07-09 17:03:18,866 - root - INFO - step: 15665 loss: 17.1201 memory: 44.58GiB(31.99%) tps: 83,229 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.6897 global_avg_mtp_loss: 14.4304 +[titan] 2025-07-09 17:03:18,866 - root - INFO - lr: 2.8564e-04 gnorm: 0.83 [ 3:28:22<18:41:47] +[titan] 2025-07-09 17:03:22,793 - root - INFO - step: 15670 loss: 17.1551 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6790 global_avg_mtp_loss: 14.4761 +[titan] 2025-07-09 17:03:22,794 - root - INFO - lr: 2.8563e-04 gnorm: 1.02 [ 3:28:26<18:41:42] +[titan] 2025-07-09 17:03:26,717 - root - INFO - step: 15675 loss: 17.0975 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4314 +[titan] 2025-07-09 17:03:26,718 - root - INFO - lr: 2.8562e-04 gnorm: 0.93 [ 3:28:29<18:41:38] +[titan] 2025-07-09 17:03:30,761 - root - INFO - step: 15680 loss: 17.0237 memory: 44.58GiB(31.99%) tps: 81,047 tflops: 279.71 mfu: 28.28% global_avg_ntp_loss: 2.6577 global_avg_mtp_loss: 14.3660 +[titan] 2025-07-09 17:03:30,761 - root - INFO - lr: 2.8561e-04 gnorm: 0.90 [ 3:28:33<18:41:34] +[titan] 2025-07-09 17:03:34,680 - root - INFO - step: 15685 loss: 17.3246 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.7273 global_avg_mtp_loss: 14.5973 +[titan] 2025-07-09 17:03:34,681 - root - INFO - lr: 2.8560e-04 gnorm: 0.78 [ 3:28:37<18:41:30] +[titan] 2025-07-09 17:03:38,625 - root - INFO - step: 15690 loss: 17.2221 memory: 44.58GiB(31.99%) tps: 83,078 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.7111 global_avg_mtp_loss: 14.5110 +[titan] 2025-07-09 17:03:38,625 - root - INFO - lr: 2.8559e-04 gnorm: 0.87 [ 3:28:41<18:41:26] +[titan] 2025-07-09 17:03:42,546 - root - INFO - step: 15695 loss: 17.2272 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.7002 global_avg_mtp_loss: 14.5270 +[titan] 2025-07-09 17:03:42,546 - root - INFO - lr: 2.8558e-04 gnorm: 0.84 [ 3:28:45<18:41:21] +[titan] 2025-07-09 17:03:45,653 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:03:46,443 - root - INFO - step: 15700 loss: 17.2244 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.6906 global_avg_mtp_loss: 14.5338 +[titan] 2025-07-09 17:03:46,443 - root - INFO - lr: 2.8557e-04 gnorm: 0.88 [ 3:28:49<18:41:17] +[titan] 2025-07-09 17:03:50,382 - root - INFO - step: 15705 loss: 16.9535 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.6546 global_avg_mtp_loss: 14.2989 +[titan] 2025-07-09 17:03:50,382 - root - INFO - lr: 2.8556e-04 gnorm: 0.84 [ 3:28:53<18:41:12] +[titan] 2025-07-09 17:03:54,305 - root - INFO - step: 15710 loss: 17.1659 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6863 global_avg_mtp_loss: 14.4796 +[titan] 2025-07-09 17:03:54,305 - root - INFO - lr: 2.8556e-04 gnorm: 0.97 [ 3:28:57<18:41:08] +[titan] 2025-07-09 17:03:58,248 - root - INFO - step: 15715 loss: 17.2758 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.6938 global_avg_mtp_loss: 14.5820 +[titan] 2025-07-09 17:03:58,248 - root - INFO - lr: 2.8555e-04 gnorm: 0.84 [ 3:29:01<18:41:04] +[titan] 2025-07-09 17:04:02,212 - root - INFO - step: 15720 loss: 17.2803 memory: 44.58GiB(31.99%) tps: 82,666 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.7003 global_avg_mtp_loss: 14.5800 +[titan] 2025-07-09 17:04:02,213 - root - INFO - lr: 2.8554e-04 gnorm: 0.85 [ 3:29:05<18:41:00] +[titan] 2025-07-09 17:04:06,125 - root - INFO - step: 15725 loss: 17.0504 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6696 global_avg_mtp_loss: 14.3808 +[titan] 2025-07-09 17:04:06,125 - root - INFO - lr: 2.8553e-04 gnorm: 0.77 [ 3:29:09<18:40:55] +[titan] 2025-07-09 17:04:10,060 - root - INFO - step: 15730 loss: 16.9933 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6613 global_avg_mtp_loss: 14.3320 +[titan] 2025-07-09 17:04:10,061 - root - INFO - lr: 2.8552e-04 gnorm: 0.87 [ 3:29:13<18:40:51] +[titan] 2025-07-09 17:04:13,954 - root - INFO - step: 15735 loss: 17.1176 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.6789 global_avg_mtp_loss: 14.4387 +[titan] 2025-07-09 17:04:13,954 - root - INFO - lr: 2.8551e-04 gnorm: 0.94 [ 3:29:17<18:40:46] +[titan] 2025-07-09 17:04:17,855 - root - INFO - step: 15740 loss: 17.0259 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6690 global_avg_mtp_loss: 14.3569 +[titan] 2025-07-09 17:04:17,856 - root - INFO - lr: 2.8550e-04 gnorm: 0.87 [ 3:29:21<18:40:42] +[titan] 2025-07-09 17:04:21,781 - root - INFO - step: 15745 loss: 16.9123 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.2711 +[titan] 2025-07-09 17:04:21,782 - root - INFO - lr: 2.8549e-04 gnorm: 0.81 [ 3:29:25<18:40:38] +[titan] 2025-07-09 17:04:24,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:04:25,696 - root - INFO - step: 15750 loss: 17.1583 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6817 global_avg_mtp_loss: 14.4767 +[titan] 2025-07-09 17:04:25,696 - root - INFO - lr: 2.8548e-04 gnorm: 0.81 [ 3:29:28<18:40:33] +[titan] 2025-07-09 17:04:29,617 - root - INFO - step: 15755 loss: 17.0901 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6679 global_avg_mtp_loss: 14.4222 +[titan] 2025-07-09 17:04:29,617 - root - INFO - lr: 2.8547e-04 gnorm: 0.85 [ 3:29:32<18:40:29] +[titan] 2025-07-09 17:04:33,527 - root - INFO - step: 15760 loss: 16.9319 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6409 global_avg_mtp_loss: 14.2910 +[titan] 2025-07-09 17:04:33,527 - root - INFO - lr: 2.8546e-04 gnorm: 0.88 [ 3:29:36<18:40:24] +[titan] 2025-07-09 17:04:37,425 - root - INFO - step: 15765 loss: 16.9686 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6549 global_avg_mtp_loss: 14.3137 +[titan] 2025-07-09 17:04:37,425 - root - INFO - lr: 2.8545e-04 gnorm: 0.82 [ 3:29:40<18:40:20] +[titan] 2025-07-09 17:04:41,330 - root - INFO - step: 15770 loss: 17.2809 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.7074 global_avg_mtp_loss: 14.5734 +[titan] 2025-07-09 17:04:41,330 - root - INFO - lr: 2.8544e-04 gnorm: 0.79 [ 3:29:44<18:40:16] +[titan] 2025-07-09 17:04:45,286 - root - INFO - step: 15775 loss: 17.0619 memory: 44.58GiB(31.99%) tps: 82,830 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.6710 global_avg_mtp_loss: 14.3909 +[titan] 2025-07-09 17:04:45,287 - root - INFO - lr: 2.8543e-04 gnorm: 0.83 [ 3:29:48<18:40:11] +[titan] 2025-07-09 17:04:49,231 - root - INFO - step: 15780 loss: 17.1552 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.6870 global_avg_mtp_loss: 14.4683 +[titan] 2025-07-09 17:04:49,231 - root - INFO - lr: 2.8542e-04 gnorm: 0.90 [ 3:29:52<18:40:07] +[titan] 2025-07-09 17:04:53,200 - root - INFO - step: 15785 loss: 17.4148 memory: 44.58GiB(31.99%) tps: 82,564 tflops: 284.94 mfu: 28.81% global_avg_ntp_loss: 2.7388 global_avg_mtp_loss: 14.6760 +[titan] 2025-07-09 17:04:53,200 - root - INFO - lr: 2.8541e-04 gnorm: 0.86 [ 3:29:56<18:40:03] +[titan] 2025-07-09 17:04:57,115 - root - INFO - step: 15790 loss: 17.0577 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6723 global_avg_mtp_loss: 14.3854 +[titan] 2025-07-09 17:04:57,115 - root - INFO - lr: 2.8540e-04 gnorm: 0.87 [ 3:30:00<18:39:59] +[titan] 2025-07-09 17:05:01,026 - root - INFO - step: 15795 loss: 17.2397 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.7097 global_avg_mtp_loss: 14.5300 +[titan] 2025-07-09 17:05:01,027 - root - INFO - lr: 2.8539e-04 gnorm: 0.84 [ 3:30:04<18:39:54] +[titan] 2025-07-09 17:05:04,147 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:05:04,933 - root - INFO - step: 15800 loss: 17.0652 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.3871 +[titan] 2025-07-09 17:05:04,933 - root - INFO - lr: 2.8538e-04 gnorm: 0.80 [ 3:30:08<18:39:50] +[titan] 2025-07-09 17:05:08,852 - root - INFO - step: 15805 loss: 17.0055 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6635 global_avg_mtp_loss: 14.3419 +[titan] 2025-07-09 17:05:08,852 - root - INFO - lr: 2.8537e-04 gnorm: 0.83 [ 3:30:12<18:39:45] +[titan] 2025-07-09 17:05:12,766 - root - INFO - step: 15810 loss: 17.1480 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.4685 +[titan] 2025-07-09 17:05:12,766 - root - INFO - lr: 2.8536e-04 gnorm: 0.86 [ 3:30:15<18:39:41] +[titan] 2025-07-09 17:05:16,687 - root - INFO - step: 15815 loss: 17.0930 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6841 global_avg_mtp_loss: 14.4089 +[titan] 2025-07-09 17:05:16,687 - root - INFO - lr: 2.8535e-04 gnorm: 0.91 [ 3:30:19<18:39:37] +[titan] 2025-07-09 17:05:20,794 - root - INFO - step: 15820 loss: 17.3773 memory: 44.58GiB(31.99%) tps: 79,784 tflops: 275.35 mfu: 27.84% global_avg_ntp_loss: 2.7453 global_avg_mtp_loss: 14.6320 +[titan] 2025-07-09 17:05:20,794 - root - INFO - lr: 2.8534e-04 gnorm: 0.91 [ 3:30:24<18:39:33] +[titan] 2025-07-09 17:05:24,697 - root - INFO - step: 15825 loss: 17.0867 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6828 global_avg_mtp_loss: 14.4039 +[titan] 2025-07-09 17:05:24,697 - root - INFO - lr: 2.8533e-04 gnorm: 0.87 [ 3:30:27<18:39:29] +[titan] 2025-07-09 17:05:28,588 - root - INFO - step: 15830 loss: 17.1222 memory: 44.58GiB(31.99%) tps: 84,217 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6673 global_avg_mtp_loss: 14.4549 +[titan] 2025-07-09 17:05:28,588 - root - INFO - lr: 2.8532e-04 gnorm: 0.94 [ 3:30:31<18:39:24] +[titan] 2025-07-09 17:05:32,496 - root - INFO - step: 15835 loss: 17.2070 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6932 global_avg_mtp_loss: 14.5139 +[titan] 2025-07-09 17:05:32,496 - root - INFO - lr: 2.8531e-04 gnorm: 0.91 [ 3:30:35<18:39:20] +[titan] 2025-07-09 17:05:36,412 - root - INFO - step: 15840 loss: 17.0426 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6693 global_avg_mtp_loss: 14.3733 +[titan] 2025-07-09 17:05:36,412 - root - INFO - lr: 2.8530e-04 gnorm: 0.83 [ 3:30:39<18:39:16] +[titan] 2025-07-09 17:05:40,326 - root - INFO - step: 15845 loss: 16.8931 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.2505 +[titan] 2025-07-09 17:05:40,326 - root - INFO - lr: 2.8529e-04 gnorm: 0.82 [ 3:30:43<18:39:11] +[titan] 2025-07-09 17:05:43,454 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:05:44,246 - root - INFO - step: 15850 loss: 17.2160 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6953 global_avg_mtp_loss: 14.5206 +[titan] 2025-07-09 17:05:44,246 - root - INFO - lr: 2.8528e-04 gnorm: 0.79 [ 3:30:47<18:39:07] +[titan] 2025-07-09 17:05:48,139 - root - INFO - step: 15855 loss: 17.2017 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.7037 global_avg_mtp_loss: 14.4980 +[titan] 2025-07-09 17:05:48,140 - root - INFO - lr: 2.8527e-04 gnorm: 0.87 [ 3:30:51<18:39:02] +[titan] 2025-07-09 17:05:52,100 - root - INFO - step: 15860 loss: 16.9669 memory: 44.58GiB(31.99%) tps: 82,737 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.3263 +[titan] 2025-07-09 17:05:52,100 - root - INFO - lr: 2.8526e-04 gnorm: 0.81 [ 3:30:55<18:38:58] +[titan] 2025-07-09 17:05:56,024 - root - INFO - step: 15865 loss: 17.1360 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6742 global_avg_mtp_loss: 14.4618 +[titan] 2025-07-09 17:05:56,024 - root - INFO - lr: 2.8526e-04 gnorm: 0.85 [ 3:30:59<18:38:54] +[titan] 2025-07-09 17:06:00,034 - root - INFO - step: 15870 loss: 17.2616 memory: 44.58GiB(31.99%) tps: 81,717 tflops: 282.02 mfu: 28.52% global_avg_ntp_loss: 2.7116 global_avg_mtp_loss: 14.5500 +[titan] 2025-07-09 17:06:00,034 - root - INFO - lr: 2.8525e-04 gnorm: 0.80 [ 3:31:03<18:38:50] +[titan] 2025-07-09 17:06:01,747 - root - INFO - Dumping profiler traces at step 15872 +[titan] 2025-07-09 17:06:01,778 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:06:04,131 - root - INFO - step: 15875 loss: 16.9175 memory: 44.58GiB(31.99%) tps: 79,991 tflops: 276.06 mfu: 27.91% global_avg_ntp_loss: 2.6429 global_avg_mtp_loss: 14.2745 +[titan] 2025-07-09 17:06:04,131 - root - INFO - lr: 2.8524e-04 gnorm: 0.83 [ 3:31:07<18:38:46] +[titan] 2025-07-09 17:06:08,039 - root - INFO - step: 15880 loss: 17.1709 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.4795 +[titan] 2025-07-09 17:06:08,039 - root - INFO - lr: 2.8523e-04 gnorm: 0.80 [ 3:31:11<18:38:42] +[titan] 2025-07-09 17:06:11,962 - root - INFO - step: 15885 loss: 17.1626 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6850 global_avg_mtp_loss: 14.4776 +[titan] 2025-07-09 17:06:11,962 - root - INFO - lr: 2.8522e-04 gnorm: 0.82 [ 3:31:15<18:38:38] +[titan] 2025-07-09 17:06:15,865 - root - INFO - step: 15890 loss: 16.9927 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6394 global_avg_mtp_loss: 14.3533 +[titan] 2025-07-09 17:06:15,865 - root - INFO - lr: 2.8521e-04 gnorm: 0.78 [ 3:31:19<18:38:33] +[titan] 2025-07-09 17:06:19,761 - root - INFO - step: 15895 loss: 17.0409 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.6653 global_avg_mtp_loss: 14.3756 +[titan] 2025-07-09 17:06:19,761 - root - INFO - lr: 2.8520e-04 gnorm: 0.80 [ 3:31:22<18:38:29] +[titan] 2025-07-09 17:06:22,889 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:06:23,677 - root - INFO - step: 15900 loss: 17.1683 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6967 global_avg_mtp_loss: 14.4715 +[titan] 2025-07-09 17:06:23,677 - root - INFO - lr: 2.8519e-04 gnorm: 0.87 [ 3:31:26<18:38:24] +[titan] 2025-07-09 17:06:27,595 - root - INFO - step: 15905 loss: 17.1301 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6714 global_avg_mtp_loss: 14.4587 +[titan] 2025-07-09 17:06:27,595 - root - INFO - lr: 2.8518e-04 gnorm: 0.81 [ 3:31:30<18:38:20] +[titan] 2025-07-09 17:06:31,495 - root - INFO - step: 15910 loss: 17.2734 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.7003 global_avg_mtp_loss: 14.5732 +[titan] 2025-07-09 17:06:31,496 - root - INFO - lr: 2.8517e-04 gnorm: 0.89 [ 3:31:34<18:38:16] +[titan] 2025-07-09 17:06:35,422 - root - INFO - step: 15915 loss: 17.3446 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.7072 global_avg_mtp_loss: 14.6375 +[titan] 2025-07-09 17:06:35,423 - root - INFO - lr: 2.8516e-04 gnorm: 0.84 [ 3:31:38<18:38:11] +[titan] 2025-07-09 17:06:39,333 - root - INFO - step: 15920 loss: 17.0040 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6572 global_avg_mtp_loss: 14.3468 +[titan] 2025-07-09 17:06:39,333 - root - INFO - lr: 2.8515e-04 gnorm: 0.94 [ 3:31:42<18:38:07] +[titan] 2025-07-09 17:06:43,235 - root - INFO - step: 15925 loss: 17.3106 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.7210 global_avg_mtp_loss: 14.5896 +[titan] 2025-07-09 17:06:43,236 - root - INFO - lr: 2.8514e-04 gnorm: 0.89 [ 3:31:46<18:38:02] +[titan] 2025-07-09 17:06:47,175 - root - INFO - step: 15930 loss: 17.1846 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.6887 global_avg_mtp_loss: 14.4959 +[titan] 2025-07-09 17:06:47,175 - root - INFO - lr: 2.8513e-04 gnorm: 0.85 [ 3:31:50<18:37:58] +[titan] 2025-07-09 17:06:51,095 - root - INFO - step: 15935 loss: 17.1512 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6866 global_avg_mtp_loss: 14.4646 +[titan] 2025-07-09 17:06:51,095 - root - INFO - lr: 2.8512e-04 gnorm: 0.78 [ 3:31:54<18:37:54] +[titan] 2025-07-09 17:06:55,007 - root - INFO - step: 15940 loss: 17.1815 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6883 global_avg_mtp_loss: 14.4931 +[titan] 2025-07-09 17:06:55,007 - root - INFO - lr: 2.8511e-04 gnorm: 0.88 [ 3:31:58<18:37:49] +[titan] 2025-07-09 17:06:58,905 - root - INFO - step: 15945 loss: 17.2475 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6996 global_avg_mtp_loss: 14.5478 +[titan] 2025-07-09 17:06:58,905 - root - INFO - lr: 2.8510e-04 gnorm: 0.95 [ 3:32:02<18:37:45] +[titan] 2025-07-09 17:07:02,035 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:07:02,825 - root - INFO - step: 15950 loss: 17.1662 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6860 global_avg_mtp_loss: 14.4802 +[titan] 2025-07-09 17:07:02,826 - root - INFO - lr: 2.8509e-04 gnorm: 0.78 [ 3:32:06<18:37:41] +[titan] 2025-07-09 17:07:06,747 - root - INFO - step: 15955 loss: 16.9840 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6578 global_avg_mtp_loss: 14.3261 +[titan] 2025-07-09 17:07:06,747 - root - INFO - lr: 2.8508e-04 gnorm: 0.86 [ 3:32:09<18:37:36] +[titan] 2025-07-09 17:07:10,662 - root - INFO - step: 15960 loss: 16.9914 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.3395 +[titan] 2025-07-09 17:07:10,662 - root - INFO - lr: 2.8507e-04 gnorm: 0.83 [ 3:32:13<18:37:32] +[titan] 2025-07-09 17:07:14,581 - root - INFO - step: 15965 loss: 17.0448 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6516 global_avg_mtp_loss: 14.3932 +[titan] 2025-07-09 17:07:14,581 - root - INFO - lr: 2.8506e-04 gnorm: 0.83 [ 3:32:17<18:37:27] +[titan] 2025-07-09 17:07:18,510 - root - INFO - step: 15970 loss: 17.2017 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6847 global_avg_mtp_loss: 14.5170 +[titan] 2025-07-09 17:07:18,510 - root - INFO - lr: 2.8505e-04 gnorm: 0.81 [ 3:32:21<18:37:23] +[titan] 2025-07-09 17:07:22,424 - root - INFO - step: 15975 loss: 16.7717 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.1625 +[titan] 2025-07-09 17:07:22,424 - root - INFO - lr: 2.8504e-04 gnorm: 0.92 [ 3:32:25<18:37:19] +[titan] 2025-07-09 17:07:26,347 - root - INFO - step: 15980 loss: 16.7883 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6146 global_avg_mtp_loss: 14.1736 +[titan] 2025-07-09 17:07:26,348 - root - INFO - lr: 2.8503e-04 gnorm: 0.86 [ 3:32:29<18:37:14] +[titan] 2025-07-09 17:07:30,353 - root - INFO - step: 15985 loss: 17.1168 memory: 44.58GiB(31.99%) tps: 81,820 tflops: 282.38 mfu: 28.55% global_avg_ntp_loss: 2.6824 global_avg_mtp_loss: 14.4343 +[titan] 2025-07-09 17:07:30,353 - root - INFO - lr: 2.8502e-04 gnorm: 0.88 [ 3:32:33<18:37:10] +[titan] 2025-07-09 17:07:34,287 - root - INFO - step: 15990 loss: 17.0415 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6454 global_avg_mtp_loss: 14.3961 +[titan] 2025-07-09 17:07:34,287 - root - INFO - lr: 2.8501e-04 gnorm: 0.85 [ 3:32:37<18:37:06] +[titan] 2025-07-09 17:07:38,192 - root - INFO - step: 15995 loss: 17.2171 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6923 global_avg_mtp_loss: 14.5248 +[titan] 2025-07-09 17:07:38,192 - root - INFO - lr: 2.8500e-04 gnorm: 0.83 [ 3:32:41<18:37:02] +[titan] 2025-07-09 17:07:41,341 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:07:42,142 - root - INFO - step: 16000 loss: 17.1872 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.6835 global_avg_mtp_loss: 14.5038 +[titan] 2025-07-09 17:07:42,142 - root - INFO - lr: 2.8499e-04 gnorm: 0.81 [ 3:32:45<18:36:58] +[titan] 2025-07-09 17:07:46,057 - root - INFO - step: 16005 loss: 16.7787 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6205 global_avg_mtp_loss: 14.1582 +[titan] 2025-07-09 17:07:46,057 - root - INFO - lr: 2.8498e-04 gnorm: 0.82 [ 3:32:49<18:36:53] +[titan] 2025-07-09 17:07:49,971 - root - INFO - step: 16010 loss: 17.1452 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6848 global_avg_mtp_loss: 14.4604 +[titan] 2025-07-09 17:07:49,971 - root - INFO - lr: 2.8497e-04 gnorm: 0.83 [ 3:32:53<18:36:49] +[titan] 2025-07-09 17:07:54,025 - root - INFO - step: 16015 loss: 17.2705 memory: 44.58GiB(31.99%) tps: 80,844 tflops: 279.01 mfu: 28.21% global_avg_ntp_loss: 2.7112 global_avg_mtp_loss: 14.5592 +[titan] 2025-07-09 17:07:54,025 - root - INFO - lr: 2.8496e-04 gnorm: 0.79 [ 3:32:57<18:36:45] +[titan] 2025-07-09 17:07:57,950 - root - INFO - step: 16020 loss: 17.3236 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.7094 global_avg_mtp_loss: 14.6142 +[titan] 2025-07-09 17:07:57,950 - root - INFO - lr: 2.8495e-04 gnorm: 0.84 [ 3:33:01<18:36:41] +[titan] 2025-07-09 17:08:01,900 - root - INFO - step: 16025 loss: 16.8451 memory: 44.58GiB(31.99%) tps: 82,965 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2269 +[titan] 2025-07-09 17:08:01,900 - root - INFO - lr: 2.8494e-04 gnorm: 0.78 [ 3:33:05<18:36:37] +[titan] 2025-07-09 17:08:05,872 - root - INFO - step: 16030 loss: 17.0430 memory: 44.58GiB(31.99%) tps: 82,501 tflops: 284.73 mfu: 28.79% global_avg_ntp_loss: 2.6516 global_avg_mtp_loss: 14.3914 +[titan] 2025-07-09 17:08:05,872 - root - INFO - lr: 2.8493e-04 gnorm: 0.82 [ 3:33:09<18:36:33] +[titan] 2025-07-09 17:08:09,850 - root - INFO - step: 16035 loss: 17.0435 memory: 44.58GiB(31.99%) tps: 82,386 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.6642 global_avg_mtp_loss: 14.3793 +[titan] 2025-07-09 17:08:09,850 - root - INFO - lr: 2.8492e-04 gnorm: 0.83 [ 3:33:13<18:36:28] +[titan] 2025-07-09 17:08:13,749 - root - INFO - step: 16040 loss: 16.6315 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.0537 +[titan] 2025-07-09 17:08:13,750 - root - INFO - lr: 2.8491e-04 gnorm: 0.90 [ 3:33:16<18:36:24] +[titan] 2025-07-09 17:08:17,649 - root - INFO - step: 16045 loss: 16.9374 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6515 global_avg_mtp_loss: 14.2859 +[titan] 2025-07-09 17:08:17,650 - root - INFO - lr: 2.8490e-04 gnorm: 0.97 [ 3:33:20<18:36:20] +[titan] 2025-07-09 17:08:20,796 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:08:21,586 - root - INFO - step: 16050 loss: 17.0987 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.6793 global_avg_mtp_loss: 14.4194 +[titan] 2025-07-09 17:08:21,586 - root - INFO - lr: 2.8489e-04 gnorm: 0.85 [ 3:33:24<18:36:15] +[titan] 2025-07-09 17:08:25,506 - root - INFO - step: 16055 loss: 16.9534 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6516 global_avg_mtp_loss: 14.3018 +[titan] 2025-07-09 17:08:25,506 - root - INFO - lr: 2.8488e-04 gnorm: 0.81 [ 3:33:28<18:36:11] +[titan] 2025-07-09 17:08:29,441 - root - INFO - step: 16060 loss: 17.2301 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.7003 global_avg_mtp_loss: 14.5298 +[titan] 2025-07-09 17:08:29,441 - root - INFO - lr: 2.8487e-04 gnorm: 0.89 [ 3:33:32<18:36:07] +[titan] 2025-07-09 17:08:33,473 - root - INFO - step: 16065 loss: 17.1774 memory: 44.58GiB(31.99%) tps: 81,273 tflops: 280.49 mfu: 28.36% global_avg_ntp_loss: 2.6990 global_avg_mtp_loss: 14.4784 +[titan] 2025-07-09 17:08:33,473 - root - INFO - lr: 2.8486e-04 gnorm: 0.82 [ 3:33:36<18:36:03] +[titan] 2025-07-09 17:08:37,470 - root - INFO - step: 16070 loss: 17.1949 memory: 44.58GiB(31.99%) tps: 81,996 tflops: 282.98 mfu: 28.61% global_avg_ntp_loss: 2.6863 global_avg_mtp_loss: 14.5086 +[titan] 2025-07-09 17:08:37,470 - root - INFO - lr: 2.8485e-04 gnorm: 0.88 [ 3:33:40<18:35:59] +[titan] 2025-07-09 17:08:41,385 - root - INFO - step: 16075 loss: 16.9804 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6359 global_avg_mtp_loss: 14.3445 +[titan] 2025-07-09 17:08:41,385 - root - INFO - lr: 2.8484e-04 gnorm: 0.81 [ 3:33:44<18:35:55] +[titan] 2025-07-09 17:08:45,313 - root - INFO - step: 16080 loss: 16.9515 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3119 +[titan] 2025-07-09 17:08:45,313 - root - INFO - lr: 2.8483e-04 gnorm: 0.81 [ 3:33:48<18:35:50] +[titan] 2025-07-09 17:08:49,251 - root - INFO - step: 16085 loss: 17.2271 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.7035 global_avg_mtp_loss: 14.5236 +[titan] 2025-07-09 17:08:49,251 - root - INFO - lr: 2.8482e-04 gnorm: 0.79 [ 3:33:52<18:35:46] +[titan] 2025-07-09 17:08:53,166 - root - INFO - step: 16090 loss: 17.1076 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6727 global_avg_mtp_loss: 14.4349 +[titan] 2025-07-09 17:08:53,166 - root - INFO - lr: 2.8481e-04 gnorm: 0.82 [ 3:33:56<18:35:42] +[titan] 2025-07-09 17:08:57,083 - root - INFO - step: 16095 loss: 17.0225 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3734 +[titan] 2025-07-09 17:08:57,083 - root - INFO - lr: 2.8480e-04 gnorm: 0.85 [ 3:34:00<18:35:37] +[titan] 2025-07-09 17:09:00,193 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:09:00,982 - root - INFO - step: 16100 loss: 17.2138 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6975 global_avg_mtp_loss: 14.5163 +[titan] 2025-07-09 17:09:00,982 - root - INFO - lr: 2.8479e-04 gnorm: 0.81 [ 3:34:04<18:35:33] +[titan] 2025-07-09 17:09:04,985 - root - INFO - step: 16105 loss: 17.0564 memory: 44.58GiB(31.99%) tps: 81,850 tflops: 282.48 mfu: 28.56% global_avg_ntp_loss: 2.6736 global_avg_mtp_loss: 14.3828 +[titan] 2025-07-09 17:09:04,986 - root - INFO - lr: 2.8478e-04 gnorm: 0.91 [ 3:34:08<18:35:29] +[titan] 2025-07-09 17:09:08,884 - root - INFO - step: 16110 loss: 16.9988 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3470 +[titan] 2025-07-09 17:09:08,884 - root - INFO - lr: 2.8477e-04 gnorm: 0.89 [ 3:34:12<18:35:24] +[titan] 2025-07-09 17:09:12,785 - root - INFO - step: 16115 loss: 17.2205 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.7070 global_avg_mtp_loss: 14.5136 +[titan] 2025-07-09 17:09:12,785 - root - INFO - lr: 2.8476e-04 gnorm: 0.85 [ 3:34:15<18:35:20] +[titan] 2025-07-09 17:09:16,685 - root - INFO - step: 16120 loss: 17.0900 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6691 global_avg_mtp_loss: 14.4209 +[titan] 2025-07-09 17:09:16,686 - root - INFO - lr: 2.8475e-04 gnorm: 0.84 [ 3:34:19<18:35:16] +[titan] 2025-07-09 17:09:20,627 - root - INFO - step: 16125 loss: 17.0832 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.6666 global_avg_mtp_loss: 14.4166 +[titan] 2025-07-09 17:09:20,627 - root - INFO - lr: 2.8474e-04 gnorm: 0.82 [ 3:34:23<18:35:11] +[titan] 2025-07-09 17:09:24,553 - root - INFO - step: 16130 loss: 17.2038 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6842 global_avg_mtp_loss: 14.5196 +[titan] 2025-07-09 17:09:24,553 - root - INFO - lr: 2.8473e-04 gnorm: 0.80 [ 3:34:27<18:35:07] +[titan] 2025-07-09 17:09:28,475 - root - INFO - step: 16135 loss: 17.0714 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6745 global_avg_mtp_loss: 14.3969 +[titan] 2025-07-09 17:09:28,475 - root - INFO - lr: 2.8472e-04 gnorm: 0.87 [ 3:34:31<18:35:03] +[titan] 2025-07-09 17:09:32,375 - root - INFO - step: 16140 loss: 17.0978 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6755 global_avg_mtp_loss: 14.4222 +[titan] 2025-07-09 17:09:32,375 - root - INFO - lr: 2.8472e-04 gnorm: 0.82 [ 3:34:35<18:34:58] +[titan] 2025-07-09 17:09:36,310 - root - INFO - step: 16145 loss: 17.0681 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.6668 global_avg_mtp_loss: 14.4013 +[titan] 2025-07-09 17:09:36,310 - root - INFO - lr: 2.8471e-04 gnorm: 0.79 [ 3:34:39<18:34:54] +[titan] 2025-07-09 17:09:39,438 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:09:40,223 - root - INFO - step: 16150 loss: 17.2931 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7140 global_avg_mtp_loss: 14.5790 +[titan] 2025-07-09 17:09:40,223 - root - INFO - lr: 2.8470e-04 gnorm: 0.75 [ 3:34:43<18:34:50] +[titan] 2025-07-09 17:09:44,149 - root - INFO - step: 16155 loss: 17.0720 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6608 global_avg_mtp_loss: 14.4112 +[titan] 2025-07-09 17:09:44,149 - root - INFO - lr: 2.8469e-04 gnorm: 0.81 [ 3:34:47<18:34:45] +[titan] 2025-07-09 17:09:48,070 - root - INFO - step: 16160 loss: 17.1602 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6758 global_avg_mtp_loss: 14.4845 +[titan] 2025-07-09 17:09:48,071 - root - INFO - lr: 2.8468e-04 gnorm: 0.78 [ 3:34:51<18:34:41] +[titan] 2025-07-09 17:09:52,042 - root - INFO - step: 16165 loss: 16.9550 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.3077 +[titan] 2025-07-09 17:09:52,043 - root - INFO - lr: 2.8467e-04 gnorm: 0.77 [ 3:34:55<18:34:37] +[titan] 2025-07-09 17:09:55,959 - root - INFO - step: 16170 loss: 17.0840 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6831 global_avg_mtp_loss: 14.4010 +[titan] 2025-07-09 17:09:55,960 - root - INFO - lr: 2.8466e-04 gnorm: 0.79 [ 3:34:59<18:34:32] +[titan] 2025-07-09 17:09:59,967 - root - INFO - step: 16175 loss: 17.0263 memory: 44.58GiB(31.99%) tps: 81,767 tflops: 282.19 mfu: 28.53% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.3554 +[titan] 2025-07-09 17:09:59,967 - root - INFO - lr: 2.8465e-04 gnorm: 0.83 [ 3:35:03<18:34:29] +[titan] 2025-07-09 17:10:03,922 - root - INFO - step: 16180 loss: 16.7749 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.6064 global_avg_mtp_loss: 14.1685 +[titan] 2025-07-09 17:10:03,923 - root - INFO - lr: 2.8464e-04 gnorm: 0.80 [ 3:35:07<18:34:24] +[titan] 2025-07-09 17:10:07,814 - root - INFO - step: 16185 loss: 17.1716 memory: 44.58GiB(31.99%) tps: 84,212 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.7043 global_avg_mtp_loss: 14.4673 +[titan] 2025-07-09 17:10:07,814 - root - INFO - lr: 2.8463e-04 gnorm: 0.89 [ 3:35:10<18:34:20] +[titan] 2025-07-09 17:10:11,717 - root - INFO - step: 16190 loss: 17.1443 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.6713 global_avg_mtp_loss: 14.4731 +[titan] 2025-07-09 17:10:11,717 - root - INFO - lr: 2.8462e-04 gnorm: 0.75 [ 3:35:14<18:34:15] +[titan] 2025-07-09 17:10:15,626 - root - INFO - step: 16195 loss: 17.2410 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6979 global_avg_mtp_loss: 14.5430 +[titan] 2025-07-09 17:10:15,626 - root - INFO - lr: 2.8461e-04 gnorm: 0.80 [ 3:35:18<18:34:11] +[titan] 2025-07-09 17:10:18,760 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:10:19,568 - root - INFO - step: 16200 loss: 17.0462 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.6684 global_avg_mtp_loss: 14.3778 +[titan] 2025-07-09 17:10:19,568 - root - INFO - lr: 2.8460e-04 gnorm: 0.80 [ 3:35:22<18:34:07] +[titan] 2025-07-09 17:10:23,486 - root - INFO - step: 16205 loss: 17.0113 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6580 global_avg_mtp_loss: 14.3533 +[titan] 2025-07-09 17:10:23,486 - root - INFO - lr: 2.8459e-04 gnorm: 0.83 [ 3:35:26<18:34:02] +[titan] 2025-07-09 17:10:27,404 - root - INFO - step: 16210 loss: 17.0154 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6574 global_avg_mtp_loss: 14.3580 +[titan] 2025-07-09 17:10:27,405 - root - INFO - lr: 2.8458e-04 gnorm: 0.76 [ 3:35:30<18:33:58] +[titan] 2025-07-09 17:10:31,326 - root - INFO - step: 16215 loss: 17.2251 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.6965 global_avg_mtp_loss: 14.5286 +[titan] 2025-07-09 17:10:31,326 - root - INFO - lr: 2.8457e-04 gnorm: 0.81 [ 3:35:34<18:33:54] +[titan] 2025-07-09 17:10:35,237 - root - INFO - step: 16220 loss: 17.1300 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6743 global_avg_mtp_loss: 14.4557 +[titan] 2025-07-09 17:10:35,238 - root - INFO - lr: 2.8456e-04 gnorm: 0.86 [ 3:35:38<18:33:49] +[titan] 2025-07-09 17:10:39,152 - root - INFO - step: 16225 loss: 17.1023 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6687 global_avg_mtp_loss: 14.4336 +[titan] 2025-07-09 17:10:39,152 - root - INFO - lr: 2.8455e-04 gnorm: 0.86 [ 3:35:42<18:33:45] +[titan] 2025-07-09 17:10:43,068 - root - INFO - step: 16230 loss: 17.0469 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.3770 +[titan] 2025-07-09 17:10:43,068 - root - INFO - lr: 2.8454e-04 gnorm: 0.81 [ 3:35:46<18:33:41] +[titan] 2025-07-09 17:10:46,983 - root - INFO - step: 16235 loss: 16.9757 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6464 global_avg_mtp_loss: 14.3293 +[titan] 2025-07-09 17:10:46,983 - root - INFO - lr: 2.8453e-04 gnorm: 0.93 [ 3:35:50<18:33:36] +[titan] 2025-07-09 17:10:50,893 - root - INFO - step: 16240 loss: 16.9715 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6415 global_avg_mtp_loss: 14.3300 +[titan] 2025-07-09 17:10:50,893 - root - INFO - lr: 2.8452e-04 gnorm: 0.80 [ 3:35:54<18:33:32] +[titan] 2025-07-09 17:10:54,812 - root - INFO - step: 16245 loss: 17.0165 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6606 global_avg_mtp_loss: 14.3559 +[titan] 2025-07-09 17:10:54,812 - root - INFO - lr: 2.8451e-04 gnorm: 0.88 [ 3:35:57<18:33:28] +[titan] 2025-07-09 17:10:57,924 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:10:58,711 - root - INFO - step: 16250 loss: 17.0162 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3644 +[titan] 2025-07-09 17:10:58,712 - root - INFO - lr: 2.8450e-04 gnorm: 0.84 [ 3:36:01<18:33:23] +[titan] 2025-07-09 17:11:02,620 - root - INFO - step: 16255 loss: 17.2072 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6987 global_avg_mtp_loss: 14.5085 +[titan] 2025-07-09 17:11:02,620 - root - INFO - lr: 2.8449e-04 gnorm: 0.87 [ 3:36:05<18:33:19] +[titan] 2025-07-09 17:11:06,562 - root - INFO - step: 16260 loss: 17.1911 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.6952 global_avg_mtp_loss: 14.4958 +[titan] 2025-07-09 17:11:06,563 - root - INFO - lr: 2.8448e-04 gnorm: 0.79 [ 3:36:09<18:33:14] +[titan] 2025-07-09 17:11:10,476 - root - INFO - step: 16265 loss: 16.9785 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3347 +[titan] 2025-07-09 17:11:10,477 - root - INFO - lr: 2.8447e-04 gnorm: 0.79 [ 3:36:13<18:33:10] +[titan] 2025-07-09 17:11:14,384 - root - INFO - step: 16270 loss: 17.2115 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6926 global_avg_mtp_loss: 14.5189 +[titan] 2025-07-09 17:11:14,384 - root - INFO - lr: 2.8446e-04 gnorm: 0.78 [ 3:36:17<18:33:06] +[titan] 2025-07-09 17:11:18,290 - root - INFO - step: 16275 loss: 17.0847 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6899 global_avg_mtp_loss: 14.3947 +[titan] 2025-07-09 17:11:18,291 - root - INFO - lr: 2.8445e-04 gnorm: 0.76 [ 3:36:21<18:33:01] +[titan] 2025-07-09 17:11:22,241 - root - INFO - step: 16280 loss: 17.2711 memory: 44.58GiB(31.99%) tps: 82,961 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.7264 global_avg_mtp_loss: 14.5447 +[titan] 2025-07-09 17:11:22,241 - root - INFO - lr: 2.8444e-04 gnorm: 0.86 [ 3:36:25<18:32:57] +[titan] 2025-07-09 17:11:26,183 - root - INFO - step: 16285 loss: 17.1725 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.6879 global_avg_mtp_loss: 14.4846 +[titan] 2025-07-09 17:11:26,183 - root - INFO - lr: 2.8443e-04 gnorm: 0.81 [ 3:36:29<18:32:53] +[titan] 2025-07-09 17:11:30,141 - root - INFO - step: 16290 loss: 16.8692 memory: 44.58GiB(31.99%) tps: 82,787 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2372 +[titan] 2025-07-09 17:11:30,141 - root - INFO - lr: 2.8442e-04 gnorm: 0.88 [ 3:36:33<18:32:49] +[titan] 2025-07-09 17:11:34,061 - root - INFO - step: 16295 loss: 17.0450 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.3787 +[titan] 2025-07-09 17:11:34,062 - root - INFO - lr: 2.8441e-04 gnorm: 0.83 [ 3:36:37<18:32:44] +[titan] 2025-07-09 17:11:37,210 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:11:37,995 - root - INFO - step: 16300 loss: 17.0196 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6651 global_avg_mtp_loss: 14.3545 +[titan] 2025-07-09 17:11:37,996 - root - INFO - lr: 2.8440e-04 gnorm: 0.77 [ 3:36:41<18:32:40] +[titan] 2025-07-09 17:11:41,906 - root - INFO - step: 16305 loss: 17.0879 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6884 global_avg_mtp_loss: 14.3996 +[titan] 2025-07-09 17:11:41,906 - root - INFO - lr: 2.8439e-04 gnorm: 0.81 [ 3:36:45<18:32:36] +[titan] 2025-07-09 17:11:45,815 - root - INFO - step: 16310 loss: 17.2259 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.7068 global_avg_mtp_loss: 14.5191 +[titan] 2025-07-09 17:11:45,816 - root - INFO - lr: 2.8438e-04 gnorm: 0.79 [ 3:36:48<18:32:31] +[titan] 2025-07-09 17:11:49,778 - root - INFO - step: 16315 loss: 17.0361 memory: 44.58GiB(31.99%) tps: 82,699 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.6557 global_avg_mtp_loss: 14.3804 +[titan] 2025-07-09 17:11:49,778 - root - INFO - lr: 2.8437e-04 gnorm: 0.80 [ 3:36:52<18:32:27] +[titan] 2025-07-09 17:11:53,703 - root - INFO - step: 16320 loss: 17.1173 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6690 global_avg_mtp_loss: 14.4483 +[titan] 2025-07-09 17:11:53,703 - root - INFO - lr: 2.8436e-04 gnorm: 0.89 [ 3:36:56<18:32:23] +[titan] 2025-07-09 17:11:57,646 - root - INFO - step: 16325 loss: 17.0225 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6654 global_avg_mtp_loss: 14.3571 +[titan] 2025-07-09 17:11:57,646 - root - INFO - lr: 2.8435e-04 gnorm: 0.82 [ 3:37:00<18:32:19] +[titan] 2025-07-09 17:12:01,571 - root - INFO - step: 16330 loss: 17.1105 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.6797 global_avg_mtp_loss: 14.4308 +[titan] 2025-07-09 17:12:01,572 - root - INFO - lr: 2.8434e-04 gnorm: 0.80 [ 3:37:04<18:32:14] +[titan] 2025-07-09 17:12:05,525 - root - INFO - step: 16335 loss: 17.0680 memory: 44.58GiB(31.99%) tps: 82,881 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.6568 global_avg_mtp_loss: 14.4111 +[titan] 2025-07-09 17:12:05,526 - root - INFO - lr: 2.8433e-04 gnorm: 0.83 [ 3:37:08<18:32:10] +[titan] 2025-07-09 17:12:09,450 - root - INFO - step: 16340 loss: 17.2726 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.7098 global_avg_mtp_loss: 14.5628 +[titan] 2025-07-09 17:12:09,451 - root - INFO - lr: 2.8432e-04 gnorm: 0.78 [ 3:37:12<18:32:06] +[titan] 2025-07-09 17:12:13,401 - root - INFO - step: 16345 loss: 16.9862 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6544 global_avg_mtp_loss: 14.3319 +[titan] 2025-07-09 17:12:13,401 - root - INFO - lr: 2.8431e-04 gnorm: 0.78 [ 3:37:16<18:32:02] +[titan] 2025-07-09 17:12:16,746 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:12:17,541 - root - INFO - step: 16350 loss: 17.1054 memory: 44.58GiB(31.99%) tps: 79,152 tflops: 273.17 mfu: 27.62% global_avg_ntp_loss: 2.6765 global_avg_mtp_loss: 14.4289 +[titan] 2025-07-09 17:12:17,542 - root - INFO - lr: 2.8430e-04 gnorm: 0.83 [ 3:37:20<18:31:58] +[titan] 2025-07-09 17:12:21,477 - root - INFO - step: 16355 loss: 17.0970 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4215 +[titan] 2025-07-09 17:12:21,477 - root - INFO - lr: 2.8429e-04 gnorm: 0.79 [ 3:37:24<18:31:54] +[titan] 2025-07-09 17:12:25,408 - root - INFO - step: 16360 loss: 17.3648 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.7219 global_avg_mtp_loss: 14.6429 +[titan] 2025-07-09 17:12:25,409 - root - INFO - lr: 2.8428e-04 gnorm: 0.80 [ 3:37:28<18:31:50] +[titan] 2025-07-09 17:12:29,326 - root - INFO - step: 16365 loss: 16.9714 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3295 +[titan] 2025-07-09 17:12:29,326 - root - INFO - lr: 2.8427e-04 gnorm: 0.76 [ 3:37:32<18:31:46] +[titan] 2025-07-09 17:12:33,238 - root - INFO - step: 16370 loss: 17.2276 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6998 global_avg_mtp_loss: 14.5278 +[titan] 2025-07-09 17:12:33,238 - root - INFO - lr: 2.8426e-04 gnorm: 0.81 [ 3:37:36<18:31:41] +[titan] 2025-07-09 17:12:37,172 - root - INFO - step: 16375 loss: 17.2935 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6953 global_avg_mtp_loss: 14.5982 +[titan] 2025-07-09 17:12:37,172 - root - INFO - lr: 2.8425e-04 gnorm: 0.78 [ 3:37:40<18:31:37] +[titan] 2025-07-09 17:12:41,082 - root - INFO - step: 16380 loss: 17.3231 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.7137 global_avg_mtp_loss: 14.6095 +[titan] 2025-07-09 17:12:41,082 - root - INFO - lr: 2.8424e-04 gnorm: 0.79 [ 3:37:44<18:31:33] +[titan] 2025-07-09 17:12:44,467 - root - INFO - Dumping profiler traces at step 16384 +[titan] 2025-07-09 17:12:44,498 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:12:45,286 - root - INFO - step: 16385 loss: 17.0440 memory: 44.58GiB(31.99%) tps: 77,951 tflops: 269.02 mfu: 27.20% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.3839 +[titan] 2025-07-09 17:12:45,286 - root - INFO - lr: 2.8423e-04 gnorm: 0.80 [ 3:37:48<18:31:30] +[titan] 2025-07-09 17:12:49,200 - root - INFO - step: 16390 loss: 16.9828 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6519 global_avg_mtp_loss: 14.3309 +[titan] 2025-07-09 17:12:49,201 - root - INFO - lr: 2.8422e-04 gnorm: 0.78 [ 3:37:52<18:31:25] +[titan] 2025-07-09 17:12:53,108 - root - INFO - step: 16395 loss: 17.0207 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6536 global_avg_mtp_loss: 14.3671 +[titan] 2025-07-09 17:12:53,108 - root - INFO - lr: 2.8421e-04 gnorm: 0.83 [ 3:37:56<18:31:21] +[titan] 2025-07-09 17:12:56,219 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:12:57,012 - root - INFO - step: 16400 loss: 17.1022 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6644 global_avg_mtp_loss: 14.4378 +[titan] 2025-07-09 17:12:57,013 - root - INFO - lr: 2.8420e-04 gnorm: 0.91 [ 3:38:00<18:31:17] +[titan] 2025-07-09 17:13:00,909 - root - INFO - step: 16405 loss: 16.6848 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.0934 +[titan] 2025-07-09 17:13:00,909 - root - INFO - lr: 2.8419e-04 gnorm: 0.80 [ 3:38:04<18:31:12] +[titan] 2025-07-09 17:13:04,811 - root - INFO - step: 16410 loss: 17.0984 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6800 global_avg_mtp_loss: 14.4184 +[titan] 2025-07-09 17:13:04,812 - root - INFO - lr: 2.8418e-04 gnorm: 0.95 [ 3:38:07<18:31:08] +[titan] 2025-07-09 17:13:08,718 - root - INFO - step: 16415 loss: 17.0147 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6497 global_avg_mtp_loss: 14.3650 +[titan] 2025-07-09 17:13:08,718 - root - INFO - lr: 2.8417e-04 gnorm: 0.90 [ 3:38:11<18:31:03] +[titan] 2025-07-09 17:13:12,626 - root - INFO - step: 16420 loss: 17.2943 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.7088 global_avg_mtp_loss: 14.5855 +[titan] 2025-07-09 17:13:12,627 - root - INFO - lr: 2.8416e-04 gnorm: 0.84 [ 3:38:15<18:30:59] +[titan] 2025-07-09 17:13:16,559 - root - INFO - step: 16425 loss: 16.8553 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.6248 global_avg_mtp_loss: 14.2304 +[titan] 2025-07-09 17:13:16,559 - root - INFO - lr: 2.8415e-04 gnorm: 0.76 [ 3:38:19<18:30:55] +[titan] 2025-07-09 17:13:20,471 - root - INFO - step: 16430 loss: 16.9452 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.2979 +[titan] 2025-07-09 17:13:20,471 - root - INFO - lr: 2.8414e-04 gnorm: 0.80 [ 3:38:23<18:30:50] +[titan] 2025-07-09 17:13:24,419 - root - INFO - step: 16435 loss: 17.0186 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.6522 global_avg_mtp_loss: 14.3665 +[titan] 2025-07-09 17:13:24,419 - root - INFO - lr: 2.8413e-04 gnorm: 0.83 [ 3:38:27<18:30:46] +[titan] 2025-07-09 17:13:28,357 - root - INFO - step: 16440 loss: 17.0816 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.6799 global_avg_mtp_loss: 14.4018 +[titan] 2025-07-09 17:13:28,357 - root - INFO - lr: 2.8412e-04 gnorm: 0.83 [ 3:38:31<18:30:42] +[titan] 2025-07-09 17:13:32,281 - root - INFO - step: 16445 loss: 16.9469 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6450 global_avg_mtp_loss: 14.3019 +[titan] 2025-07-09 17:13:32,281 - root - INFO - lr: 2.8411e-04 gnorm: 0.78 [ 3:38:35<18:30:37] +[titan] 2025-07-09 17:13:35,623 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:13:36,409 - root - INFO - step: 16450 loss: 16.8288 memory: 44.58GiB(31.99%) tps: 79,382 tflops: 273.96 mfu: 27.70% global_avg_ntp_loss: 2.6180 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 17:13:36,409 - root - INFO - lr: 2.8410e-04 gnorm: 0.86 [ 3:38:39<18:30:34] +[titan] 2025-07-09 17:13:40,341 - root - INFO - step: 16455 loss: 17.0495 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.3786 +[titan] 2025-07-09 17:13:40,341 - root - INFO - lr: 2.8409e-04 gnorm: 0.84 [ 3:38:43<18:30:30] +[titan] 2025-07-09 17:13:44,288 - root - INFO - step: 16460 loss: 16.7915 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6229 global_avg_mtp_loss: 14.1686 +[titan] 2025-07-09 17:13:44,288 - root - INFO - lr: 2.8408e-04 gnorm: 0.82 [ 3:38:47<18:30:26] +[titan] 2025-07-09 17:13:48,202 - root - INFO - step: 16465 loss: 17.1464 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6876 global_avg_mtp_loss: 14.4588 +[titan] 2025-07-09 17:13:48,202 - root - INFO - lr: 2.8407e-04 gnorm: 0.88 [ 3:38:51<18:30:21] +[titan] 2025-07-09 17:13:52,186 - root - INFO - step: 16470 loss: 16.9153 memory: 44.58GiB(31.99%) tps: 82,248 tflops: 283.85 mfu: 28.70% global_avg_ntp_loss: 2.6408 global_avg_mtp_loss: 14.2745 +[titan] 2025-07-09 17:13:52,186 - root - INFO - lr: 2.8406e-04 gnorm: 0.90 [ 3:38:55<18:30:17] +[titan] 2025-07-09 17:13:56,125 - root - INFO - step: 16475 loss: 17.0492 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6591 global_avg_mtp_loss: 14.3902 +[titan] 2025-07-09 17:13:56,125 - root - INFO - lr: 2.8405e-04 gnorm: 0.79 [ 3:38:59<18:30:13] +[titan] 2025-07-09 17:14:00,045 - root - INFO - step: 16480 loss: 17.1710 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4816 +[titan] 2025-07-09 17:14:00,045 - root - INFO - lr: 2.8403e-04 gnorm: 0.78 [ 3:39:03<18:30:09] +[titan] 2025-07-09 17:14:03,993 - root - INFO - step: 16485 loss: 17.2005 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.6934 global_avg_mtp_loss: 14.5071 +[titan] 2025-07-09 17:14:03,993 - root - INFO - lr: 2.8402e-04 gnorm: 0.78 [ 3:39:07<18:30:05] +[titan] 2025-07-09 17:14:07,896 - root - INFO - step: 16490 loss: 17.0974 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6676 global_avg_mtp_loss: 14.4298 +[titan] 2025-07-09 17:14:07,896 - root - INFO - lr: 2.8401e-04 gnorm: 0.79 [ 3:39:11<18:30:00] +[titan] 2025-07-09 17:14:11,793 - root - INFO - step: 16495 loss: 17.1184 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.7322 global_avg_mtp_loss: 14.3862 +[titan] 2025-07-09 17:14:11,794 - root - INFO - lr: 2.8400e-04 gnorm: 1.79 [ 3:39:14<18:29:56] +[titan] 2025-07-09 17:14:14,901 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:14:15,698 - root - INFO - step: 16500 loss: 17.1420 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6785 global_avg_mtp_loss: 14.4635 +[titan] 2025-07-09 17:14:15,698 - root - INFO - lr: 2.8399e-04 gnorm: 0.84 [ 3:39:18<18:29:51] +[titan] 2025-07-09 17:14:19,612 - root - INFO - step: 16505 loss: 17.3032 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.7183 global_avg_mtp_loss: 14.5848 +[titan] 2025-07-09 17:14:19,612 - root - INFO - lr: 2.8398e-04 gnorm: 0.85 [ 3:39:22<18:29:47] +[titan] 2025-07-09 17:14:23,555 - root - INFO - step: 16510 loss: 16.9642 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.3330 +[titan] 2025-07-09 17:14:23,555 - root - INFO - lr: 2.8397e-04 gnorm: 0.88 [ 3:39:26<18:29:43] +[titan] 2025-07-09 17:14:27,481 - root - INFO - step: 16515 loss: 17.1459 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6675 global_avg_mtp_loss: 14.4784 +[titan] 2025-07-09 17:14:27,481 - root - INFO - lr: 2.8396e-04 gnorm: 0.92 [ 3:39:30<18:29:38] +[titan] 2025-07-09 17:14:31,385 - root - INFO - step: 16520 loss: 16.8564 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.6272 global_avg_mtp_loss: 14.2291 +[titan] 2025-07-09 17:14:31,385 - root - INFO - lr: 2.8395e-04 gnorm: 0.85 [ 3:39:34<18:29:34] +[titan] 2025-07-09 17:14:35,281 - root - INFO - step: 16525 loss: 16.8970 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6462 global_avg_mtp_loss: 14.2508 +[titan] 2025-07-09 17:14:35,281 - root - INFO - lr: 2.8394e-04 gnorm: 0.83 [ 3:39:38<18:29:30] +[titan] 2025-07-09 17:14:39,181 - root - INFO - step: 16530 loss: 16.8023 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6163 global_avg_mtp_loss: 14.1860 +[titan] 2025-07-09 17:14:39,182 - root - INFO - lr: 2.8393e-04 gnorm: 0.83 [ 3:39:42<18:29:25] +[titan] 2025-07-09 17:14:43,072 - root - INFO - step: 16535 loss: 16.8815 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6257 global_avg_mtp_loss: 14.2558 +[titan] 2025-07-09 17:14:43,073 - root - INFO - lr: 2.8392e-04 gnorm: 0.90 [ 3:39:46<18:29:21] +[titan] 2025-07-09 17:14:46,975 - root - INFO - step: 16540 loss: 16.9338 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.2971 +[titan] 2025-07-09 17:14:46,976 - root - INFO - lr: 2.8391e-04 gnorm: 0.86 [ 3:39:50<18:29:16] +[titan] 2025-07-09 17:14:50,870 - root - INFO - step: 16545 loss: 17.1487 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6848 global_avg_mtp_loss: 14.4639 +[titan] 2025-07-09 17:14:50,870 - root - INFO - lr: 2.8390e-04 gnorm: 0.86 [ 3:39:54<18:29:12] +[titan] 2025-07-09 17:14:54,037 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:14:54,822 - root - INFO - step: 16550 loss: 17.0735 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.3940 +[titan] 2025-07-09 17:14:54,822 - root - INFO - lr: 2.8389e-04 gnorm: 0.81 [ 3:39:57<18:29:08] +[titan] 2025-07-09 17:14:58,732 - root - INFO - step: 16555 loss: 16.8543 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.2343 +[titan] 2025-07-09 17:14:58,733 - root - INFO - lr: 2.8388e-04 gnorm: 0.81 [ 3:40:01<18:29:03] +[titan] 2025-07-09 17:15:02,653 - root - INFO - step: 16560 loss: 16.9926 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6533 global_avg_mtp_loss: 14.3393 +[titan] 2025-07-09 17:15:02,654 - root - INFO - lr: 2.8387e-04 gnorm: 0.92 [ 3:40:05<18:28:59] +[titan] 2025-07-09 17:15:06,557 - root - INFO - step: 16565 loss: 16.8967 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6272 global_avg_mtp_loss: 14.2695 +[titan] 2025-07-09 17:15:06,557 - root - INFO - lr: 2.8386e-04 gnorm: 0.83 [ 3:40:09<18:28:54] +[titan] 2025-07-09 17:15:10,466 - root - INFO - step: 16570 loss: 17.0891 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.4376 +[titan] 2025-07-09 17:15:10,466 - root - INFO - lr: 2.8385e-04 gnorm: 0.88 [ 3:40:13<18:28:50] +[titan] 2025-07-09 17:15:14,384 - root - INFO - step: 16575 loss: 16.8249 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6303 global_avg_mtp_loss: 14.1946 +[titan] 2025-07-09 17:15:14,385 - root - INFO - lr: 2.8384e-04 gnorm: 0.79 [ 3:40:17<18:28:46] +[titan] 2025-07-09 17:15:18,329 - root - INFO - step: 16580 loss: 17.0366 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6618 global_avg_mtp_loss: 14.3748 +[titan] 2025-07-09 17:15:18,329 - root - INFO - lr: 2.8383e-04 gnorm: 0.82 [ 3:40:21<18:28:42] +[titan] 2025-07-09 17:15:22,233 - root - INFO - step: 16585 loss: 17.2260 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.7033 global_avg_mtp_loss: 14.5227 +[titan] 2025-07-09 17:15:22,233 - root - INFO - lr: 2.8382e-04 gnorm: 0.81 [ 3:40:25<18:28:37] +[titan] 2025-07-09 17:15:26,127 - root - INFO - step: 16590 loss: 16.9423 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.6550 global_avg_mtp_loss: 14.2873 +[titan] 2025-07-09 17:15:26,127 - root - INFO - lr: 2.8381e-04 gnorm: 0.87 [ 3:40:29<18:28:33] +[titan] 2025-07-09 17:15:30,060 - root - INFO - step: 16595 loss: 17.2059 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6943 global_avg_mtp_loss: 14.5116 +[titan] 2025-07-09 17:15:30,060 - root - INFO - lr: 2.8380e-04 gnorm: 0.80 [ 3:40:33<18:28:28] +[titan] 2025-07-09 17:15:33,161 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:15:33,954 - root - INFO - step: 16600 loss: 17.0000 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6552 global_avg_mtp_loss: 14.3447 +[titan] 2025-07-09 17:15:33,954 - root - INFO - lr: 2.8379e-04 gnorm: 0.83 [ 3:40:37<18:28:24] +[titan] 2025-07-09 17:15:37,864 - root - INFO - step: 16605 loss: 16.9942 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6470 global_avg_mtp_loss: 14.3472 +[titan] 2025-07-09 17:15:37,864 - root - INFO - lr: 2.8378e-04 gnorm: 0.80 [ 3:40:41<18:28:20] +[titan] 2025-07-09 17:15:41,788 - root - INFO - step: 16610 loss: 16.9735 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6497 global_avg_mtp_loss: 14.3239 +[titan] 2025-07-09 17:15:41,789 - root - INFO - lr: 2.8377e-04 gnorm: 0.80 [ 3:40:44<18:28:15] +[titan] 2025-07-09 17:15:45,702 - root - INFO - step: 16615 loss: 17.0769 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6696 global_avg_mtp_loss: 14.4074 +[titan] 2025-07-09 17:15:45,702 - root - INFO - lr: 2.8376e-04 gnorm: 0.84 [ 3:40:48<18:28:11] +[titan] 2025-07-09 17:15:49,606 - root - INFO - step: 16620 loss: 17.0890 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6588 global_avg_mtp_loss: 14.4302 +[titan] 2025-07-09 17:15:49,607 - root - INFO - lr: 2.8375e-04 gnorm: 0.81 [ 3:40:52<18:28:07] +[titan] 2025-07-09 17:15:53,503 - root - INFO - step: 16625 loss: 17.3705 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.7200 global_avg_mtp_loss: 14.6505 +[titan] 2025-07-09 17:15:53,503 - root - INFO - lr: 2.8374e-04 gnorm: 0.86 [ 3:40:56<18:28:02] +[titan] 2025-07-09 17:15:57,414 - root - INFO - step: 16630 loss: 16.9314 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6522 global_avg_mtp_loss: 14.2792 +[titan] 2025-07-09 17:15:57,415 - root - INFO - lr: 2.8373e-04 gnorm: 0.92 [ 3:41:00<18:27:58] +[titan] 2025-07-09 17:16:01,308 - root - INFO - step: 16635 loss: 17.2839 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.7135 global_avg_mtp_loss: 14.5705 +[titan] 2025-07-09 17:16:01,308 - root - INFO - lr: 2.8372e-04 gnorm: 0.84 [ 3:41:04<18:27:53] +[titan] 2025-07-09 17:16:05,204 - root - INFO - step: 16640 loss: 17.2171 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6891 global_avg_mtp_loss: 14.5280 +[titan] 2025-07-09 17:16:05,204 - root - INFO - lr: 2.8371e-04 gnorm: 0.85 [ 3:41:08<18:27:49] +[titan] 2025-07-09 17:16:09,105 - root - INFO - step: 16645 loss: 16.7922 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.1858 +[titan] 2025-07-09 17:16:09,105 - root - INFO - lr: 2.8370e-04 gnorm: 0.87 [ 3:41:12<18:27:44] +[titan] 2025-07-09 17:16:12,229 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:16:13,017 - root - INFO - step: 16650 loss: 17.0461 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6665 global_avg_mtp_loss: 14.3796 +[titan] 2025-07-09 17:16:13,018 - root - INFO - lr: 2.8369e-04 gnorm: 0.79 [ 3:41:16<18:27:40] +[titan] 2025-07-09 17:16:16,928 - root - INFO - step: 16655 loss: 17.0285 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6672 global_avg_mtp_loss: 14.3614 +[titan] 2025-07-09 17:16:16,928 - root - INFO - lr: 2.8368e-04 gnorm: 0.82 [ 3:41:20<18:27:36] +[titan] 2025-07-09 17:16:20,821 - root - INFO - step: 16660 loss: 16.9985 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6521 global_avg_mtp_loss: 14.3464 +[titan] 2025-07-09 17:16:20,821 - root - INFO - lr: 2.8367e-04 gnorm: 0.88 [ 3:41:23<18:27:31] +[titan] 2025-07-09 17:16:24,736 - root - INFO - step: 16665 loss: 16.9052 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6362 global_avg_mtp_loss: 14.2690 +[titan] 2025-07-09 17:16:24,736 - root - INFO - lr: 2.8366e-04 gnorm: 0.90 [ 3:41:27<18:27:27] +[titan] 2025-07-09 17:16:28,654 - root - INFO - step: 16670 loss: 16.9643 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6561 global_avg_mtp_loss: 14.3082 +[titan] 2025-07-09 17:16:28,654 - root - INFO - lr: 2.8365e-04 gnorm: 0.80 [ 3:41:31<18:27:23] +[titan] 2025-07-09 17:16:32,572 - root - INFO - step: 16675 loss: 16.9869 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6521 global_avg_mtp_loss: 14.3348 +[titan] 2025-07-09 17:16:32,573 - root - INFO - lr: 2.8364e-04 gnorm: 0.89 [ 3:41:35<18:27:18] +[titan] 2025-07-09 17:16:36,510 - root - INFO - step: 16680 loss: 17.1231 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.6725 global_avg_mtp_loss: 14.4506 +[titan] 2025-07-09 17:16:36,511 - root - INFO - lr: 2.8363e-04 gnorm: 0.84 [ 3:41:39<18:27:14] +[titan] 2025-07-09 17:16:40,446 - root - INFO - step: 16685 loss: 16.9765 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6438 global_avg_mtp_loss: 14.3327 +[titan] 2025-07-09 17:16:40,446 - root - INFO - lr: 2.8362e-04 gnorm: 0.75 [ 3:41:43<18:27:10] +[titan] 2025-07-09 17:16:44,366 - root - INFO - step: 16690 loss: 16.7572 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.1469 +[titan] 2025-07-09 17:16:44,367 - root - INFO - lr: 2.8361e-04 gnorm: 0.87 [ 3:41:47<18:27:05] +[titan] 2025-07-09 17:16:48,291 - root - INFO - step: 16695 loss: 16.9502 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.3139 +[titan] 2025-07-09 17:16:48,291 - root - INFO - lr: 2.8360e-04 gnorm: 0.81 [ 3:41:51<18:27:01] +[titan] 2025-07-09 17:16:51,431 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:16:52,247 - root - INFO - step: 16700 loss: 17.1865 memory: 44.58GiB(31.99%) tps: 82,835 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.6940 global_avg_mtp_loss: 14.4925 +[titan] 2025-07-09 17:16:52,247 - root - INFO - lr: 2.8359e-04 gnorm: 0.81 [ 3:41:55<18:26:57] +[titan] 2025-07-09 17:16:56,183 - root - INFO - step: 16705 loss: 16.9008 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.2689 +[titan] 2025-07-09 17:16:56,183 - root - INFO - lr: 2.8358e-04 gnorm: 0.80 [ 3:41:59<18:26:53] +[titan] 2025-07-09 17:17:00,106 - root - INFO - step: 16710 loss: 17.1045 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6850 global_avg_mtp_loss: 14.4194 +[titan] 2025-07-09 17:17:00,106 - root - INFO - lr: 2.8357e-04 gnorm: 0.78 [ 3:42:03<18:26:48] +[titan] 2025-07-09 17:17:04,016 - root - INFO - step: 16715 loss: 17.1158 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6831 global_avg_mtp_loss: 14.4327 +[titan] 2025-07-09 17:17:04,016 - root - INFO - lr: 2.8356e-04 gnorm: 0.76 [ 3:42:07<18:26:44] +[titan] 2025-07-09 17:17:07,909 - root - INFO - step: 16720 loss: 17.1966 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.7049 global_avg_mtp_loss: 14.4917 +[titan] 2025-07-09 17:17:07,909 - root - INFO - lr: 2.8355e-04 gnorm: 0.88 [ 3:42:11<18:26:40] +[titan] 2025-07-09 17:17:11,822 - root - INFO - step: 16725 loss: 16.9851 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.3385 +[titan] 2025-07-09 17:17:11,823 - root - INFO - lr: 2.8354e-04 gnorm: 0.81 [ 3:42:14<18:26:35] +[titan] 2025-07-09 17:17:15,746 - root - INFO - step: 16730 loss: 17.2083 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.15% global_avg_ntp_loss: 2.6880 global_avg_mtp_loss: 14.5203 +[titan] 2025-07-09 17:17:15,746 - root - INFO - lr: 2.8353e-04 gnorm: 0.82 [ 3:42:18<18:26:31] +[titan] 2025-07-09 17:17:19,688 - root - INFO - step: 16735 loss: 17.0662 memory: 44.58GiB(31.99%) tps: 83,140 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.6712 global_avg_mtp_loss: 14.3950 +[titan] 2025-07-09 17:17:19,688 - root - INFO - lr: 2.8352e-04 gnorm: 0.89 [ 3:42:22<18:26:27] +[titan] 2025-07-09 17:17:23,583 - root - INFO - step: 16740 loss: 16.9204 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6352 global_avg_mtp_loss: 14.2852 +[titan] 2025-07-09 17:17:23,583 - root - INFO - lr: 2.8351e-04 gnorm: 0.83 [ 3:42:26<18:26:22] +[titan] 2025-07-09 17:17:27,500 - root - INFO - step: 16745 loss: 17.0051 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.3456 +[titan] 2025-07-09 17:17:27,501 - root - INFO - lr: 2.8350e-04 gnorm: 0.81 [ 3:42:30<18:26:18] +[titan] 2025-07-09 17:17:30,617 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:17:31,408 - root - INFO - step: 16750 loss: 17.0701 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6737 global_avg_mtp_loss: 14.3964 +[titan] 2025-07-09 17:17:31,408 - root - INFO - lr: 2.8348e-04 gnorm: 0.82 [ 3:42:34<18:26:14] +[titan] 2025-07-09 17:17:35,310 - root - INFO - step: 16755 loss: 16.9573 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6461 global_avg_mtp_loss: 14.3112 +[titan] 2025-07-09 17:17:35,310 - root - INFO - lr: 2.8347e-04 gnorm: 0.85 [ 3:42:38<18:26:09] +[titan] 2025-07-09 17:17:39,226 - root - INFO - step: 16760 loss: 17.1157 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6758 global_avg_mtp_loss: 14.4399 +[titan] 2025-07-09 17:17:39,227 - root - INFO - lr: 2.8346e-04 gnorm: 0.81 [ 3:42:42<18:26:05] +[titan] 2025-07-09 17:17:43,117 - root - INFO - step: 16765 loss: 17.2578 memory: 44.58GiB(31.99%) tps: 84,235 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.7163 global_avg_mtp_loss: 14.5415 +[titan] 2025-07-09 17:17:43,117 - root - INFO - lr: 2.8345e-04 gnorm: 0.78 [ 3:42:46<18:26:00] +[titan] 2025-07-09 17:17:47,013 - root - INFO - step: 16770 loss: 17.0112 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6620 global_avg_mtp_loss: 14.3492 +[titan] 2025-07-09 17:17:47,014 - root - INFO - lr: 2.8344e-04 gnorm: 0.84 [ 3:42:50<18:25:56] +[titan] 2025-07-09 17:17:50,957 - root - INFO - step: 16775 loss: 17.0906 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.6640 global_avg_mtp_loss: 14.4266 +[titan] 2025-07-09 17:17:50,958 - root - INFO - lr: 2.8343e-04 gnorm: 0.84 [ 3:42:54<18:25:52] +[titan] 2025-07-09 17:17:54,864 - root - INFO - step: 16780 loss: 16.8490 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.6246 global_avg_mtp_loss: 14.2244 +[titan] 2025-07-09 17:17:54,864 - root - INFO - lr: 2.8342e-04 gnorm: 0.76 [ 3:42:57<18:25:47] +[titan] 2025-07-09 17:17:58,775 - root - INFO - step: 16785 loss: 16.9353 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6365 global_avg_mtp_loss: 14.2988 +[titan] 2025-07-09 17:17:58,775 - root - INFO - lr: 2.8341e-04 gnorm: 0.85 [ 3:43:01<18:25:43] +[titan] 2025-07-09 17:18:02,676 - root - INFO - step: 16790 loss: 17.0126 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6576 global_avg_mtp_loss: 14.3551 +[titan] 2025-07-09 17:18:02,677 - root - INFO - lr: 2.8340e-04 gnorm: 0.81 [ 3:43:05<18:25:39] +[titan] 2025-07-09 17:18:06,636 - root - INFO - step: 16795 loss: 16.9611 memory: 44.58GiB(31.99%) tps: 82,763 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.6506 global_avg_mtp_loss: 14.3104 +[titan] 2025-07-09 17:18:06,636 - root - INFO - lr: 2.8339e-04 gnorm: 0.83 [ 3:43:09<18:25:34] +[titan] 2025-07-09 17:18:09,750 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:18:10,537 - root - INFO - step: 16800 loss: 17.1422 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.4652 +[titan] 2025-07-09 17:18:10,537 - root - INFO - lr: 2.8338e-04 gnorm: 0.82 [ 3:43:13<18:25:30] +[titan] 2025-07-09 17:18:14,471 - root - INFO - step: 16805 loss: 16.7791 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6058 global_avg_mtp_loss: 14.1733 +[titan] 2025-07-09 17:18:14,471 - root - INFO - lr: 2.8337e-04 gnorm: 0.86 [ 3:43:17<18:25:26] +[titan] 2025-07-09 17:18:18,394 - root - INFO - step: 16810 loss: 17.0971 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6746 global_avg_mtp_loss: 14.4225 +[titan] 2025-07-09 17:18:18,394 - root - INFO - lr: 2.8336e-04 gnorm: 0.88 [ 3:43:21<18:25:21] +[titan] 2025-07-09 17:18:22,292 - root - INFO - step: 16815 loss: 17.1172 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6714 global_avg_mtp_loss: 14.4458 +[titan] 2025-07-09 17:18:22,292 - root - INFO - lr: 2.8335e-04 gnorm: 0.85 [ 3:43:25<18:25:17] +[titan] 2025-07-09 17:18:26,226 - root - INFO - step: 16820 loss: 17.0740 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.6673 global_avg_mtp_loss: 14.4067 +[titan] 2025-07-09 17:18:26,226 - root - INFO - lr: 2.8334e-04 gnorm: 0.87 [ 3:43:29<18:25:13] +[titan] 2025-07-09 17:18:30,143 - root - INFO - step: 16825 loss: 17.1121 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6734 global_avg_mtp_loss: 14.4386 +[titan] 2025-07-09 17:18:30,143 - root - INFO - lr: 2.8333e-04 gnorm: 0.94 [ 3:43:33<18:25:08] +[titan] 2025-07-09 17:18:34,065 - root - INFO - step: 16830 loss: 17.0073 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.6650 global_avg_mtp_loss: 14.3423 +[titan] 2025-07-09 17:18:34,065 - root - INFO - lr: 2.8332e-04 gnorm: 0.81 [ 3:43:37<18:25:04] +[titan] 2025-07-09 17:18:37,967 - root - INFO - step: 16835 loss: 16.9086 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.2723 +[titan] 2025-07-09 17:18:37,967 - root - INFO - lr: 2.8331e-04 gnorm: 0.86 [ 3:43:41<18:25:00] +[titan] 2025-07-09 17:18:41,910 - root - INFO - step: 16840 loss: 16.9687 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6716 global_avg_mtp_loss: 14.2971 +[titan] 2025-07-09 17:18:41,910 - root - INFO - lr: 2.8330e-04 gnorm: 0.91 [ 3:43:45<18:24:56] +[titan] 2025-07-09 17:18:45,841 - root - INFO - step: 16845 loss: 16.9771 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6439 global_avg_mtp_loss: 14.3332 +[titan] 2025-07-09 17:18:45,841 - root - INFO - lr: 2.8329e-04 gnorm: 0.84 [ 3:43:48<18:24:51] +[titan] 2025-07-09 17:18:48,963 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:18:49,749 - root - INFO - step: 16850 loss: 17.1708 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.7080 global_avg_mtp_loss: 14.4628 +[titan] 2025-07-09 17:18:49,749 - root - INFO - lr: 2.8328e-04 gnorm: 1.13 [ 3:43:52<18:24:47] +[titan] 2025-07-09 17:18:53,651 - root - INFO - step: 16855 loss: 17.2530 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6901 global_avg_mtp_loss: 14.5628 +[titan] 2025-07-09 17:18:53,651 - root - INFO - lr: 2.8327e-04 gnorm: 0.87 [ 3:43:56<18:24:43] +[titan] 2025-07-09 17:18:57,565 - root - INFO - step: 16860 loss: 17.1132 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.4336 +[titan] 2025-07-09 17:18:57,566 - root - INFO - lr: 2.8326e-04 gnorm: 0.98 [ 3:44:00<18:24:38] +[titan] 2025-07-09 17:19:01,476 - root - INFO - step: 16865 loss: 16.8632 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.2346 +[titan] 2025-07-09 17:19:01,477 - root - INFO - lr: 2.8325e-04 gnorm: 0.81 [ 3:44:04<18:24:34] +[titan] 2025-07-09 17:19:05,399 - root - INFO - step: 16870 loss: 17.1203 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.4420 +[titan] 2025-07-09 17:19:05,399 - root - INFO - lr: 2.8324e-04 gnorm: 0.82 [ 3:44:08<18:24:30] +[titan] 2025-07-09 17:19:09,323 - root - INFO - step: 16875 loss: 16.9317 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.15% global_avg_ntp_loss: 2.6450 global_avg_mtp_loss: 14.2867 +[titan] 2025-07-09 17:19:09,323 - root - INFO - lr: 2.8323e-04 gnorm: 0.85 [ 3:44:12<18:24:25] +[titan] 2025-07-09 17:19:13,239 - root - INFO - step: 16880 loss: 16.9747 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3287 +[titan] 2025-07-09 17:19:13,239 - root - INFO - lr: 2.8322e-04 gnorm: 0.92 [ 3:44:16<18:24:21] +[titan] 2025-07-09 17:19:17,172 - root - INFO - step: 16885 loss: 17.0931 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.6656 global_avg_mtp_loss: 14.4274 +[titan] 2025-07-09 17:19:17,172 - root - INFO - lr: 2.8321e-04 gnorm: 0.84 [ 3:44:20<18:24:17] +[titan] 2025-07-09 17:19:21,080 - root - INFO - step: 16890 loss: 16.7037 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6169 global_avg_mtp_loss: 14.0869 +[titan] 2025-07-09 17:19:21,080 - root - INFO - lr: 2.8320e-04 gnorm: 0.83 [ 3:44:24<18:24:12] +[titan] 2025-07-09 17:19:25,079 - root - INFO - step: 16895 loss: 17.1511 memory: 44.58GiB(31.99%) tps: 81,940 tflops: 282.79 mfu: 28.59% global_avg_ntp_loss: 2.6870 global_avg_mtp_loss: 14.4642 +[titan] 2025-07-09 17:19:25,080 - root - INFO - lr: 2.8319e-04 gnorm: 0.89 [ 3:44:28<18:24:08] +[titan] 2025-07-09 17:19:26,013 - root - INFO - Dumping profiler traces at step 16896 +[titan] 2025-07-09 17:19:26,044 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:19:28,385 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:19:29,175 - root - INFO - step: 16900 loss: 17.0970 memory: 44.58GiB(31.99%) tps: 80,019 tflops: 276.16 mfu: 27.92% global_avg_ntp_loss: 2.6671 global_avg_mtp_loss: 14.4299 +[titan] 2025-07-09 17:19:29,175 - root - INFO - lr: 2.8318e-04 gnorm: 0.80 [ 3:44:32<18:24:05] +[titan] 2025-07-09 17:19:33,088 - root - INFO - step: 16905 loss: 17.1681 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6875 global_avg_mtp_loss: 14.4806 +[titan] 2025-07-09 17:19:33,088 - root - INFO - lr: 2.8317e-04 gnorm: 0.80 [ 3:44:36<18:24:01] +[titan] 2025-07-09 17:19:36,981 - root - INFO - step: 16910 loss: 17.3267 memory: 44.58GiB(31.99%) tps: 84,190 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.7344 global_avg_mtp_loss: 14.5923 +[titan] 2025-07-09 17:19:36,981 - root - INFO - lr: 2.8315e-04 gnorm: 0.87 [ 3:44:40<18:23:56] +[titan] 2025-07-09 17:19:40,907 - root - INFO - step: 16915 loss: 17.2787 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.7146 global_avg_mtp_loss: 14.5641 +[titan] 2025-07-09 17:19:40,907 - root - INFO - lr: 2.8314e-04 gnorm: 0.93 [ 3:44:44<18:23:52] +[titan] 2025-07-09 17:19:44,823 - root - INFO - step: 16920 loss: 17.0463 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6737 global_avg_mtp_loss: 14.3726 +[titan] 2025-07-09 17:19:44,823 - root - INFO - lr: 2.8313e-04 gnorm: 0.83 [ 3:44:47<18:23:48] +[titan] 2025-07-09 17:19:48,717 - root - INFO - step: 16925 loss: 17.1211 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.6642 global_avg_mtp_loss: 14.4569 +[titan] 2025-07-09 17:19:48,717 - root - INFO - lr: 2.8312e-04 gnorm: 0.76 [ 3:44:51<18:23:43] +[titan] 2025-07-09 17:19:52,612 - root - INFO - step: 16930 loss: 17.0543 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6592 global_avg_mtp_loss: 14.3951 +[titan] 2025-07-09 17:19:52,612 - root - INFO - lr: 2.8311e-04 gnorm: 0.76 [ 3:44:55<18:23:39] +[titan] 2025-07-09 17:19:56,506 - root - INFO - step: 16935 loss: 17.0680 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6753 global_avg_mtp_loss: 14.3927 +[titan] 2025-07-09 17:19:56,506 - root - INFO - lr: 2.8310e-04 gnorm: 0.82 [ 3:44:59<18:23:34] +[titan] 2025-07-09 17:20:00,402 - root - INFO - step: 16940 loss: 16.9047 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.2659 +[titan] 2025-07-09 17:20:00,402 - root - INFO - lr: 2.8309e-04 gnorm: 0.85 [ 3:45:03<18:23:30] +[titan] 2025-07-09 17:20:04,300 - root - INFO - step: 16945 loss: 17.1089 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6767 global_avg_mtp_loss: 14.4322 +[titan] 2025-07-09 17:20:04,300 - root - INFO - lr: 2.8308e-04 gnorm: 0.77 [ 3:45:07<18:23:25] +[titan] 2025-07-09 17:20:07,411 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:20:08,202 - root - INFO - step: 16950 loss: 17.1994 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6872 global_avg_mtp_loss: 14.5122 +[titan] 2025-07-09 17:20:08,203 - root - INFO - lr: 2.8307e-04 gnorm: 0.95 [ 3:45:11<18:23:21] +[titan] 2025-07-09 17:20:12,126 - root - INFO - step: 16955 loss: 17.2346 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.5523 +[titan] 2025-07-09 17:20:12,127 - root - INFO - lr: 2.8306e-04 gnorm: 0.92 [ 3:45:15<18:23:17] +[titan] 2025-07-09 17:20:16,054 - root - INFO - step: 16960 loss: 17.3011 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.7201 global_avg_mtp_loss: 14.5810 +[titan] 2025-07-09 17:20:16,055 - root - INFO - lr: 2.8305e-04 gnorm: 0.80 [ 3:45:19<18:23:12] +[titan] 2025-07-09 17:20:19,996 - root - INFO - step: 16965 loss: 17.3855 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.7270 global_avg_mtp_loss: 14.6585 +[titan] 2025-07-09 17:20:19,996 - root - INFO - lr: 2.8304e-04 gnorm: 0.84 [ 3:45:23<18:23:08] +[titan] 2025-07-09 17:20:23,898 - root - INFO - step: 16970 loss: 17.2710 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.5721 +[titan] 2025-07-09 17:20:23,899 - root - INFO - lr: 2.8303e-04 gnorm: 0.83 [ 3:45:27<18:23:04] +[titan] 2025-07-09 17:20:27,832 - root - INFO - step: 16975 loss: 17.0343 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.6381 global_avg_mtp_loss: 14.3962 +[titan] 2025-07-09 17:20:27,832 - root - INFO - lr: 2.8302e-04 gnorm: 0.81 [ 3:45:30<18:23:00] +[titan] 2025-07-09 17:20:31,759 - root - INFO - step: 16980 loss: 17.1080 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6931 global_avg_mtp_loss: 14.4149 +[titan] 2025-07-09 17:20:31,759 - root - INFO - lr: 2.8301e-04 gnorm: 0.83 [ 3:45:34<18:22:55] +[titan] 2025-07-09 17:20:35,689 - root - INFO - step: 16985 loss: 17.0478 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.6538 global_avg_mtp_loss: 14.3940 +[titan] 2025-07-09 17:20:35,690 - root - INFO - lr: 2.8300e-04 gnorm: 0.77 [ 3:45:38<18:22:51] +[titan] 2025-07-09 17:20:39,621 - root - INFO - step: 16990 loss: 17.0654 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6701 global_avg_mtp_loss: 14.3953 +[titan] 2025-07-09 17:20:39,621 - root - INFO - lr: 2.8299e-04 gnorm: 0.84 [ 3:45:42<18:22:47] +[titan] 2025-07-09 17:20:43,520 - root - INFO - step: 16995 loss: 16.9063 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6255 global_avg_mtp_loss: 14.2808 +[titan] 2025-07-09 17:20:43,520 - root - INFO - lr: 2.8298e-04 gnorm: 0.83 [ 3:45:46<18:22:42] +[titan] 2025-07-09 17:20:46,631 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:20:47,424 - root - INFO - step: 17000 loss: 16.7651 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5988 global_avg_mtp_loss: 14.1663 +[titan] 2025-07-09 17:20:47,424 - root - INFO - lr: 2.8297e-04 gnorm: 0.82 [ 3:45:50<18:22:38] +[titan] 2025-07-09 17:20:51,344 - root - INFO - step: 17005 loss: 17.2558 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.7087 global_avg_mtp_loss: 14.5471 +[titan] 2025-07-09 17:20:51,345 - root - INFO - lr: 2.8296e-04 gnorm: 0.82 [ 3:45:54<18:22:34] +[titan] 2025-07-09 17:20:55,278 - root - INFO - step: 17010 loss: 17.0389 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6529 global_avg_mtp_loss: 14.3861 +[titan] 2025-07-09 17:20:55,278 - root - INFO - lr: 2.8295e-04 gnorm: 0.76 [ 3:45:58<18:22:29] +[titan] 2025-07-09 17:20:59,231 - root - INFO - step: 17015 loss: 17.1796 memory: 44.58GiB(31.99%) tps: 82,912 tflops: 286.14 mfu: 28.93% global_avg_ntp_loss: 2.6829 global_avg_mtp_loss: 14.4967 +[titan] 2025-07-09 17:20:59,231 - root - INFO - lr: 2.8294e-04 gnorm: 0.79 [ 3:46:02<18:22:25] +[titan] 2025-07-09 17:21:03,167 - root - INFO - step: 17020 loss: 17.1467 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.6840 global_avg_mtp_loss: 14.4627 +[titan] 2025-07-09 17:21:03,167 - root - INFO - lr: 2.8293e-04 gnorm: 0.91 [ 3:46:06<18:22:21] +[titan] 2025-07-09 17:21:07,099 - root - INFO - step: 17025 loss: 17.0751 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.3906 +[titan] 2025-07-09 17:21:07,099 - root - INFO - lr: 2.8292e-04 gnorm: 0.80 [ 3:46:10<18:22:17] +[titan] 2025-07-09 17:21:10,994 - root - INFO - step: 17030 loss: 17.0033 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6616 global_avg_mtp_loss: 14.3417 +[titan] 2025-07-09 17:21:10,994 - root - INFO - lr: 2.8291e-04 gnorm: 0.81 [ 3:46:14<18:22:12] +[titan] 2025-07-09 17:21:14,918 - root - INFO - step: 17035 loss: 16.9588 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6469 global_avg_mtp_loss: 14.3118 +[titan] 2025-07-09 17:21:14,918 - root - INFO - lr: 2.8289e-04 gnorm: 0.78 [ 3:46:18<18:22:08] +[titan] 2025-07-09 17:21:18,836 - root - INFO - step: 17040 loss: 17.0862 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6617 global_avg_mtp_loss: 14.4246 +[titan] 2025-07-09 17:21:18,836 - root - INFO - lr: 2.8288e-04 gnorm: 0.81 [ 3:46:21<18:22:04] +[titan] 2025-07-09 17:21:22,796 - root - INFO - step: 17045 loss: 16.7954 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.1802 +[titan] 2025-07-09 17:21:22,796 - root - INFO - lr: 2.8287e-04 gnorm: 0.80 [ 3:46:25<18:22:00] +[titan] 2025-07-09 17:21:25,942 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:21:26,739 - root - INFO - step: 17050 loss: 17.0250 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.3736 +[titan] 2025-07-09 17:21:26,739 - root - INFO - lr: 2.8286e-04 gnorm: 0.83 [ 3:46:29<18:21:55] +[titan] 2025-07-09 17:21:30,657 - root - INFO - step: 17055 loss: 17.1053 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.4354 +[titan] 2025-07-09 17:21:30,657 - root - INFO - lr: 2.8285e-04 gnorm: 0.77 [ 3:46:33<18:21:51] +[titan] 2025-07-09 17:21:34,556 - root - INFO - step: 17060 loss: 16.9346 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.3034 +[titan] 2025-07-09 17:21:34,556 - root - INFO - lr: 2.8284e-04 gnorm: 0.82 [ 3:46:37<18:21:47] +[titan] 2025-07-09 17:21:38,502 - root - INFO - step: 17065 loss: 17.2407 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.6944 global_avg_mtp_loss: 14.5463 +[titan] 2025-07-09 17:21:38,502 - root - INFO - lr: 2.8283e-04 gnorm: 0.82 [ 3:46:41<18:21:43] +[titan] 2025-07-09 17:21:42,415 - root - INFO - step: 17070 loss: 16.9746 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.3301 +[titan] 2025-07-09 17:21:42,415 - root - INFO - lr: 2.8282e-04 gnorm: 0.83 [ 3:46:45<18:21:38] +[titan] 2025-07-09 17:21:46,317 - root - INFO - step: 17075 loss: 17.1087 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.6706 global_avg_mtp_loss: 14.4381 +[titan] 2025-07-09 17:21:46,317 - root - INFO - lr: 2.8281e-04 gnorm: 0.81 [ 3:46:49<18:21:34] +[titan] 2025-07-09 17:21:50,262 - root - INFO - step: 17080 loss: 16.8742 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.6302 global_avg_mtp_loss: 14.2440 +[titan] 2025-07-09 17:21:50,262 - root - INFO - lr: 2.8280e-04 gnorm: 0.82 [ 3:46:53<18:21:30] +[titan] 2025-07-09 17:21:54,220 - root - INFO - step: 17085 loss: 17.0399 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.3772 +[titan] 2025-07-09 17:21:54,220 - root - INFO - lr: 2.8279e-04 gnorm: 0.77 [ 3:46:57<18:21:26] +[titan] 2025-07-09 17:21:58,131 - root - INFO - step: 17090 loss: 17.0171 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.3462 +[titan] 2025-07-09 17:21:58,131 - root - INFO - lr: 2.8278e-04 gnorm: 0.77 [ 3:47:01<18:21:21] +[titan] 2025-07-09 17:22:02,027 - root - INFO - step: 17095 loss: 16.8016 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.6122 global_avg_mtp_loss: 14.1894 +[titan] 2025-07-09 17:22:02,027 - root - INFO - lr: 2.8277e-04 gnorm: 0.83 [ 3:47:05<18:21:17] +[titan] 2025-07-09 17:22:05,166 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:22:05,956 - root - INFO - step: 17100 loss: 17.2201 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6882 global_avg_mtp_loss: 14.5319 +[titan] 2025-07-09 17:22:05,956 - root - INFO - lr: 2.8276e-04 gnorm: 0.95 [ 3:47:09<18:21:13] +[titan] 2025-07-09 17:22:09,862 - root - INFO - step: 17105 loss: 17.0401 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6566 global_avg_mtp_loss: 14.3835 +[titan] 2025-07-09 17:22:09,862 - root - INFO - lr: 2.8275e-04 gnorm: 0.81 [ 3:47:12<18:21:08] +[titan] 2025-07-09 17:22:13,806 - root - INFO - step: 17110 loss: 17.2455 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6909 global_avg_mtp_loss: 14.5546 +[titan] 2025-07-09 17:22:13,806 - root - INFO - lr: 2.8274e-04 gnorm: 0.80 [ 3:47:16<18:21:04] +[titan] 2025-07-09 17:22:17,716 - root - INFO - step: 17115 loss: 17.1651 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.7043 global_avg_mtp_loss: 14.4608 +[titan] 2025-07-09 17:22:17,716 - root - INFO - lr: 2.8273e-04 gnorm: 0.79 [ 3:47:20<18:21:00] +[titan] 2025-07-09 17:22:21,659 - root - INFO - step: 17120 loss: 16.8124 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.1914 +[titan] 2025-07-09 17:22:21,660 - root - INFO - lr: 2.8272e-04 gnorm: 0.90 [ 3:47:24<18:20:55] +[titan] 2025-07-09 17:22:25,561 - root - INFO - step: 17125 loss: 16.9922 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6551 global_avg_mtp_loss: 14.3371 +[titan] 2025-07-09 17:22:25,562 - root - INFO - lr: 2.8271e-04 gnorm: 0.83 [ 3:47:28<18:20:51] +[titan] 2025-07-09 17:22:29,471 - root - INFO - step: 17130 loss: 17.0987 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6752 global_avg_mtp_loss: 14.4235 +[titan] 2025-07-09 17:22:29,471 - root - INFO - lr: 2.8270e-04 gnorm: 0.86 [ 3:47:32<18:20:47] +[titan] 2025-07-09 17:22:33,457 - root - INFO - step: 17135 loss: 17.2085 memory: 44.58GiB(31.99%) tps: 82,221 tflops: 283.76 mfu: 28.69% global_avg_ntp_loss: 2.6902 global_avg_mtp_loss: 14.5183 +[titan] 2025-07-09 17:22:33,457 - root - INFO - lr: 2.8269e-04 gnorm: 0.89 [ 3:47:36<18:20:43] +[titan] 2025-07-09 17:22:37,380 - root - INFO - step: 17140 loss: 16.7832 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6105 global_avg_mtp_loss: 14.1726 +[titan] 2025-07-09 17:22:37,380 - root - INFO - lr: 2.8268e-04 gnorm: 0.85 [ 3:47:40<18:20:38] +[titan] 2025-07-09 17:22:41,285 - root - INFO - step: 17145 loss: 17.1774 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6978 global_avg_mtp_loss: 14.4796 +[titan] 2025-07-09 17:22:41,285 - root - INFO - lr: 2.8266e-04 gnorm: 0.81 [ 3:47:44<18:20:34] +[titan] 2025-07-09 17:22:44,395 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:22:45,181 - root - INFO - step: 17150 loss: 17.2281 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6922 global_avg_mtp_loss: 14.5359 +[titan] 2025-07-09 17:22:45,181 - root - INFO - lr: 2.8265e-04 gnorm: 0.79 [ 3:47:48<18:20:30] +[titan] 2025-07-09 17:22:49,091 - root - INFO - step: 17155 loss: 17.2317 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.7105 global_avg_mtp_loss: 14.5211 +[titan] 2025-07-09 17:22:49,091 - root - INFO - lr: 2.8264e-04 gnorm: 0.82 [ 3:47:52<18:20:25] +[titan] 2025-07-09 17:22:52,998 - root - INFO - step: 17160 loss: 17.0362 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6607 global_avg_mtp_loss: 14.3754 +[titan] 2025-07-09 17:22:52,998 - root - INFO - lr: 2.8263e-04 gnorm: 0.79 [ 3:47:56<18:20:21] +[titan] 2025-07-09 17:22:56,939 - root - INFO - step: 17165 loss: 17.0686 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.4063 +[titan] 2025-07-09 17:22:56,939 - root - INFO - lr: 2.8262e-04 gnorm: 0.81 [ 3:48:00<18:20:17] +[titan] 2025-07-09 17:23:00,844 - root - INFO - step: 17170 loss: 17.0011 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6762 global_avg_mtp_loss: 14.3250 +[titan] 2025-07-09 17:23:00,844 - root - INFO - lr: 2.8261e-04 gnorm: 0.80 [ 3:48:03<18:20:12] +[titan] 2025-07-09 17:23:04,764 - root - INFO - step: 17175 loss: 17.1610 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6875 global_avg_mtp_loss: 14.4735 +[titan] 2025-07-09 17:23:04,764 - root - INFO - lr: 2.8260e-04 gnorm: 0.91 [ 3:48:07<18:20:08] +[titan] 2025-07-09 17:23:08,671 - root - INFO - step: 17180 loss: 17.0434 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6625 global_avg_mtp_loss: 14.3809 +[titan] 2025-07-09 17:23:08,671 - root - INFO - lr: 2.8259e-04 gnorm: 0.78 [ 3:48:11<18:20:04] +[titan] 2025-07-09 17:23:12,607 - root - INFO - step: 17185 loss: 17.2393 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.6975 global_avg_mtp_loss: 14.5418 +[titan] 2025-07-09 17:23:12,608 - root - INFO - lr: 2.8258e-04 gnorm: 0.75 [ 3:48:15<18:19:59] +[titan] 2025-07-09 17:23:16,521 - root - INFO - step: 17190 loss: 17.0238 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6607 global_avg_mtp_loss: 14.3631 +[titan] 2025-07-09 17:23:16,521 - root - INFO - lr: 2.8257e-04 gnorm: 0.79 [ 3:48:19<18:19:55] +[titan] 2025-07-09 17:23:20,426 - root - INFO - step: 17195 loss: 17.0706 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6675 global_avg_mtp_loss: 14.4030 +[titan] 2025-07-09 17:23:20,427 - root - INFO - lr: 2.8256e-04 gnorm: 0.80 [ 3:48:23<18:19:51] +[titan] 2025-07-09 17:23:23,540 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:23:24,324 - root - INFO - step: 17200 loss: 16.8782 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.2455 +[titan] 2025-07-09 17:23:24,324 - root - INFO - lr: 2.8255e-04 gnorm: 0.84 [ 3:48:27<18:19:46] +[titan] 2025-07-09 17:23:28,245 - root - INFO - step: 17205 loss: 16.9605 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.6445 global_avg_mtp_loss: 14.3160 +[titan] 2025-07-09 17:23:28,246 - root - INFO - lr: 2.8254e-04 gnorm: 0.75 [ 3:48:31<18:19:42] +[titan] 2025-07-09 17:23:32,151 - root - INFO - step: 17210 loss: 17.0352 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6490 global_avg_mtp_loss: 14.3863 +[titan] 2025-07-09 17:23:32,152 - root - INFO - lr: 2.8253e-04 gnorm: 0.83 [ 3:48:35<18:19:38] +[titan] 2025-07-09 17:23:36,060 - root - INFO - step: 17215 loss: 17.1341 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.4643 +[titan] 2025-07-09 17:23:36,060 - root - INFO - lr: 2.8252e-04 gnorm: 0.81 [ 3:48:39<18:19:33] +[titan] 2025-07-09 17:23:39,967 - root - INFO - step: 17220 loss: 16.8144 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.1980 +[titan] 2025-07-09 17:23:39,967 - root - INFO - lr: 2.8251e-04 gnorm: 0.84 [ 3:48:43<18:19:29] +[titan] 2025-07-09 17:23:43,875 - root - INFO - step: 17225 loss: 17.1588 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6882 global_avg_mtp_loss: 14.4706 +[titan] 2025-07-09 17:23:43,876 - root - INFO - lr: 2.8250e-04 gnorm: 0.77 [ 3:48:46<18:19:25] +[titan] 2025-07-09 17:23:47,809 - root - INFO - step: 17230 loss: 16.7230 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6085 global_avg_mtp_loss: 14.1145 +[titan] 2025-07-09 17:23:47,810 - root - INFO - lr: 2.8249e-04 gnorm: 0.77 [ 3:48:50<18:19:20] +[titan] 2025-07-09 17:23:51,716 - root - INFO - step: 17235 loss: 16.6883 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.0920 +[titan] 2025-07-09 17:23:51,717 - root - INFO - lr: 2.8248e-04 gnorm: 0.80 [ 3:48:54<18:19:16] +[titan] 2025-07-09 17:23:55,615 - root - INFO - step: 17240 loss: 17.1121 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6599 global_avg_mtp_loss: 14.4523 +[titan] 2025-07-09 17:23:55,616 - root - INFO - lr: 2.8246e-04 gnorm: 0.79 [ 3:48:58<18:19:12] +[titan] 2025-07-09 17:23:59,532 - root - INFO - step: 17245 loss: 17.1214 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6818 global_avg_mtp_loss: 14.4396 +[titan] 2025-07-09 17:23:59,532 - root - INFO - lr: 2.8245e-04 gnorm: 0.86 [ 3:49:02<18:19:07] +[titan] 2025-07-09 17:24:02,638 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:24:03,429 - root - INFO - step: 17250 loss: 17.0072 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.6571 global_avg_mtp_loss: 14.3502 +[titan] 2025-07-09 17:24:03,429 - root - INFO - lr: 2.8244e-04 gnorm: 0.81 [ 3:49:06<18:19:03] +[titan] 2025-07-09 17:24:07,345 - root - INFO - step: 17255 loss: 17.1257 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6940 global_avg_mtp_loss: 14.4317 +[titan] 2025-07-09 17:24:07,345 - root - INFO - lr: 2.8243e-04 gnorm: 0.84 [ 3:49:10<18:18:59] +[titan] 2025-07-09 17:24:11,253 - root - INFO - step: 17260 loss: 17.1899 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6822 global_avg_mtp_loss: 14.5077 +[titan] 2025-07-09 17:24:11,253 - root - INFO - lr: 2.8242e-04 gnorm: 0.78 [ 3:49:14<18:18:54] +[titan] 2025-07-09 17:24:15,153 - root - INFO - step: 17265 loss: 17.0713 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6683 global_avg_mtp_loss: 14.4029 +[titan] 2025-07-09 17:24:15,153 - root - INFO - lr: 2.8241e-04 gnorm: 0.84 [ 3:49:18<18:18:50] +[titan] 2025-07-09 17:24:19,068 - root - INFO - step: 17270 loss: 17.0680 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6616 global_avg_mtp_loss: 14.4064 +[titan] 2025-07-09 17:24:19,068 - root - INFO - lr: 2.8240e-04 gnorm: 0.78 [ 3:49:22<18:18:46] +[titan] 2025-07-09 17:24:22,965 - root - INFO - step: 17275 loss: 17.1954 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6863 global_avg_mtp_loss: 14.5091 +[titan] 2025-07-09 17:24:22,965 - root - INFO - lr: 2.8239e-04 gnorm: 0.83 [ 3:49:26<18:18:41] +[titan] 2025-07-09 17:24:26,876 - root - INFO - step: 17280 loss: 16.7793 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.1642 +[titan] 2025-07-09 17:24:26,877 - root - INFO - lr: 2.8238e-04 gnorm: 0.88 [ 3:49:29<18:18:37] +[titan] 2025-07-09 17:24:30,767 - root - INFO - step: 17285 loss: 17.1797 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.6943 global_avg_mtp_loss: 14.4854 +[titan] 2025-07-09 17:24:30,768 - root - INFO - lr: 2.8237e-04 gnorm: 0.90 [ 3:49:33<18:18:32] +[titan] 2025-07-09 17:24:34,691 - root - INFO - step: 17290 loss: 16.9615 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6366 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 17:24:34,691 - root - INFO - lr: 2.8236e-04 gnorm: 0.88 [ 3:49:37<18:18:28] +[titan] 2025-07-09 17:24:38,601 - root - INFO - step: 17295 loss: 17.2127 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6937 global_avg_mtp_loss: 14.5190 +[titan] 2025-07-09 17:24:38,601 - root - INFO - lr: 2.8235e-04 gnorm: 0.79 [ 3:49:41<18:18:24] +[titan] 2025-07-09 17:24:41,717 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:24:42,528 - root - INFO - step: 17300 loss: 17.2025 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.5036 +[titan] 2025-07-09 17:24:42,529 - root - INFO - lr: 2.8234e-04 gnorm: 0.79 [ 3:49:45<18:18:20] +[titan] 2025-07-09 17:24:46,467 - root - INFO - step: 17305 loss: 17.1293 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.6800 global_avg_mtp_loss: 14.4493 +[titan] 2025-07-09 17:24:46,467 - root - INFO - lr: 2.8233e-04 gnorm: 0.79 [ 3:49:49<18:18:15] +[titan] 2025-07-09 17:24:50,361 - root - INFO - step: 17310 loss: 17.3113 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.7204 global_avg_mtp_loss: 14.5909 +[titan] 2025-07-09 17:24:50,361 - root - INFO - lr: 2.8232e-04 gnorm: 0.80 [ 3:49:53<18:18:11] +[titan] 2025-07-09 17:24:54,258 - root - INFO - step: 17315 loss: 17.1642 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.6785 global_avg_mtp_loss: 14.4857 +[titan] 2025-07-09 17:24:54,258 - root - INFO - lr: 2.8231e-04 gnorm: 0.81 [ 3:49:57<18:18:06] +[titan] 2025-07-09 17:24:58,195 - root - INFO - step: 17320 loss: 17.0926 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.6622 global_avg_mtp_loss: 14.4303 +[titan] 2025-07-09 17:24:58,195 - root - INFO - lr: 2.8230e-04 gnorm: 0.81 [ 3:50:01<18:18:02] +[titan] 2025-07-09 17:25:02,132 - root - INFO - step: 17325 loss: 17.2046 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6854 global_avg_mtp_loss: 14.5192 +[titan] 2025-07-09 17:25:02,133 - root - INFO - lr: 2.8228e-04 gnorm: 0.90 [ 3:50:05<18:17:58] +[titan] 2025-07-09 17:25:06,079 - root - INFO - step: 17330 loss: 17.2490 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.6861 global_avg_mtp_loss: 14.5630 +[titan] 2025-07-09 17:25:06,080 - root - INFO - lr: 2.8227e-04 gnorm: 0.87 [ 3:50:09<18:17:54] +[titan] 2025-07-09 17:25:10,000 - root - INFO - step: 17335 loss: 17.1644 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6829 global_avg_mtp_loss: 14.4815 +[titan] 2025-07-09 17:25:10,000 - root - INFO - lr: 2.8226e-04 gnorm: 0.90 [ 3:50:13<18:17:50] +[titan] 2025-07-09 17:25:13,918 - root - INFO - step: 17340 loss: 17.1597 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6876 global_avg_mtp_loss: 14.4721 +[titan] 2025-07-09 17:25:13,919 - root - INFO - lr: 2.8225e-04 gnorm: 0.88 [ 3:50:16<18:17:45] +[titan] 2025-07-09 17:25:17,832 - root - INFO - step: 17345 loss: 16.7973 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.1776 +[titan] 2025-07-09 17:25:17,833 - root - INFO - lr: 2.8224e-04 gnorm: 0.80 [ 3:50:20<18:17:41] +[titan] 2025-07-09 17:25:20,942 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:25:21,734 - root - INFO - step: 17350 loss: 17.2542 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.7022 global_avg_mtp_loss: 14.5520 +[titan] 2025-07-09 17:25:21,734 - root - INFO - lr: 2.8223e-04 gnorm: 0.86 [ 3:50:24<18:17:37] +[titan] 2025-07-09 17:25:25,627 - root - INFO - step: 17355 loss: 17.0590 memory: 44.58GiB(31.99%) tps: 84,176 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.4174 +[titan] 2025-07-09 17:25:25,628 - root - INFO - lr: 2.8222e-04 gnorm: 0.83 [ 3:50:28<18:17:32] +[titan] 2025-07-09 17:25:29,544 - root - INFO - step: 17360 loss: 17.0760 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.3991 +[titan] 2025-07-09 17:25:29,544 - root - INFO - lr: 2.8221e-04 gnorm: 0.85 [ 3:50:32<18:17:28] +[titan] 2025-07-09 17:25:33,477 - root - INFO - step: 17365 loss: 16.9483 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6578 global_avg_mtp_loss: 14.2905 +[titan] 2025-07-09 17:25:33,477 - root - INFO - lr: 2.8220e-04 gnorm: 0.77 [ 3:50:36<18:17:24] +[titan] 2025-07-09 17:25:37,423 - root - INFO - step: 17370 loss: 17.0953 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.6618 global_avg_mtp_loss: 14.4336 +[titan] 2025-07-09 17:25:37,423 - root - INFO - lr: 2.8219e-04 gnorm: 0.90 [ 3:50:40<18:17:19] +[titan] 2025-07-09 17:25:41,317 - root - INFO - step: 17375 loss: 17.1045 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6650 global_avg_mtp_loss: 14.4396 +[titan] 2025-07-09 17:25:41,317 - root - INFO - lr: 2.8218e-04 gnorm: 0.83 [ 3:50:44<18:17:15] +[titan] 2025-07-09 17:25:45,248 - root - INFO - step: 17380 loss: 17.1193 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.4570 +[titan] 2025-07-09 17:25:45,249 - root - INFO - lr: 2.8217e-04 gnorm: 0.84 [ 3:50:48<18:17:11] +[titan] 2025-07-09 17:25:49,188 - root - INFO - step: 17385 loss: 17.3034 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.7028 global_avg_mtp_loss: 14.6006 +[titan] 2025-07-09 17:25:49,188 - root - INFO - lr: 2.8216e-04 gnorm: 0.82 [ 3:50:52<18:17:07] +[titan] 2025-07-09 17:25:53,099 - root - INFO - step: 17390 loss: 17.1076 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6669 global_avg_mtp_loss: 14.4407 +[titan] 2025-07-09 17:25:53,099 - root - INFO - lr: 2.8215e-04 gnorm: 0.81 [ 3:50:56<18:17:02] +[titan] 2025-07-09 17:25:57,001 - root - INFO - step: 17395 loss: 17.0553 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6516 global_avg_mtp_loss: 14.4037 +[titan] 2025-07-09 17:25:57,001 - root - INFO - lr: 2.8214e-04 gnorm: 0.78 [ 3:51:00<18:16:58] +[titan] 2025-07-09 17:26:00,148 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:26:00,933 - root - INFO - step: 17400 loss: 17.2689 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.7076 global_avg_mtp_loss: 14.5614 +[titan] 2025-07-09 17:26:00,933 - root - INFO - lr: 2.8213e-04 gnorm: 0.79 [ 3:51:04<18:16:54] +[titan] 2025-07-09 17:26:04,916 - root - INFO - step: 17405 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 82,275 tflops: 283.95 mfu: 28.71% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.1483 +[titan] 2025-07-09 17:26:04,916 - root - INFO - lr: 2.8211e-04 gnorm: 0.84 [ 3:51:07<18:16:50] +[titan] 2025-07-09 17:26:07,410 - root - INFO - Dumping profiler traces at step 17408 +[titan] 2025-07-09 17:26:07,442 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:26:09,010 - root - INFO - step: 17410 loss: 16.8918 memory: 44.58GiB(31.99%) tps: 80,053 tflops: 276.28 mfu: 27.94% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.2617 +[titan] 2025-07-09 17:26:09,010 - root - INFO - lr: 2.8210e-04 gnorm: 0.82 [ 3:51:12<18:16:46] +[titan] 2025-07-09 17:26:12,968 - root - INFO - step: 17415 loss: 16.9766 memory: 44.58GiB(31.99%) tps: 82,790 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.6472 global_avg_mtp_loss: 14.3294 +[titan] 2025-07-09 17:26:12,968 - root - INFO - lr: 2.8209e-04 gnorm: 0.86 [ 3:51:16<18:16:42] +[titan] 2025-07-09 17:26:16,880 - root - INFO - step: 17420 loss: 16.8846 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6443 global_avg_mtp_loss: 14.2402 +[titan] 2025-07-09 17:26:16,881 - root - INFO - lr: 2.8208e-04 gnorm: 0.80 [ 3:51:19<18:16:38] +[titan] 2025-07-09 17:26:20,880 - root - INFO - step: 17425 loss: 16.9967 memory: 44.58GiB(31.99%) tps: 81,944 tflops: 282.80 mfu: 28.59% global_avg_ntp_loss: 2.6585 global_avg_mtp_loss: 14.3381 +[titan] 2025-07-09 17:26:20,880 - root - INFO - lr: 2.8207e-04 gnorm: 0.86 [ 3:51:23<18:16:34] +[titan] 2025-07-09 17:26:24,810 - root - INFO - step: 17430 loss: 16.8026 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.2136 +[titan] 2025-07-09 17:26:24,810 - root - INFO - lr: 2.8206e-04 gnorm: 0.87 [ 3:51:27<18:16:30] +[titan] 2025-07-09 17:26:28,718 - root - INFO - step: 17435 loss: 17.0282 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6600 global_avg_mtp_loss: 14.3682 +[titan] 2025-07-09 17:26:28,718 - root - INFO - lr: 2.8205e-04 gnorm: 0.87 [ 3:51:31<18:16:25] +[titan] 2025-07-09 17:26:32,619 - root - INFO - step: 17440 loss: 17.0591 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6600 global_avg_mtp_loss: 14.3991 +[titan] 2025-07-09 17:26:32,620 - root - INFO - lr: 2.8204e-04 gnorm: 0.83 [ 3:51:35<18:16:21] +[titan] 2025-07-09 17:26:36,549 - root - INFO - step: 17445 loss: 17.1239 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6716 global_avg_mtp_loss: 14.4523 +[titan] 2025-07-09 17:26:36,549 - root - INFO - lr: 2.8203e-04 gnorm: 0.78 [ 3:51:39<18:16:17] +[titan] 2025-07-09 17:26:39,668 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:26:40,471 - root - INFO - step: 17450 loss: 17.0032 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6423 global_avg_mtp_loss: 14.3609 +[titan] 2025-07-09 17:26:40,471 - root - INFO - lr: 2.8202e-04 gnorm: 0.87 [ 3:51:43<18:16:12] +[titan] 2025-07-09 17:26:44,404 - root - INFO - step: 17455 loss: 17.0141 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.6562 global_avg_mtp_loss: 14.3579 +[titan] 2025-07-09 17:26:44,404 - root - INFO - lr: 2.8201e-04 gnorm: 0.82 [ 3:51:47<18:16:08] +[titan] 2025-07-09 17:26:48,328 - root - INFO - step: 17460 loss: 16.9790 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6444 global_avg_mtp_loss: 14.3346 +[titan] 2025-07-09 17:26:48,328 - root - INFO - lr: 2.8200e-04 gnorm: 0.85 [ 3:51:51<18:16:04] +[titan] 2025-07-09 17:26:52,224 - root - INFO - step: 17465 loss: 17.0158 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6594 global_avg_mtp_loss: 14.3564 +[titan] 2025-07-09 17:26:52,224 - root - INFO - lr: 2.8199e-04 gnorm: 0.81 [ 3:51:55<18:16:00] +[titan] 2025-07-09 17:26:56,120 - root - INFO - step: 17470 loss: 16.9594 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6266 global_avg_mtp_loss: 14.3328 +[titan] 2025-07-09 17:26:56,120 - root - INFO - lr: 2.8198e-04 gnorm: 0.81 [ 3:51:59<18:15:55] +[titan] 2025-07-09 17:27:00,027 - root - INFO - step: 17475 loss: 17.1396 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6787 global_avg_mtp_loss: 14.4609 +[titan] 2025-07-09 17:27:00,028 - root - INFO - lr: 2.8197e-04 gnorm: 0.81 [ 3:52:03<18:15:51] +[titan] 2025-07-09 17:27:03,920 - root - INFO - step: 17480 loss: 16.9676 memory: 44.58GiB(31.99%) tps: 84,181 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3182 +[titan] 2025-07-09 17:27:03,921 - root - INFO - lr: 2.8195e-04 gnorm: 0.89 [ 3:52:06<18:15:46] +[titan] 2025-07-09 17:27:07,849 - root - INFO - step: 17485 loss: 16.8713 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6249 global_avg_mtp_loss: 14.2463 +[titan] 2025-07-09 17:27:07,849 - root - INFO - lr: 2.8194e-04 gnorm: 0.82 [ 3:52:10<18:15:42] +[titan] 2025-07-09 17:27:11,798 - root - INFO - step: 17490 loss: 16.9485 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6467 global_avg_mtp_loss: 14.3018 +[titan] 2025-07-09 17:27:11,798 - root - INFO - lr: 2.8193e-04 gnorm: 0.94 [ 3:52:14<18:15:38] +[titan] 2025-07-09 17:27:15,701 - root - INFO - step: 17495 loss: 16.7303 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.1102 +[titan] 2025-07-09 17:27:15,702 - root - INFO - lr: 2.8192e-04 gnorm: 0.89 [ 3:52:18<18:15:34] +[titan] 2025-07-09 17:27:18,820 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:27:19,604 - root - INFO - step: 17500 loss: 16.9687 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6393 global_avg_mtp_loss: 14.3294 +[titan] 2025-07-09 17:27:19,604 - root - INFO - lr: 2.8191e-04 gnorm: 0.79 [ 3:52:22<18:15:29] +[titan] 2025-07-09 17:27:23,498 - root - INFO - step: 17505 loss: 17.1592 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.6871 global_avg_mtp_loss: 14.4721 +[titan] 2025-07-09 17:27:23,499 - root - INFO - lr: 2.8190e-04 gnorm: 0.91 [ 3:52:26<18:15:25] +[titan] 2025-07-09 17:27:27,400 - root - INFO - step: 17510 loss: 17.0978 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6795 global_avg_mtp_loss: 14.4183 +[titan] 2025-07-09 17:27:27,400 - root - INFO - lr: 2.8189e-04 gnorm: 0.83 [ 3:52:30<18:15:20] +[titan] 2025-07-09 17:27:31,291 - root - INFO - step: 17515 loss: 17.0064 memory: 44.58GiB(31.99%) tps: 84,217 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3573 +[titan] 2025-07-09 17:27:31,291 - root - INFO - lr: 2.8188e-04 gnorm: 0.79 [ 3:52:34<18:15:16] +[titan] 2025-07-09 17:27:35,182 - root - INFO - step: 17520 loss: 17.2201 memory: 44.58GiB(31.99%) tps: 84,218 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6930 global_avg_mtp_loss: 14.5271 +[titan] 2025-07-09 17:27:35,182 - root - INFO - lr: 2.8187e-04 gnorm: 0.74 [ 3:52:38<18:15:12] +[titan] 2025-07-09 17:27:39,118 - root - INFO - step: 17525 loss: 17.1684 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.6967 global_avg_mtp_loss: 14.4717 +[titan] 2025-07-09 17:27:39,119 - root - INFO - lr: 2.8186e-04 gnorm: 0.83 [ 3:52:42<18:15:07] +[titan] 2025-07-09 17:27:43,071 - root - INFO - step: 17530 loss: 17.1556 memory: 44.58GiB(31.99%) tps: 82,923 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.6772 global_avg_mtp_loss: 14.4784 +[titan] 2025-07-09 17:27:43,071 - root - INFO - lr: 2.8185e-04 gnorm: 0.83 [ 3:52:46<18:15:03] +[titan] 2025-07-09 17:27:46,979 - root - INFO - step: 17535 loss: 17.0188 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6533 global_avg_mtp_loss: 14.3656 +[titan] 2025-07-09 17:27:46,979 - root - INFO - lr: 2.8184e-04 gnorm: 0.77 [ 3:52:50<18:14:59] +[titan] 2025-07-09 17:27:50,889 - root - INFO - step: 17540 loss: 17.0251 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6547 global_avg_mtp_loss: 14.3704 +[titan] 2025-07-09 17:27:50,889 - root - INFO - lr: 2.8183e-04 gnorm: 0.82 [ 3:52:53<18:14:55] +[titan] 2025-07-09 17:27:54,811 - root - INFO - step: 17545 loss: 17.2163 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.5248 +[titan] 2025-07-09 17:27:54,811 - root - INFO - lr: 2.8182e-04 gnorm: 0.83 [ 3:52:57<18:14:50] +[titan] 2025-07-09 17:27:57,942 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:27:58,727 - root - INFO - step: 17550 loss: 17.3110 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.7114 global_avg_mtp_loss: 14.5996 +[titan] 2025-07-09 17:27:58,727 - root - INFO - lr: 2.8180e-04 gnorm: 0.83 [ 3:53:01<18:14:46] +[titan] 2025-07-09 17:28:02,640 - root - INFO - step: 17555 loss: 17.0694 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6622 global_avg_mtp_loss: 14.4072 +[titan] 2025-07-09 17:28:02,640 - root - INFO - lr: 2.8179e-04 gnorm: 0.80 [ 3:53:05<18:14:42] +[titan] 2025-07-09 17:28:06,544 - root - INFO - step: 17560 loss: 16.9799 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6482 global_avg_mtp_loss: 14.3318 +[titan] 2025-07-09 17:28:06,544 - root - INFO - lr: 2.8178e-04 gnorm: 0.80 [ 3:53:09<18:14:37] +[titan] 2025-07-09 17:28:10,507 - root - INFO - step: 17565 loss: 17.0137 memory: 44.58GiB(31.99%) tps: 82,685 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.6598 global_avg_mtp_loss: 14.3539 +[titan] 2025-07-09 17:28:10,507 - root - INFO - lr: 2.8177e-04 gnorm: 0.87 [ 3:53:13<18:14:33] +[titan] 2025-07-09 17:28:14,415 - root - INFO - step: 17570 loss: 17.1199 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6690 global_avg_mtp_loss: 14.4509 +[titan] 2025-07-09 17:28:14,415 - root - INFO - lr: 2.8176e-04 gnorm: 0.83 [ 3:53:17<18:14:29] +[titan] 2025-07-09 17:28:18,345 - root - INFO - step: 17575 loss: 17.1490 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6750 global_avg_mtp_loss: 14.4741 +[titan] 2025-07-09 17:28:18,345 - root - INFO - lr: 2.8175e-04 gnorm: 0.83 [ 3:53:21<18:14:25] +[titan] 2025-07-09 17:28:22,287 - root - INFO - step: 17580 loss: 16.8492 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6287 global_avg_mtp_loss: 14.2205 +[titan] 2025-07-09 17:28:22,287 - root - INFO - lr: 2.8174e-04 gnorm: 0.87 [ 3:53:25<18:14:20] +[titan] 2025-07-09 17:28:26,200 - root - INFO - step: 17585 loss: 17.0267 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.3732 +[titan] 2025-07-09 17:28:26,200 - root - INFO - lr: 2.8173e-04 gnorm: 0.82 [ 3:53:29<18:14:16] +[titan] 2025-07-09 17:28:30,103 - root - INFO - step: 17590 loss: 17.0342 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6572 global_avg_mtp_loss: 14.3770 +[titan] 2025-07-09 17:28:30,104 - root - INFO - lr: 2.8172e-04 gnorm: 0.81 [ 3:53:33<18:14:12] +[titan] 2025-07-09 17:28:34,004 - root - INFO - step: 17595 loss: 17.1467 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6876 global_avg_mtp_loss: 14.4591 +[titan] 2025-07-09 17:28:34,004 - root - INFO - lr: 2.8171e-04 gnorm: 0.84 [ 3:53:37<18:14:07] +[titan] 2025-07-09 17:28:37,109 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:28:37,902 - root - INFO - step: 17600 loss: 17.1253 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6671 global_avg_mtp_loss: 14.4582 +[titan] 2025-07-09 17:28:37,902 - root - INFO - lr: 2.8170e-04 gnorm: 0.84 [ 3:53:40<18:14:03] +[titan] 2025-07-09 17:28:41,828 - root - INFO - step: 17605 loss: 16.9368 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.6327 global_avg_mtp_loss: 14.3041 +[titan] 2025-07-09 17:28:41,828 - root - INFO - lr: 2.8169e-04 gnorm: 0.74 [ 3:53:44<18:13:59] +[titan] 2025-07-09 17:28:45,734 - root - INFO - step: 17610 loss: 17.0154 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.3527 +[titan] 2025-07-09 17:28:45,734 - root - INFO - lr: 2.8168e-04 gnorm: 0.84 [ 3:53:48<18:13:54] +[titan] 2025-07-09 17:28:49,659 - root - INFO - step: 17615 loss: 16.6188 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.0060 +[titan] 2025-07-09 17:28:49,660 - root - INFO - lr: 2.8166e-04 gnorm: 0.93 [ 3:53:52<18:13:50] +[titan] 2025-07-09 17:28:53,577 - root - INFO - step: 17620 loss: 16.9468 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6342 global_avg_mtp_loss: 14.3126 +[titan] 2025-07-09 17:28:53,577 - root - INFO - lr: 2.8165e-04 gnorm: 0.87 [ 3:53:56<18:13:46] +[titan] 2025-07-09 17:28:57,475 - root - INFO - step: 17625 loss: 17.2181 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6978 global_avg_mtp_loss: 14.5203 +[titan] 2025-07-09 17:28:57,476 - root - INFO - lr: 2.8164e-04 gnorm: 0.85 [ 3:54:00<18:13:42] +[titan] 2025-07-09 17:29:01,401 - root - INFO - step: 17630 loss: 16.9497 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6362 global_avg_mtp_loss: 14.3135 +[titan] 2025-07-09 17:29:01,402 - root - INFO - lr: 2.8163e-04 gnorm: 0.86 [ 3:54:04<18:13:37] +[titan] 2025-07-09 17:29:05,312 - root - INFO - step: 17635 loss: 17.1286 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6694 global_avg_mtp_loss: 14.4592 +[titan] 2025-07-09 17:29:05,312 - root - INFO - lr: 2.8162e-04 gnorm: 0.81 [ 3:54:08<18:13:33] +[titan] 2025-07-09 17:29:09,236 - root - INFO - step: 17640 loss: 17.1297 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6763 global_avg_mtp_loss: 14.4533 +[titan] 2025-07-09 17:29:09,236 - root - INFO - lr: 2.8161e-04 gnorm: 0.84 [ 3:54:12<18:13:29] +[titan] 2025-07-09 17:29:13,150 - root - INFO - step: 17645 loss: 17.0743 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6630 global_avg_mtp_loss: 14.4113 +[titan] 2025-07-09 17:29:13,150 - root - INFO - lr: 2.8160e-04 gnorm: 0.79 [ 3:54:16<18:13:24] +[titan] 2025-07-09 17:29:16,279 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:29:17,072 - root - INFO - step: 17650 loss: 16.9841 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.3201 +[titan] 2025-07-09 17:29:17,072 - root - INFO - lr: 2.8159e-04 gnorm: 0.83 [ 3:54:20<18:13:20] +[titan] 2025-07-09 17:29:20,970 - root - INFO - step: 17655 loss: 16.8585 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.2431 +[titan] 2025-07-09 17:29:20,971 - root - INFO - lr: 2.8158e-04 gnorm: 0.78 [ 3:54:24<18:13:16] +[titan] 2025-07-09 17:29:24,868 - root - INFO - step: 17660 loss: 16.9481 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6447 global_avg_mtp_loss: 14.3034 +[titan] 2025-07-09 17:29:24,869 - root - INFO - lr: 2.8157e-04 gnorm: 0.76 [ 3:54:27<18:13:11] +[titan] 2025-07-09 17:29:28,781 - root - INFO - step: 17665 loss: 16.9066 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.2729 +[titan] 2025-07-09 17:29:28,781 - root - INFO - lr: 2.8156e-04 gnorm: 0.88 [ 3:54:31<18:13:07] +[titan] 2025-07-09 17:29:32,707 - root - INFO - step: 17670 loss: 16.8758 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.2378 +[titan] 2025-07-09 17:29:32,707 - root - INFO - lr: 2.8155e-04 gnorm: 0.80 [ 3:54:35<18:13:03] +[titan] 2025-07-09 17:29:36,622 - root - INFO - step: 17675 loss: 17.1608 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6800 global_avg_mtp_loss: 14.4808 +[titan] 2025-07-09 17:29:36,622 - root - INFO - lr: 2.8153e-04 gnorm: 0.88 [ 3:54:39<18:12:58] +[titan] 2025-07-09 17:29:40,550 - root - INFO - step: 17680 loss: 16.9880 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.6574 global_avg_mtp_loss: 14.3306 +[titan] 2025-07-09 17:29:40,550 - root - INFO - lr: 2.8152e-04 gnorm: 0.87 [ 3:54:43<18:12:54] +[titan] 2025-07-09 17:29:44,503 - root - INFO - step: 17685 loss: 17.0375 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.6667 global_avg_mtp_loss: 14.3708 +[titan] 2025-07-09 17:29:44,503 - root - INFO - lr: 2.8151e-04 gnorm: 0.83 [ 3:54:47<18:12:50] +[titan] 2025-07-09 17:29:48,437 - root - INFO - step: 17690 loss: 16.7815 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6434 global_avg_mtp_loss: 14.1381 +[titan] 2025-07-09 17:29:48,437 - root - INFO - lr: 2.8150e-04 gnorm: 0.86 [ 3:54:51<18:12:46] +[titan] 2025-07-09 17:29:52,337 - root - INFO - step: 17695 loss: 17.2081 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6893 global_avg_mtp_loss: 14.5188 +[titan] 2025-07-09 17:29:52,338 - root - INFO - lr: 2.8149e-04 gnorm: 0.80 [ 3:54:55<18:12:42] +[titan] 2025-07-09 17:29:55,457 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:29:56,252 - root - INFO - step: 17700 loss: 17.1044 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6617 global_avg_mtp_loss: 14.4427 +[titan] 2025-07-09 17:29:56,252 - root - INFO - lr: 2.8148e-04 gnorm: 0.86 [ 3:54:59<18:12:37] +[titan] 2025-07-09 17:30:00,185 - root - INFO - step: 17705 loss: 17.0039 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.6500 global_avg_mtp_loss: 14.3539 +[titan] 2025-07-09 17:30:00,186 - root - INFO - lr: 2.8147e-04 gnorm: 0.81 [ 3:55:03<18:12:33] +[titan] 2025-07-09 17:30:04,109 - root - INFO - step: 17710 loss: 16.8768 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6291 global_avg_mtp_loss: 14.2477 +[titan] 2025-07-09 17:30:04,109 - root - INFO - lr: 2.8146e-04 gnorm: 0.76 [ 3:55:07<18:12:29] +[titan] 2025-07-09 17:30:08,023 - root - INFO - step: 17715 loss: 17.1044 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6576 global_avg_mtp_loss: 14.4468 +[titan] 2025-07-09 17:30:08,023 - root - INFO - lr: 2.8145e-04 gnorm: 0.77 [ 3:55:11<18:12:24] +[titan] 2025-07-09 17:30:11,925 - root - INFO - step: 17720 loss: 16.9467 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6440 global_avg_mtp_loss: 14.3027 +[titan] 2025-07-09 17:30:11,926 - root - INFO - lr: 2.8144e-04 gnorm: 0.80 [ 3:55:14<18:12:20] +[titan] 2025-07-09 17:30:15,898 - root - INFO - step: 17725 loss: 17.1473 memory: 44.58GiB(31.99%) tps: 82,490 tflops: 284.69 mfu: 28.79% global_avg_ntp_loss: 2.6911 global_avg_mtp_loss: 14.4562 +[titan] 2025-07-09 17:30:15,898 - root - INFO - lr: 2.8143e-04 gnorm: 0.79 [ 3:55:18<18:12:16] +[titan] 2025-07-09 17:30:19,818 - root - INFO - step: 17730 loss: 17.0711 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6646 global_avg_mtp_loss: 14.4066 +[titan] 2025-07-09 17:30:19,819 - root - INFO - lr: 2.8142e-04 gnorm: 0.77 [ 3:55:22<18:12:12] +[titan] 2025-07-09 17:30:23,724 - root - INFO - step: 17735 loss: 17.1879 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6884 global_avg_mtp_loss: 14.4995 +[titan] 2025-07-09 17:30:23,724 - root - INFO - lr: 2.8141e-04 gnorm: 0.82 [ 3:55:26<18:12:07] +[titan] 2025-07-09 17:30:27,641 - root - INFO - step: 17740 loss: 17.1761 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6815 global_avg_mtp_loss: 14.4946 +[titan] 2025-07-09 17:30:27,642 - root - INFO - lr: 2.8139e-04 gnorm: 0.76 [ 3:55:30<18:12:03] +[titan] 2025-07-09 17:30:31,603 - root - INFO - step: 17745 loss: 16.9229 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.6346 global_avg_mtp_loss: 14.2883 +[titan] 2025-07-09 17:30:31,603 - root - INFO - lr: 2.8138e-04 gnorm: 0.84 [ 3:55:34<18:11:59] +[titan] 2025-07-09 17:30:34,724 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:30:35,516 - root - INFO - step: 17750 loss: 17.3565 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.7303 global_avg_mtp_loss: 14.6262 +[titan] 2025-07-09 17:30:35,516 - root - INFO - lr: 2.8137e-04 gnorm: 0.80 [ 3:55:38<18:11:55] +[titan] 2025-07-09 17:30:39,451 - root - INFO - step: 17755 loss: 17.0490 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.3829 +[titan] 2025-07-09 17:30:39,452 - root - INFO - lr: 2.8136e-04 gnorm: 0.77 [ 3:55:42<18:11:51] +[titan] 2025-07-09 17:30:43,380 - root - INFO - step: 17760 loss: 17.0851 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6565 global_avg_mtp_loss: 14.4286 +[titan] 2025-07-09 17:30:43,380 - root - INFO - lr: 2.8135e-04 gnorm: 0.78 [ 3:55:46<18:11:46] +[titan] 2025-07-09 17:30:47,272 - root - INFO - step: 17765 loss: 16.9650 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.6457 global_avg_mtp_loss: 14.3193 +[titan] 2025-07-09 17:30:47,272 - root - INFO - lr: 2.8134e-04 gnorm: 0.82 [ 3:55:50<18:11:42] +[titan] 2025-07-09 17:30:51,181 - root - INFO - step: 17770 loss: 17.2002 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6974 global_avg_mtp_loss: 14.5028 +[titan] 2025-07-09 17:30:51,181 - root - INFO - lr: 2.8133e-04 gnorm: 0.83 [ 3:55:54<18:11:38] +[titan] 2025-07-09 17:30:55,108 - root - INFO - step: 17775 loss: 17.1135 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.6703 global_avg_mtp_loss: 14.4431 +[titan] 2025-07-09 17:30:55,109 - root - INFO - lr: 2.8132e-04 gnorm: 0.83 [ 3:55:58<18:11:33] +[titan] 2025-07-09 17:30:59,065 - root - INFO - step: 17780 loss: 16.9953 memory: 44.58GiB(31.99%) tps: 82,823 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.6463 global_avg_mtp_loss: 14.3490 +[titan] 2025-07-09 17:30:59,065 - root - INFO - lr: 2.8131e-04 gnorm: 0.86 [ 3:56:02<18:11:29] +[titan] 2025-07-09 17:31:02,994 - root - INFO - step: 17785 loss: 17.0477 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.6586 global_avg_mtp_loss: 14.3891 +[titan] 2025-07-09 17:31:02,995 - root - INFO - lr: 2.8130e-04 gnorm: 0.78 [ 3:56:06<18:11:25] +[titan] 2025-07-09 17:31:06,905 - root - INFO - step: 17790 loss: 16.9267 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.2860 +[titan] 2025-07-09 17:31:06,905 - root - INFO - lr: 2.8129e-04 gnorm: 0.75 [ 3:56:09<18:11:21] +[titan] 2025-07-09 17:31:10,808 - root - INFO - step: 17795 loss: 17.0457 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6467 global_avg_mtp_loss: 14.3990 +[titan] 2025-07-09 17:31:10,809 - root - INFO - lr: 2.8127e-04 gnorm: 0.81 [ 3:56:13<18:11:16] +[titan] 2025-07-09 17:31:13,932 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:31:14,737 - root - INFO - step: 17800 loss: 16.8381 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2206 +[titan] 2025-07-09 17:31:14,738 - root - INFO - lr: 2.8126e-04 gnorm: 0.79 [ 3:56:17<18:11:12] +[titan] 2025-07-09 17:31:18,675 - root - INFO - step: 17805 loss: 17.1979 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.6899 global_avg_mtp_loss: 14.5080 +[titan] 2025-07-09 17:31:18,675 - root - INFO - lr: 2.8125e-04 gnorm: 0.82 [ 3:56:21<18:11:08] +[titan] 2025-07-09 17:31:22,597 - root - INFO - step: 17810 loss: 17.0488 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6605 global_avg_mtp_loss: 14.3883 +[titan] 2025-07-09 17:31:22,597 - root - INFO - lr: 2.8124e-04 gnorm: 0.86 [ 3:56:25<18:11:04] +[titan] 2025-07-09 17:31:26,498 - root - INFO - step: 17815 loss: 17.1305 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6883 global_avg_mtp_loss: 14.4421 +[titan] 2025-07-09 17:31:26,498 - root - INFO - lr: 2.8123e-04 gnorm: 0.80 [ 3:56:29<18:10:59] +[titan] 2025-07-09 17:31:30,404 - root - INFO - step: 17820 loss: 16.8950 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6280 global_avg_mtp_loss: 14.2669 +[titan] 2025-07-09 17:31:30,404 - root - INFO - lr: 2.8122e-04 gnorm: 0.81 [ 3:56:33<18:10:55] +[titan] 2025-07-09 17:31:34,320 - root - INFO - step: 17825 loss: 16.9367 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.2964 +[titan] 2025-07-09 17:31:34,320 - root - INFO - lr: 2.8121e-04 gnorm: 0.86 [ 3:56:37<18:10:51] +[titan] 2025-07-09 17:31:38,218 - root - INFO - step: 17830 loss: 16.8513 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2321 +[titan] 2025-07-09 17:31:38,218 - root - INFO - lr: 2.8120e-04 gnorm: 0.82 [ 3:56:41<18:10:46] +[titan] 2025-07-09 17:31:42,119 - root - INFO - step: 17835 loss: 16.9205 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6558 global_avg_mtp_loss: 14.2647 +[titan] 2025-07-09 17:31:42,120 - root - INFO - lr: 2.8119e-04 gnorm: 0.84 [ 3:56:45<18:10:42] +[titan] 2025-07-09 17:31:46,042 - root - INFO - step: 17840 loss: 16.9320 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.6314 global_avg_mtp_loss: 14.3006 +[titan] 2025-07-09 17:31:46,043 - root - INFO - lr: 2.8118e-04 gnorm: 0.81 [ 3:56:49<18:10:38] +[titan] 2025-07-09 17:31:49,979 - root - INFO - step: 17845 loss: 17.1016 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6741 global_avg_mtp_loss: 14.4275 +[titan] 2025-07-09 17:31:49,979 - root - INFO - lr: 2.8117e-04 gnorm: 0.81 [ 3:56:53<18:10:34] +[titan] 2025-07-09 17:31:53,091 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:31:53,875 - root - INFO - step: 17850 loss: 17.1115 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6672 global_avg_mtp_loss: 14.4442 +[titan] 2025-07-09 17:31:53,875 - root - INFO - lr: 2.8115e-04 gnorm: 0.79 [ 3:56:56<18:10:29] +[titan] 2025-07-09 17:31:57,799 - root - INFO - step: 17855 loss: 17.1052 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6651 global_avg_mtp_loss: 14.4401 +[titan] 2025-07-09 17:31:57,800 - root - INFO - lr: 2.8114e-04 gnorm: 0.81 [ 3:57:00<18:10:25] +[titan] 2025-07-09 17:32:01,723 - root - INFO - step: 17860 loss: 16.8916 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6337 global_avg_mtp_loss: 14.2579 +[titan] 2025-07-09 17:32:01,723 - root - INFO - lr: 2.8113e-04 gnorm: 0.79 [ 3:57:04<18:10:21] +[titan] 2025-07-09 17:32:05,645 - root - INFO - step: 17865 loss: 17.0194 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.3748 +[titan] 2025-07-09 17:32:05,645 - root - INFO - lr: 2.8112e-04 gnorm: 0.90 [ 3:57:08<18:10:16] +[titan] 2025-07-09 17:32:09,544 - root - INFO - step: 17870 loss: 17.0039 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.3505 +[titan] 2025-07-09 17:32:09,544 - root - INFO - lr: 2.8111e-04 gnorm: 0.90 [ 3:57:12<18:10:12] +[titan] 2025-07-09 17:32:13,448 - root - INFO - step: 17875 loss: 16.9463 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6445 global_avg_mtp_loss: 14.3018 +[titan] 2025-07-09 17:32:13,448 - root - INFO - lr: 2.8110e-04 gnorm: 0.79 [ 3:57:16<18:10:08] +[titan] 2025-07-09 17:32:17,358 - root - INFO - step: 17880 loss: 16.8805 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6299 global_avg_mtp_loss: 14.2506 +[titan] 2025-07-09 17:32:17,358 - root - INFO - lr: 2.8109e-04 gnorm: 0.77 [ 3:57:20<18:10:03] +[titan] 2025-07-09 17:32:21,289 - root - INFO - step: 17885 loss: 16.9912 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6468 global_avg_mtp_loss: 14.3444 +[titan] 2025-07-09 17:32:21,289 - root - INFO - lr: 2.8108e-04 gnorm: 0.81 [ 3:57:24<18:09:59] +[titan] 2025-07-09 17:32:25,193 - root - INFO - step: 17890 loss: 17.0550 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6671 global_avg_mtp_loss: 14.3878 +[titan] 2025-07-09 17:32:25,193 - root - INFO - lr: 2.8107e-04 gnorm: 0.82 [ 3:57:28<18:09:55] +[titan] 2025-07-09 17:32:29,100 - root - INFO - step: 17895 loss: 16.9627 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6378 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 17:32:29,100 - root - INFO - lr: 2.8106e-04 gnorm: 0.80 [ 3:57:32<18:09:50] +[titan] 2025-07-09 17:32:32,245 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:32:33,037 - root - INFO - step: 17900 loss: 16.7116 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.0984 +[titan] 2025-07-09 17:32:33,037 - root - INFO - lr: 2.8105e-04 gnorm: 0.88 [ 3:57:36<18:09:46] +[titan] 2025-07-09 17:32:36,954 - root - INFO - step: 17905 loss: 17.1826 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6734 global_avg_mtp_loss: 14.5093 +[titan] 2025-07-09 17:32:36,954 - root - INFO - lr: 2.8103e-04 gnorm: 0.82 [ 3:57:39<18:09:42] +[titan] 2025-07-09 17:32:40,887 - root - INFO - step: 17910 loss: 16.9959 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6489 global_avg_mtp_loss: 14.3470 +[titan] 2025-07-09 17:32:40,887 - root - INFO - lr: 2.8102e-04 gnorm: 0.86 [ 3:57:43<18:09:38] +[titan] 2025-07-09 17:32:44,810 - root - INFO - step: 17915 loss: 16.7014 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6132 global_avg_mtp_loss: 14.0883 +[titan] 2025-07-09 17:32:44,810 - root - INFO - lr: 2.8101e-04 gnorm: 0.82 [ 3:57:47<18:09:34] +[titan] 2025-07-09 17:32:48,790 - root - INFO - step: 17920 loss: 16.9185 memory: 44.58GiB(31.99%) tps: 82,343 tflops: 284.18 mfu: 28.73% global_avg_ntp_loss: 2.6362 global_avg_mtp_loss: 14.2823 +[titan] 2025-07-09 17:32:48,790 - root - INFO - lr: 2.8100e-04 gnorm: 0.85 [ 3:57:51<18:09:30] +[titan] 2025-07-09 17:32:48,943 - root - INFO - Dumping profiler traces at step 17920 +[titan] 2025-07-09 17:32:48,976 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:32:52,909 - root - INFO - step: 17925 loss: 16.8906 memory: 44.58GiB(31.99%) tps: 79,562 tflops: 274.58 mfu: 27.76% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.2574 +[titan] 2025-07-09 17:32:52,909 - root - INFO - lr: 2.8099e-04 gnorm: 0.85 [ 3:57:55<18:09:26] +[titan] 2025-07-09 17:32:56,842 - root - INFO - step: 17930 loss: 16.9226 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.6290 global_avg_mtp_loss: 14.2936 +[titan] 2025-07-09 17:32:56,842 - root - INFO - lr: 2.8098e-04 gnorm: 0.83 [ 3:57:59<18:09:22] +[titan] 2025-07-09 17:33:00,756 - root - INFO - step: 17935 loss: 17.0487 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6644 global_avg_mtp_loss: 14.3843 +[titan] 2025-07-09 17:33:00,756 - root - INFO - lr: 2.8097e-04 gnorm: 0.82 [ 3:58:03<18:09:18] +[titan] 2025-07-09 17:33:04,673 - root - INFO - step: 17940 loss: 16.8958 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6170 global_avg_mtp_loss: 14.2788 +[titan] 2025-07-09 17:33:04,673 - root - INFO - lr: 2.8096e-04 gnorm: 0.84 [ 3:58:07<18:09:13] +[titan] 2025-07-09 17:33:08,592 - root - INFO - step: 17945 loss: 16.8018 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.1808 +[titan] 2025-07-09 17:33:08,593 - root - INFO - lr: 2.8095e-04 gnorm: 0.88 [ 3:58:11<18:09:09] +[titan] 2025-07-09 17:33:11,723 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:33:12,513 - root - INFO - step: 17950 loss: 16.9738 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6449 global_avg_mtp_loss: 14.3289 +[titan] 2025-07-09 17:33:12,513 - root - INFO - lr: 2.8094e-04 gnorm: 0.85 [ 3:58:15<18:09:05] +[titan] 2025-07-09 17:33:16,433 - root - INFO - step: 17955 loss: 17.2565 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.7206 global_avg_mtp_loss: 14.5359 +[titan] 2025-07-09 17:33:16,434 - root - INFO - lr: 2.8092e-04 gnorm: 0.83 [ 3:58:19<18:09:01] +[titan] 2025-07-09 17:33:20,332 - root - INFO - step: 17960 loss: 16.7783 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6204 global_avg_mtp_loss: 14.1579 +[titan] 2025-07-09 17:33:20,332 - root - INFO - lr: 2.8091e-04 gnorm: 0.87 [ 3:58:23<18:08:56] +[titan] 2025-07-09 17:33:24,230 - root - INFO - step: 17965 loss: 16.8577 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.2380 +[titan] 2025-07-09 17:33:24,231 - root - INFO - lr: 2.8090e-04 gnorm: 0.80 [ 3:58:27<18:08:52] +[titan] 2025-07-09 17:33:28,143 - root - INFO - step: 17970 loss: 16.9855 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6588 global_avg_mtp_loss: 14.3268 +[titan] 2025-07-09 17:33:28,143 - root - INFO - lr: 2.8089e-04 gnorm: 0.80 [ 3:58:31<18:08:48] +[titan] 2025-07-09 17:33:32,118 - root - INFO - step: 17975 loss: 16.9273 memory: 44.58GiB(31.99%) tps: 82,450 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.2827 +[titan] 2025-07-09 17:33:32,118 - root - INFO - lr: 2.8088e-04 gnorm: 0.80 [ 3:58:35<18:08:44] +[titan] 2025-07-09 17:33:36,035 - root - INFO - step: 17980 loss: 16.9972 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6354 global_avg_mtp_loss: 14.3618 +[titan] 2025-07-09 17:33:36,035 - root - INFO - lr: 2.8087e-04 gnorm: 0.75 [ 3:58:39<18:08:39] +[titan] 2025-07-09 17:33:39,940 - root - INFO - step: 17985 loss: 16.9125 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6387 global_avg_mtp_loss: 14.2738 +[titan] 2025-07-09 17:33:39,940 - root - INFO - lr: 2.8086e-04 gnorm: 0.77 [ 3:58:42<18:08:35] +[titan] 2025-07-09 17:33:43,826 - root - INFO - step: 17990 loss: 17.1661 memory: 44.58GiB(31.99%) tps: 84,317 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.6892 global_avg_mtp_loss: 14.4768 +[titan] 2025-07-09 17:33:43,827 - root - INFO - lr: 2.8085e-04 gnorm: 0.86 [ 3:58:46<18:08:30] +[titan] 2025-07-09 17:33:47,729 - root - INFO - step: 17995 loss: 16.9162 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.2779 +[titan] 2025-07-09 17:33:47,730 - root - INFO - lr: 2.8084e-04 gnorm: 0.83 [ 3:58:50<18:08:26] +[titan] 2025-07-09 17:33:50,847 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:33:51,637 - root - INFO - step: 18000 loss: 16.8997 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2686 +[titan] 2025-07-09 17:33:51,638 - root - INFO - lr: 2.8083e-04 gnorm: 0.78 [ 3:58:54<18:08:22] +[titan] 2025-07-09 17:33:55,552 - root - INFO - step: 18005 loss: 16.9555 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6396 global_avg_mtp_loss: 14.3159 +[titan] 2025-07-09 17:33:55,553 - root - INFO - lr: 2.8081e-04 gnorm: 0.84 [ 3:58:58<18:08:18] +[titan] 2025-07-09 17:33:59,477 - root - INFO - step: 18010 loss: 17.2642 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6939 global_avg_mtp_loss: 14.5704 +[titan] 2025-07-09 17:33:59,478 - root - INFO - lr: 2.8080e-04 gnorm: 0.93 [ 3:59:02<18:08:13] +[titan] 2025-07-09 17:34:03,412 - root - INFO - step: 18015 loss: 16.9625 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6432 global_avg_mtp_loss: 14.3193 +[titan] 2025-07-09 17:34:03,412 - root - INFO - lr: 2.8079e-04 gnorm: 0.86 [ 3:59:06<18:08:09] +[titan] 2025-07-09 17:34:07,303 - root - INFO - step: 18020 loss: 17.0253 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6581 global_avg_mtp_loss: 14.3672 +[titan] 2025-07-09 17:34:07,303 - root - INFO - lr: 2.8078e-04 gnorm: 0.84 [ 3:59:10<18:08:05] +[titan] 2025-07-09 17:34:11,246 - root - INFO - step: 18025 loss: 17.1553 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6824 global_avg_mtp_loss: 14.4728 +[titan] 2025-07-09 17:34:11,247 - root - INFO - lr: 2.8077e-04 gnorm: 0.76 [ 3:59:14<18:08:01] +[titan] 2025-07-09 17:34:15,159 - root - INFO - step: 18030 loss: 16.9882 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6569 global_avg_mtp_loss: 14.3313 +[titan] 2025-07-09 17:34:15,160 - root - INFO - lr: 2.8076e-04 gnorm: 0.79 [ 3:59:18<18:07:56] +[titan] 2025-07-09 17:34:19,089 - root - INFO - step: 18035 loss: 17.0039 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.6503 global_avg_mtp_loss: 14.3536 +[titan] 2025-07-09 17:34:19,090 - root - INFO - lr: 2.8075e-04 gnorm: 0.80 [ 3:59:22<18:07:52] +[titan] 2025-07-09 17:34:23,015 - root - INFO - step: 18040 loss: 17.0764 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.4140 +[titan] 2025-07-09 17:34:23,015 - root - INFO - lr: 2.8074e-04 gnorm: 0.77 [ 3:59:26<18:07:48] +[titan] 2025-07-09 17:34:26,946 - root - INFO - step: 18045 loss: 17.2037 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.6812 global_avg_mtp_loss: 14.5225 +[titan] 2025-07-09 17:34:26,947 - root - INFO - lr: 2.8073e-04 gnorm: 0.80 [ 3:59:29<18:07:44] +[titan] 2025-07-09 17:34:30,083 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:34:30,869 - root - INFO - step: 18050 loss: 17.1199 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6849 global_avg_mtp_loss: 14.4350 +[titan] 2025-07-09 17:34:30,869 - root - INFO - lr: 2.8072e-04 gnorm: 0.79 [ 3:59:33<18:07:39] +[titan] 2025-07-09 17:34:34,780 - root - INFO - step: 18055 loss: 16.9301 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6515 global_avg_mtp_loss: 14.2786 +[titan] 2025-07-09 17:34:34,781 - root - INFO - lr: 2.8070e-04 gnorm: 0.80 [ 3:59:37<18:07:35] +[titan] 2025-07-09 17:34:38,682 - root - INFO - step: 18060 loss: 17.0514 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.6536 global_avg_mtp_loss: 14.3978 +[titan] 2025-07-09 17:34:38,683 - root - INFO - lr: 2.8069e-04 gnorm: 0.89 [ 3:59:41<18:07:31] +[titan] 2025-07-09 17:34:42,620 - root - INFO - step: 18065 loss: 16.7420 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.1412 +[titan] 2025-07-09 17:34:42,620 - root - INFO - lr: 2.8068e-04 gnorm: 0.82 [ 3:59:45<18:07:26] +[titan] 2025-07-09 17:34:46,522 - root - INFO - step: 18070 loss: 16.9837 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6333 global_avg_mtp_loss: 14.3503 +[titan] 2025-07-09 17:34:46,523 - root - INFO - lr: 2.8067e-04 gnorm: 0.93 [ 3:59:49<18:07:22] +[titan] 2025-07-09 17:34:50,429 - root - INFO - step: 18075 loss: 16.9353 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6238 global_avg_mtp_loss: 14.3114 +[titan] 2025-07-09 17:34:50,430 - root - INFO - lr: 2.8066e-04 gnorm: 0.76 [ 3:59:53<18:07:18] +[titan] 2025-07-09 17:34:54,336 - root - INFO - step: 18080 loss: 17.4419 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.7401 global_avg_mtp_loss: 14.7018 +[titan] 2025-07-09 17:34:54,336 - root - INFO - lr: 2.8065e-04 gnorm: 0.91 [ 3:59:57<18:07:13] +[titan] 2025-07-09 17:34:58,320 - root - INFO - step: 18085 loss: 17.0472 memory: 44.58GiB(31.99%) tps: 82,246 tflops: 283.85 mfu: 28.70% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.3764 +[titan] 2025-07-09 17:34:58,320 - root - INFO - lr: 2.8064e-04 gnorm: 0.87 [ 4:00:01<18:07:10] +[titan] 2025-07-09 17:35:02,232 - root - INFO - step: 18090 loss: 16.9805 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6550 global_avg_mtp_loss: 14.3255 +[titan] 2025-07-09 17:35:02,232 - root - INFO - lr: 2.8063e-04 gnorm: 0.77 [ 4:00:05<18:07:05] +[titan] 2025-07-09 17:35:06,131 - root - INFO - step: 18095 loss: 16.9272 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6391 global_avg_mtp_loss: 14.2880 +[titan] 2025-07-09 17:35:06,132 - root - INFO - lr: 2.8062e-04 gnorm: 0.77 [ 4:00:09<18:07:01] +[titan] 2025-07-09 17:35:09,282 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:35:10,065 - root - INFO - step: 18100 loss: 17.0089 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3630 +[titan] 2025-07-09 17:35:10,066 - root - INFO - lr: 2.8061e-04 gnorm: 0.87 [ 4:00:13<18:06:57] +[titan] 2025-07-09 17:35:13,982 - root - INFO - step: 18105 loss: 16.9555 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3159 +[titan] 2025-07-09 17:35:13,982 - root - INFO - lr: 2.8059e-04 gnorm: 0.76 [ 4:00:16<18:06:52] +[titan] 2025-07-09 17:35:17,872 - root - INFO - step: 18110 loss: 16.9895 memory: 44.58GiB(31.99%) tps: 84,244 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.6819 global_avg_mtp_loss: 14.3075 +[titan] 2025-07-09 17:35:17,872 - root - INFO - lr: 2.8058e-04 gnorm: 0.84 [ 4:00:20<18:06:48] +[titan] 2025-07-09 17:35:21,800 - root - INFO - step: 18115 loss: 16.9029 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2693 +[titan] 2025-07-09 17:35:21,800 - root - INFO - lr: 2.8057e-04 gnorm: 0.82 [ 4:00:24<18:06:44] +[titan] 2025-07-09 17:35:25,704 - root - INFO - step: 18120 loss: 17.1045 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6562 global_avg_mtp_loss: 14.4483 +[titan] 2025-07-09 17:35:25,705 - root - INFO - lr: 2.8056e-04 gnorm: 0.79 [ 4:00:28<18:06:39] +[titan] 2025-07-09 17:35:29,622 - root - INFO - step: 18125 loss: 17.1021 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6676 global_avg_mtp_loss: 14.4345 +[titan] 2025-07-09 17:35:29,622 - root - INFO - lr: 2.8055e-04 gnorm: 0.81 [ 4:00:32<18:06:35] +[titan] 2025-07-09 17:35:33,550 - root - INFO - step: 18130 loss: 17.2521 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6968 global_avg_mtp_loss: 14.5553 +[titan] 2025-07-09 17:35:33,550 - root - INFO - lr: 2.8054e-04 gnorm: 0.83 [ 4:00:36<18:06:31] +[titan] 2025-07-09 17:35:37,446 - root - INFO - step: 18135 loss: 16.9517 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6523 global_avg_mtp_loss: 14.2994 +[titan] 2025-07-09 17:35:37,446 - root - INFO - lr: 2.8053e-04 gnorm: 0.80 [ 4:00:40<18:06:27] +[titan] 2025-07-09 17:35:41,387 - root - INFO - step: 18140 loss: 17.1779 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6729 global_avg_mtp_loss: 14.5050 +[titan] 2025-07-09 17:35:41,387 - root - INFO - lr: 2.8052e-04 gnorm: 0.77 [ 4:00:44<18:06:22] +[titan] 2025-07-09 17:35:45,316 - root - INFO - step: 18145 loss: 17.1198 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.6723 global_avg_mtp_loss: 14.4475 +[titan] 2025-07-09 17:35:45,316 - root - INFO - lr: 2.8051e-04 gnorm: 0.84 [ 4:00:48<18:06:18] +[titan] 2025-07-09 17:35:48,428 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:35:49,223 - root - INFO - step: 18150 loss: 16.8760 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.2453 +[titan] 2025-07-09 17:35:49,224 - root - INFO - lr: 2.8049e-04 gnorm: 0.88 [ 4:00:52<18:06:14] +[titan] 2025-07-09 17:35:53,184 - root - INFO - step: 18155 loss: 16.8295 memory: 44.58GiB(31.99%) tps: 82,738 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.2086 +[titan] 2025-07-09 17:35:53,184 - root - INFO - lr: 2.8048e-04 gnorm: 0.80 [ 4:00:56<18:06:10] +[titan] 2025-07-09 17:35:57,112 - root - INFO - step: 18160 loss: 16.9753 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6389 global_avg_mtp_loss: 14.3363 +[titan] 2025-07-09 17:35:57,112 - root - INFO - lr: 2.8047e-04 gnorm: 0.81 [ 4:01:00<18:06:06] +[titan] 2025-07-09 17:36:01,042 - root - INFO - step: 18165 loss: 17.2583 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.7097 global_avg_mtp_loss: 14.5486 +[titan] 2025-07-09 17:36:01,043 - root - INFO - lr: 2.8046e-04 gnorm: 0.75 [ 4:01:04<18:06:01] +[titan] 2025-07-09 17:36:04,988 - root - INFO - step: 18170 loss: 17.2187 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.6976 global_avg_mtp_loss: 14.5212 +[titan] 2025-07-09 17:36:04,988 - root - INFO - lr: 2.8045e-04 gnorm: 0.82 [ 4:01:07<18:05:57] +[titan] 2025-07-09 17:36:08,919 - root - INFO - step: 18175 loss: 17.1999 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6831 global_avg_mtp_loss: 14.5169 +[titan] 2025-07-09 17:36:08,920 - root - INFO - lr: 2.8044e-04 gnorm: 0.85 [ 4:01:11<18:05:53] +[titan] 2025-07-09 17:36:12,844 - root - INFO - step: 18180 loss: 17.0226 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6573 global_avg_mtp_loss: 14.3653 +[titan] 2025-07-09 17:36:12,844 - root - INFO - lr: 2.8043e-04 gnorm: 0.83 [ 4:01:15<18:05:49] +[titan] 2025-07-09 17:36:16,755 - root - INFO - step: 18185 loss: 17.0983 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6647 global_avg_mtp_loss: 14.4336 +[titan] 2025-07-09 17:36:16,756 - root - INFO - lr: 2.8042e-04 gnorm: 0.88 [ 4:01:19<18:05:44] +[titan] 2025-07-09 17:36:20,730 - root - INFO - step: 18190 loss: 16.9988 memory: 44.58GiB(31.99%) tps: 82,446 tflops: 284.53 mfu: 28.77% global_avg_ntp_loss: 2.6582 global_avg_mtp_loss: 14.3406 +[titan] 2025-07-09 17:36:20,731 - root - INFO - lr: 2.8041e-04 gnorm: 0.80 [ 4:01:23<18:05:40] +[titan] 2025-07-09 17:36:24,655 - root - INFO - step: 18195 loss: 16.9357 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.2939 +[titan] 2025-07-09 17:36:24,655 - root - INFO - lr: 2.8039e-04 gnorm: 0.91 [ 4:01:27<18:05:36] +[titan] 2025-07-09 17:36:27,824 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:36:28,616 - root - INFO - step: 18200 loss: 17.0809 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6625 global_avg_mtp_loss: 14.4184 +[titan] 2025-07-09 17:36:28,616 - root - INFO - lr: 2.8038e-04 gnorm: 0.80 [ 4:01:31<18:05:32] +[titan] 2025-07-09 17:36:32,554 - root - INFO - step: 18205 loss: 16.9252 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6314 global_avg_mtp_loss: 14.2937 +[titan] 2025-07-09 17:36:32,554 - root - INFO - lr: 2.8037e-04 gnorm: 0.86 [ 4:01:35<18:05:28] +[titan] 2025-07-09 17:36:36,476 - root - INFO - step: 18210 loss: 16.8650 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.2389 +[titan] 2025-07-09 17:36:36,477 - root - INFO - lr: 2.8036e-04 gnorm: 0.85 [ 4:01:39<18:05:24] +[titan] 2025-07-09 17:36:40,410 - root - INFO - step: 18215 loss: 17.0810 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4105 +[titan] 2025-07-09 17:36:40,411 - root - INFO - lr: 2.8035e-04 gnorm: 0.83 [ 4:01:43<18:05:20] +[titan] 2025-07-09 17:36:44,314 - root - INFO - step: 18220 loss: 16.7660 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6100 global_avg_mtp_loss: 14.1560 +[titan] 2025-07-09 17:36:44,314 - root - INFO - lr: 2.8034e-04 gnorm: 0.81 [ 4:01:47<18:05:15] +[titan] 2025-07-09 17:36:48,214 - root - INFO - step: 18225 loss: 17.2017 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.6907 global_avg_mtp_loss: 14.5111 +[titan] 2025-07-09 17:36:48,214 - root - INFO - lr: 2.8033e-04 gnorm: 0.89 [ 4:01:51<18:05:11] +[titan] 2025-07-09 17:36:52,132 - root - INFO - step: 18230 loss: 17.0866 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6651 global_avg_mtp_loss: 14.4215 +[titan] 2025-07-09 17:36:52,132 - root - INFO - lr: 2.8032e-04 gnorm: 0.88 [ 4:01:55<18:05:07] +[titan] 2025-07-09 17:36:56,025 - root - INFO - step: 18235 loss: 16.5822 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0110 +[titan] 2025-07-09 17:36:56,025 - root - INFO - lr: 2.8031e-04 gnorm: 0.81 [ 4:01:59<18:05:02] +[titan] 2025-07-09 17:36:59,967 - root - INFO - step: 18240 loss: 17.0944 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.6778 global_avg_mtp_loss: 14.4166 +[titan] 2025-07-09 17:36:59,968 - root - INFO - lr: 2.8029e-04 gnorm: 0.75 [ 4:02:02<18:04:58] +[titan] 2025-07-09 17:37:03,891 - root - INFO - step: 18245 loss: 16.6933 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5901 global_avg_mtp_loss: 14.1032 +[titan] 2025-07-09 17:37:03,892 - root - INFO - lr: 2.8028e-04 gnorm: 0.78 [ 4:02:06<18:04:54] +[titan] 2025-07-09 17:37:07,038 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:37:07,824 - root - INFO - step: 18250 loss: 17.0333 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6457 global_avg_mtp_loss: 14.3876 +[titan] 2025-07-09 17:37:07,824 - root - INFO - lr: 2.8027e-04 gnorm: 0.79 [ 4:02:10<18:04:50] +[titan] 2025-07-09 17:37:11,803 - root - INFO - step: 18255 loss: 17.0698 memory: 44.58GiB(31.99%) tps: 82,358 tflops: 284.23 mfu: 28.74% global_avg_ntp_loss: 2.6631 global_avg_mtp_loss: 14.4067 +[titan] 2025-07-09 17:37:11,803 - root - INFO - lr: 2.8026e-04 gnorm: 0.78 [ 4:02:14<18:04:46] +[titan] 2025-07-09 17:37:15,745 - root - INFO - step: 18260 loss: 17.1109 memory: 44.58GiB(31.99%) tps: 83,134 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.4339 +[titan] 2025-07-09 17:37:15,745 - root - INFO - lr: 2.8025e-04 gnorm: 0.83 [ 4:02:18<18:04:41] +[titan] 2025-07-09 17:37:19,669 - root - INFO - step: 18265 loss: 17.1259 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6650 global_avg_mtp_loss: 14.4609 +[titan] 2025-07-09 17:37:19,669 - root - INFO - lr: 2.8024e-04 gnorm: 0.78 [ 4:02:22<18:04:37] +[titan] 2025-07-09 17:37:23,639 - root - INFO - step: 18270 loss: 16.9582 memory: 44.58GiB(31.99%) tps: 82,537 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.6270 global_avg_mtp_loss: 14.3312 +[titan] 2025-07-09 17:37:23,639 - root - INFO - lr: 2.8023e-04 gnorm: 0.80 [ 4:02:26<18:04:33] +[titan] 2025-07-09 17:37:27,554 - root - INFO - step: 18275 loss: 16.9923 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6423 global_avg_mtp_loss: 14.3501 +[titan] 2025-07-09 17:37:27,554 - root - INFO - lr: 2.8022e-04 gnorm: 0.78 [ 4:02:30<18:04:29] +[titan] 2025-07-09 17:37:31,454 - root - INFO - step: 18280 loss: 17.0635 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6619 global_avg_mtp_loss: 14.4015 +[titan] 2025-07-09 17:37:31,454 - root - INFO - lr: 2.8021e-04 gnorm: 0.77 [ 4:02:34<18:04:25] +[titan] 2025-07-09 17:37:35,362 - root - INFO - step: 18285 loss: 16.9609 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6429 global_avg_mtp_loss: 14.3180 +[titan] 2025-07-09 17:37:35,363 - root - INFO - lr: 2.8019e-04 gnorm: 0.75 [ 4:02:38<18:04:20] +[titan] 2025-07-09 17:37:39,272 - root - INFO - step: 18290 loss: 17.1785 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6914 global_avg_mtp_loss: 14.4870 +[titan] 2025-07-09 17:37:39,272 - root - INFO - lr: 2.8018e-04 gnorm: 0.79 [ 4:02:42<18:04:16] +[titan] 2025-07-09 17:37:43,225 - root - INFO - step: 18295 loss: 17.1008 memory: 44.58GiB(31.99%) tps: 82,893 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.6593 global_avg_mtp_loss: 14.4415 +[titan] 2025-07-09 17:37:43,226 - root - INFO - lr: 2.8017e-04 gnorm: 0.85 [ 4:02:46<18:04:12] +[titan] 2025-07-09 17:37:46,350 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:37:47,139 - root - INFO - step: 18300 loss: 16.9002 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.2557 +[titan] 2025-07-09 17:37:47,139 - root - INFO - lr: 2.8016e-04 gnorm: 1.13 [ 4:02:50<18:04:08] +[titan] 2025-07-09 17:37:51,048 - root - INFO - step: 18305 loss: 16.8937 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6308 global_avg_mtp_loss: 14.2628 +[titan] 2025-07-09 17:37:51,048 - root - INFO - lr: 2.8015e-04 gnorm: 0.86 [ 4:02:54<18:04:03] +[titan] 2025-07-09 17:37:55,025 - root - INFO - step: 18310 loss: 16.9904 memory: 44.58GiB(31.99%) tps: 82,386 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.3437 +[titan] 2025-07-09 17:37:55,025 - root - INFO - lr: 2.8014e-04 gnorm: 0.84 [ 4:02:58<18:03:59] +[titan] 2025-07-09 17:37:58,954 - root - INFO - step: 18315 loss: 16.9864 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6587 global_avg_mtp_loss: 14.3277 +[titan] 2025-07-09 17:37:58,954 - root - INFO - lr: 2.8013e-04 gnorm: 0.83 [ 4:03:01<18:03:55] +[titan] 2025-07-09 17:38:02,865 - root - INFO - step: 18320 loss: 17.0293 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6532 global_avg_mtp_loss: 14.3761 +[titan] 2025-07-09 17:38:02,865 - root - INFO - lr: 2.8012e-04 gnorm: 0.84 [ 4:03:05<18:03:51] +[titan] 2025-07-09 17:38:06,760 - root - INFO - step: 18325 loss: 16.7817 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.1616 +[titan] 2025-07-09 17:38:06,760 - root - INFO - lr: 2.8010e-04 gnorm: 0.89 [ 4:03:09<18:03:46] +[titan] 2025-07-09 17:38:10,674 - root - INFO - step: 18330 loss: 16.8780 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 17:38:10,674 - root - INFO - lr: 2.8009e-04 gnorm: 0.84 [ 4:03:13<18:03:42] +[titan] 2025-07-09 17:38:14,587 - root - INFO - step: 18335 loss: 17.2056 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6932 global_avg_mtp_loss: 14.5124 +[titan] 2025-07-09 17:38:14,587 - root - INFO - lr: 2.8008e-04 gnorm: 0.76 [ 4:03:17<18:03:38] +[titan] 2025-07-09 17:38:18,527 - root - INFO - step: 18340 loss: 17.1288 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6859 global_avg_mtp_loss: 14.4429 +[titan] 2025-07-09 17:38:18,527 - root - INFO - lr: 2.8007e-04 gnorm: 0.82 [ 4:03:21<18:03:34] +[titan] 2025-07-09 17:38:22,432 - root - INFO - step: 18345 loss: 16.9771 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6615 global_avg_mtp_loss: 14.3156 +[titan] 2025-07-09 17:38:22,432 - root - INFO - lr: 2.8006e-04 gnorm: 0.81 [ 4:03:25<18:03:29] +[titan] 2025-07-09 17:38:25,566 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:38:26,356 - root - INFO - step: 18350 loss: 16.8334 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6182 global_avg_mtp_loss: 14.2152 +[titan] 2025-07-09 17:38:26,357 - root - INFO - lr: 2.8005e-04 gnorm: 0.76 [ 4:03:29<18:03:25] +[titan] 2025-07-09 17:38:30,299 - root - INFO - step: 18355 loss: 17.0020 memory: 44.58GiB(31.99%) tps: 83,126 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.6590 global_avg_mtp_loss: 14.3430 +[titan] 2025-07-09 17:38:30,299 - root - INFO - lr: 2.8004e-04 gnorm: 0.91 [ 4:03:33<18:03:21] +[titan] 2025-07-09 17:38:34,211 - root - INFO - step: 18360 loss: 17.0986 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6688 global_avg_mtp_loss: 14.4297 +[titan] 2025-07-09 17:38:34,211 - root - INFO - lr: 2.8003e-04 gnorm: 0.83 [ 4:03:37<18:03:17] +[titan] 2025-07-09 17:38:38,127 - root - INFO - step: 18365 loss: 17.0505 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6677 global_avg_mtp_loss: 14.3827 +[titan] 2025-07-09 17:38:38,127 - root - INFO - lr: 2.8001e-04 gnorm: 0.83 [ 4:03:41<18:03:12] +[titan] 2025-07-09 17:38:42,013 - root - INFO - step: 18370 loss: 16.8973 memory: 44.58GiB(31.99%) tps: 84,336 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 2.6370 global_avg_mtp_loss: 14.2603 +[titan] 2025-07-09 17:38:42,013 - root - INFO - lr: 2.8000e-04 gnorm: 0.77 [ 4:03:44<18:03:08] +[titan] 2025-07-09 17:38:45,910 - root - INFO - step: 18375 loss: 17.1393 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6752 global_avg_mtp_loss: 14.4641 +[titan] 2025-07-09 17:38:45,910 - root - INFO - lr: 2.7999e-04 gnorm: 0.86 [ 4:03:48<18:03:04] +[titan] 2025-07-09 17:38:49,833 - root - INFO - step: 18380 loss: 16.9551 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.3135 +[titan] 2025-07-09 17:38:49,833 - root - INFO - lr: 2.7998e-04 gnorm: 0.78 [ 4:03:52<18:02:59] +[titan] 2025-07-09 17:38:53,742 - root - INFO - step: 18385 loss: 17.1062 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.4439 +[titan] 2025-07-09 17:38:53,742 - root - INFO - lr: 2.7997e-04 gnorm: 0.75 [ 4:03:56<18:02:55] +[titan] 2025-07-09 17:38:57,709 - root - INFO - step: 18390 loss: 17.1091 memory: 44.58GiB(31.99%) tps: 82,604 tflops: 285.08 mfu: 28.83% global_avg_ntp_loss: 2.6727 global_avg_mtp_loss: 14.4364 +[titan] 2025-07-09 17:38:57,709 - root - INFO - lr: 2.7996e-04 gnorm: 0.81 [ 4:04:00<18:02:51] +[titan] 2025-07-09 17:39:01,611 - root - INFO - step: 18395 loss: 16.8829 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6290 global_avg_mtp_loss: 14.2539 +[titan] 2025-07-09 17:39:01,611 - root - INFO - lr: 2.7995e-04 gnorm: 0.88 [ 4:04:04<18:02:47] +[titan] 2025-07-09 17:39:04,748 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:39:05,558 - root - INFO - step: 18400 loss: 16.9645 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.6616 global_avg_mtp_loss: 14.3029 +[titan] 2025-07-09 17:39:05,558 - root - INFO - lr: 2.7994e-04 gnorm: 0.89 [ 4:04:08<18:02:43] +[titan] 2025-07-09 17:39:09,453 - root - INFO - step: 18405 loss: 16.9961 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6438 global_avg_mtp_loss: 14.3523 +[titan] 2025-07-09 17:39:09,453 - root - INFO - lr: 2.7993e-04 gnorm: 0.81 [ 4:04:12<18:02:38] +[titan] 2025-07-09 17:39:13,387 - root - INFO - step: 18410 loss: 16.9457 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.6401 global_avg_mtp_loss: 14.3056 +[titan] 2025-07-09 17:39:13,388 - root - INFO - lr: 2.7991e-04 gnorm: 0.90 [ 4:04:16<18:02:34] +[titan] 2025-07-09 17:39:17,295 - root - INFO - step: 18415 loss: 16.9208 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6358 global_avg_mtp_loss: 14.2850 +[titan] 2025-07-09 17:39:17,295 - root - INFO - lr: 2.7990e-04 gnorm: 0.80 [ 4:04:20<18:02:30] +[titan] 2025-07-09 17:39:21,213 - root - INFO - step: 18420 loss: 17.0123 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6512 global_avg_mtp_loss: 14.3612 +[titan] 2025-07-09 17:39:21,213 - root - INFO - lr: 2.7989e-04 gnorm: 0.91 [ 4:04:24<18:02:25] +[titan] 2025-07-09 17:39:25,140 - root - INFO - step: 18425 loss: 16.9680 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6483 global_avg_mtp_loss: 14.3197 +[titan] 2025-07-09 17:39:25,141 - root - INFO - lr: 2.7988e-04 gnorm: 0.84 [ 4:04:28<18:02:21] +[titan] 2025-07-09 17:39:29,136 - root - INFO - step: 18430 loss: 17.0159 memory: 44.58GiB(31.99%) tps: 82,025 tflops: 283.08 mfu: 28.62% global_avg_ntp_loss: 2.6382 global_avg_mtp_loss: 14.3777 +[titan] 2025-07-09 17:39:29,136 - root - INFO - lr: 2.7987e-04 gnorm: 0.78 [ 4:04:32<18:02:17] +[titan] 2025-07-09 17:39:30,871 - root - INFO - Dumping profiler traces at step 18432 +[titan] 2025-07-09 17:39:30,902 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:39:33,244 - root - INFO - step: 18435 loss: 16.9819 memory: 44.58GiB(31.99%) tps: 79,760 tflops: 275.27 mfu: 27.83% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.3346 +[titan] 2025-07-09 17:39:33,245 - root - INFO - lr: 2.7986e-04 gnorm: 0.82 [ 4:04:36<18:02:14] +[titan] 2025-07-09 17:39:37,148 - root - INFO - step: 18440 loss: 16.9730 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.3324 +[titan] 2025-07-09 17:39:37,149 - root - INFO - lr: 2.7985e-04 gnorm: 0.80 [ 4:04:40<18:02:10] +[titan] 2025-07-09 17:39:41,072 - root - INFO - step: 18445 loss: 16.9449 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6485 global_avg_mtp_loss: 14.2964 +[titan] 2025-07-09 17:39:41,073 - root - INFO - lr: 2.7984e-04 gnorm: 0.81 [ 4:04:44<18:02:05] +[titan] 2025-07-09 17:39:44,202 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:39:45,002 - root - INFO - step: 18450 loss: 17.1928 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6884 global_avg_mtp_loss: 14.5044 +[titan] 2025-07-09 17:39:45,003 - root - INFO - lr: 2.7982e-04 gnorm: 0.76 [ 4:04:47<18:02:01] +[titan] 2025-07-09 17:39:48,904 - root - INFO - step: 18455 loss: 17.2687 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6964 global_avg_mtp_loss: 14.5723 +[titan] 2025-07-09 17:39:48,904 - root - INFO - lr: 2.7981e-04 gnorm: 0.82 [ 4:04:51<18:01:57] +[titan] 2025-07-09 17:39:52,826 - root - INFO - step: 18460 loss: 16.9825 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6389 global_avg_mtp_loss: 14.3435 +[titan] 2025-07-09 17:39:52,827 - root - INFO - lr: 2.7980e-04 gnorm: 0.80 [ 4:04:55<18:01:53] +[titan] 2025-07-09 17:39:56,727 - root - INFO - step: 18465 loss: 16.5740 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5559 global_avg_mtp_loss: 14.0181 +[titan] 2025-07-09 17:39:56,727 - root - INFO - lr: 2.7979e-04 gnorm: 0.88 [ 4:04:59<18:01:48] +[titan] 2025-07-09 17:40:00,644 - root - INFO - step: 18470 loss: 16.9570 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6280 global_avg_mtp_loss: 14.3290 +[titan] 2025-07-09 17:40:00,644 - root - INFO - lr: 2.7978e-04 gnorm: 0.85 [ 4:05:03<18:01:44] +[titan] 2025-07-09 17:40:04,547 - root - INFO - step: 18475 loss: 16.9969 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6609 global_avg_mtp_loss: 14.3360 +[titan] 2025-07-09 17:40:04,547 - root - INFO - lr: 2.7977e-04 gnorm: 0.92 [ 4:05:07<18:01:40] +[titan] 2025-07-09 17:40:08,471 - root - INFO - step: 18480 loss: 17.0472 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.3844 +[titan] 2025-07-09 17:40:08,471 - root - INFO - lr: 2.7976e-04 gnorm: 0.83 [ 4:05:11<18:01:35] +[titan] 2025-07-09 17:40:12,379 - root - INFO - step: 18485 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.3049 +[titan] 2025-07-09 17:40:12,380 - root - INFO - lr: 2.7974e-04 gnorm: 0.83 [ 4:05:15<18:01:31] +[titan] 2025-07-09 17:40:16,305 - root - INFO - step: 18490 loss: 16.9031 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6291 global_avg_mtp_loss: 14.2740 +[titan] 2025-07-09 17:40:16,305 - root - INFO - lr: 2.7973e-04 gnorm: 0.88 [ 4:05:19<18:01:27] +[titan] 2025-07-09 17:40:20,234 - root - INFO - step: 18495 loss: 16.8297 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2124 +[titan] 2025-07-09 17:40:20,234 - root - INFO - lr: 2.7972e-04 gnorm: 0.78 [ 4:05:23<18:01:23] +[titan] 2025-07-09 17:40:23,355 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:40:24,149 - root - INFO - step: 18500 loss: 17.1902 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6843 global_avg_mtp_loss: 14.5059 +[titan] 2025-07-09 17:40:24,149 - root - INFO - lr: 2.7971e-04 gnorm: 0.80 [ 4:05:27<18:01:18] +[titan] 2025-07-09 17:40:28,077 - root - INFO - step: 18505 loss: 17.0918 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6690 global_avg_mtp_loss: 14.4228 +[titan] 2025-07-09 17:40:28,078 - root - INFO - lr: 2.7970e-04 gnorm: 0.77 [ 4:05:31<18:01:14] +[titan] 2025-07-09 17:40:31,989 - root - INFO - step: 18510 loss: 17.2595 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6904 global_avg_mtp_loss: 14.5691 +[titan] 2025-07-09 17:40:31,989 - root - INFO - lr: 2.7969e-04 gnorm: 0.78 [ 4:05:34<18:01:10] +[titan] 2025-07-09 17:40:35,968 - root - INFO - step: 18515 loss: 17.0790 memory: 44.58GiB(31.99%) tps: 82,345 tflops: 284.19 mfu: 28.73% global_avg_ntp_loss: 2.6629 global_avg_mtp_loss: 14.4161 +[titan] 2025-07-09 17:40:35,969 - root - INFO - lr: 2.7968e-04 gnorm: 0.79 [ 4:05:38<18:01:06] +[titan] 2025-07-09 17:40:39,871 - root - INFO - step: 18520 loss: 16.8492 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6170 global_avg_mtp_loss: 14.2322 +[titan] 2025-07-09 17:40:39,872 - root - INFO - lr: 2.7967e-04 gnorm: 0.79 [ 4:05:42<18:01:02] +[titan] 2025-07-09 17:40:43,788 - root - INFO - step: 18525 loss: 17.0584 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6553 global_avg_mtp_loss: 14.4031 +[titan] 2025-07-09 17:40:43,788 - root - INFO - lr: 2.7965e-04 gnorm: 0.84 [ 4:05:46<18:00:57] +[titan] 2025-07-09 17:40:47,715 - root - INFO - step: 18530 loss: 17.1212 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4507 +[titan] 2025-07-09 17:40:47,716 - root - INFO - lr: 2.7964e-04 gnorm: 0.82 [ 4:05:50<18:00:53] +[titan] 2025-07-09 17:40:51,651 - root - INFO - step: 18535 loss: 16.9009 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6399 global_avg_mtp_loss: 14.2610 +[titan] 2025-07-09 17:40:51,651 - root - INFO - lr: 2.7963e-04 gnorm: 0.84 [ 4:05:54<18:00:49] +[titan] 2025-07-09 17:40:55,580 - root - INFO - step: 18540 loss: 17.1111 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.4329 +[titan] 2025-07-09 17:40:55,580 - root - INFO - lr: 2.7962e-04 gnorm: 0.84 [ 4:05:58<18:00:45] +[titan] 2025-07-09 17:40:59,501 - root - INFO - step: 18545 loss: 17.0209 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6599 global_avg_mtp_loss: 14.3610 +[titan] 2025-07-09 17:40:59,501 - root - INFO - lr: 2.7961e-04 gnorm: 0.84 [ 4:06:02<18:00:41] +[titan] 2025-07-09 17:41:02,637 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:41:03,429 - root - INFO - step: 18550 loss: 16.8565 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.2214 +[titan] 2025-07-09 17:41:03,429 - root - INFO - lr: 2.7960e-04 gnorm: 0.82 [ 4:06:06<18:00:36] +[titan] 2025-07-09 17:41:07,390 - root - INFO - step: 18555 loss: 17.1023 memory: 44.58GiB(31.99%) tps: 82,719 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.6759 global_avg_mtp_loss: 14.4264 +[titan] 2025-07-09 17:41:07,391 - root - INFO - lr: 2.7959e-04 gnorm: 0.78 [ 4:06:10<18:00:32] +[titan] 2025-07-09 17:41:11,375 - root - INFO - step: 18560 loss: 17.1444 memory: 44.58GiB(31.99%) tps: 82,240 tflops: 283.82 mfu: 28.70% global_avg_ntp_loss: 2.6849 global_avg_mtp_loss: 14.4595 +[titan] 2025-07-09 17:41:11,375 - root - INFO - lr: 2.7958e-04 gnorm: 0.83 [ 4:06:14<18:00:28] +[titan] 2025-07-09 17:41:15,310 - root - INFO - step: 18565 loss: 16.7873 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.1832 +[titan] 2025-07-09 17:41:15,310 - root - INFO - lr: 2.7956e-04 gnorm: 0.88 [ 4:06:18<18:00:24] +[titan] 2025-07-09 17:41:19,221 - root - INFO - step: 18570 loss: 17.2673 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6918 global_avg_mtp_loss: 14.5755 +[titan] 2025-07-09 17:41:19,221 - root - INFO - lr: 2.7955e-04 gnorm: 0.79 [ 4:06:22<18:00:20] +[titan] 2025-07-09 17:41:23,163 - root - INFO - step: 18575 loss: 16.8452 memory: 44.58GiB(31.99%) tps: 83,143 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.2064 +[titan] 2025-07-09 17:41:23,163 - root - INFO - lr: 2.7954e-04 gnorm: 0.78 [ 4:06:26<18:00:16] +[titan] 2025-07-09 17:41:27,069 - root - INFO - step: 18580 loss: 16.9192 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.6488 global_avg_mtp_loss: 14.2704 +[titan] 2025-07-09 17:41:27,069 - root - INFO - lr: 2.7953e-04 gnorm: 0.75 [ 4:06:30<18:00:11] +[titan] 2025-07-09 17:41:30,979 - root - INFO - step: 18585 loss: 17.0028 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6732 global_avg_mtp_loss: 14.3296 +[titan] 2025-07-09 17:41:30,980 - root - INFO - lr: 2.7952e-04 gnorm: 0.80 [ 4:06:33<18:00:07] +[titan] 2025-07-09 17:41:34,901 - root - INFO - step: 18590 loss: 16.9876 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6635 global_avg_mtp_loss: 14.3241 +[titan] 2025-07-09 17:41:34,901 - root - INFO - lr: 2.7951e-04 gnorm: 0.82 [ 4:06:37<18:00:03] +[titan] 2025-07-09 17:41:38,796 - root - INFO - step: 18595 loss: 17.2787 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.7073 global_avg_mtp_loss: 14.5715 +[titan] 2025-07-09 17:41:38,796 - root - INFO - lr: 2.7950e-04 gnorm: 0.88 [ 4:06:41<17:59:58] +[titan] 2025-07-09 17:41:41,928 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:41:42,719 - root - INFO - step: 18600 loss: 17.2340 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6999 global_avg_mtp_loss: 14.5341 +[titan] 2025-07-09 17:41:42,719 - root - INFO - lr: 2.7948e-04 gnorm: 0.75 [ 4:06:45<17:59:54] +[titan] 2025-07-09 17:41:46,629 - root - INFO - step: 18605 loss: 17.0594 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6594 global_avg_mtp_loss: 14.4000 +[titan] 2025-07-09 17:41:46,629 - root - INFO - lr: 2.7947e-04 gnorm: 0.88 [ 4:06:49<17:59:50] +[titan] 2025-07-09 17:41:50,544 - root - INFO - step: 18610 loss: 16.9540 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6467 global_avg_mtp_loss: 14.3073 +[titan] 2025-07-09 17:41:50,544 - root - INFO - lr: 2.7946e-04 gnorm: 0.79 [ 4:06:53<17:59:46] +[titan] 2025-07-09 17:41:54,464 - root - INFO - step: 18615 loss: 16.9970 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.6448 global_avg_mtp_loss: 14.3521 +[titan] 2025-07-09 17:41:54,464 - root - INFO - lr: 2.7945e-04 gnorm: 0.86 [ 4:06:57<17:59:41] +[titan] 2025-07-09 17:41:58,378 - root - INFO - step: 18620 loss: 16.8935 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2615 +[titan] 2025-07-09 17:41:58,378 - root - INFO - lr: 2.7944e-04 gnorm: 0.87 [ 4:07:01<17:59:37] +[titan] 2025-07-09 17:42:02,275 - root - INFO - step: 18625 loss: 17.0297 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3741 +[titan] 2025-07-09 17:42:02,275 - root - INFO - lr: 2.7943e-04 gnorm: 0.82 [ 4:07:05<17:59:33] +[titan] 2025-07-09 17:42:06,171 - root - INFO - step: 18630 loss: 16.9561 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6303 global_avg_mtp_loss: 14.3258 +[titan] 2025-07-09 17:42:06,171 - root - INFO - lr: 2.7942e-04 gnorm: 0.79 [ 4:07:09<17:59:28] +[titan] 2025-07-09 17:42:10,099 - root - INFO - step: 18635 loss: 16.9979 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6391 global_avg_mtp_loss: 14.3588 +[titan] 2025-07-09 17:42:10,100 - root - INFO - lr: 2.7941e-04 gnorm: 0.79 [ 4:07:13<17:59:24] +[titan] 2025-07-09 17:42:14,003 - root - INFO - step: 18640 loss: 17.2895 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.7039 global_avg_mtp_loss: 14.5856 +[titan] 2025-07-09 17:42:14,003 - root - INFO - lr: 2.7939e-04 gnorm: 0.81 [ 4:07:16<17:59:20] +[titan] 2025-07-09 17:42:17,916 - root - INFO - step: 18645 loss: 17.1605 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6891 global_avg_mtp_loss: 14.4714 +[titan] 2025-07-09 17:42:17,916 - root - INFO - lr: 2.7938e-04 gnorm: 0.80 [ 4:07:20<17:59:16] +[titan] 2025-07-09 17:42:21,042 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:42:21,830 - root - INFO - step: 18650 loss: 16.9176 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6410 global_avg_mtp_loss: 14.2766 +[titan] 2025-07-09 17:42:21,830 - root - INFO - lr: 2.7937e-04 gnorm: 0.86 [ 4:07:24<17:59:11] +[titan] 2025-07-09 17:42:25,729 - root - INFO - step: 18655 loss: 17.0868 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6778 global_avg_mtp_loss: 14.4091 +[titan] 2025-07-09 17:42:25,729 - root - INFO - lr: 2.7936e-04 gnorm: 0.80 [ 4:07:28<17:59:07] +[titan] 2025-07-09 17:42:29,672 - root - INFO - step: 18660 loss: 17.0927 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.6707 global_avg_mtp_loss: 14.4220 +[titan] 2025-07-09 17:42:29,672 - root - INFO - lr: 2.7935e-04 gnorm: 0.84 [ 4:07:32<17:59:03] +[titan] 2025-07-09 17:42:33,588 - root - INFO - step: 18665 loss: 17.2236 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6921 global_avg_mtp_loss: 14.5316 +[titan] 2025-07-09 17:42:33,588 - root - INFO - lr: 2.7934e-04 gnorm: 0.86 [ 4:07:36<17:58:59] +[titan] 2025-07-09 17:42:37,521 - root - INFO - step: 18670 loss: 17.0703 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.6581 global_avg_mtp_loss: 14.4122 +[titan] 2025-07-09 17:42:37,522 - root - INFO - lr: 2.7933e-04 gnorm: 0.90 [ 4:07:40<17:58:55] +[titan] 2025-07-09 17:42:41,427 - root - INFO - step: 18675 loss: 17.0661 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6605 global_avg_mtp_loss: 14.4056 +[titan] 2025-07-09 17:42:41,427 - root - INFO - lr: 2.7931e-04 gnorm: 0.93 [ 4:07:44<17:58:50] +[titan] 2025-07-09 17:42:45,356 - root - INFO - step: 18680 loss: 17.0354 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.3655 +[titan] 2025-07-09 17:42:45,356 - root - INFO - lr: 2.7930e-04 gnorm: 0.84 [ 4:07:48<17:58:46] +[titan] 2025-07-09 17:42:49,250 - root - INFO - step: 18685 loss: 17.0609 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.6663 global_avg_mtp_loss: 14.3946 +[titan] 2025-07-09 17:42:49,250 - root - INFO - lr: 2.7929e-04 gnorm: 0.82 [ 4:07:52<17:58:42] +[titan] 2025-07-09 17:42:53,197 - root - INFO - step: 18690 loss: 16.9560 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.6482 global_avg_mtp_loss: 14.3079 +[titan] 2025-07-09 17:42:53,197 - root - INFO - lr: 2.7928e-04 gnorm: 0.83 [ 4:07:56<17:58:38] +[titan] 2025-07-09 17:42:57,110 - root - INFO - step: 18695 loss: 16.6946 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5936 global_avg_mtp_loss: 14.1010 +[titan] 2025-07-09 17:42:57,110 - root - INFO - lr: 2.7927e-04 gnorm: 0.80 [ 4:08:00<17:58:33] +[titan] 2025-07-09 17:43:00,242 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:43:01,053 - root - INFO - step: 18700 loss: 16.8062 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.1890 +[titan] 2025-07-09 17:43:01,054 - root - INFO - lr: 2.7926e-04 gnorm: 0.78 [ 4:08:04<17:58:29] +[titan] 2025-07-09 17:43:04,984 - root - INFO - step: 18705 loss: 17.2941 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.7142 global_avg_mtp_loss: 14.5799 +[titan] 2025-07-09 17:43:04,984 - root - INFO - lr: 2.7925e-04 gnorm: 0.91 [ 4:08:07<17:58:25] +[titan] 2025-07-09 17:43:08,901 - root - INFO - step: 18710 loss: 16.9771 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3357 +[titan] 2025-07-09 17:43:08,902 - root - INFO - lr: 2.7923e-04 gnorm: 0.82 [ 4:08:11<17:58:21] +[titan] 2025-07-09 17:43:12,839 - root - INFO - step: 18715 loss: 17.0224 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6519 global_avg_mtp_loss: 14.3705 +[titan] 2025-07-09 17:43:12,840 - root - INFO - lr: 2.7922e-04 gnorm: 0.82 [ 4:08:15<17:58:17] +[titan] 2025-07-09 17:43:16,750 - root - INFO - step: 18720 loss: 17.3561 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.7123 global_avg_mtp_loss: 14.6438 +[titan] 2025-07-09 17:43:16,751 - root - INFO - lr: 2.7921e-04 gnorm: 0.81 [ 4:08:19<17:58:12] +[titan] 2025-07-09 17:43:20,659 - root - INFO - step: 18725 loss: 16.8813 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.2348 +[titan] 2025-07-09 17:43:20,659 - root - INFO - lr: 2.7920e-04 gnorm: 0.78 [ 4:08:23<17:58:08] +[titan] 2025-07-09 17:43:24,589 - root - INFO - step: 18730 loss: 17.0166 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6360 global_avg_mtp_loss: 14.3806 +[titan] 2025-07-09 17:43:24,589 - root - INFO - lr: 2.7919e-04 gnorm: 0.81 [ 4:08:27<17:58:04] +[titan] 2025-07-09 17:43:28,481 - root - INFO - step: 18735 loss: 17.1424 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.6679 global_avg_mtp_loss: 14.4745 +[titan] 2025-07-09 17:43:28,481 - root - INFO - lr: 2.7918e-04 gnorm: 0.92 [ 4:08:31<17:57:59] +[titan] 2025-07-09 17:43:32,418 - root - INFO - step: 18740 loss: 16.9111 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.2897 +[titan] 2025-07-09 17:43:32,419 - root - INFO - lr: 2.7917e-04 gnorm: 0.81 [ 4:08:35<17:57:55] +[titan] 2025-07-09 17:43:36,339 - root - INFO - step: 18745 loss: 17.0717 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6558 global_avg_mtp_loss: 14.4159 +[titan] 2025-07-09 17:43:36,340 - root - INFO - lr: 2.7915e-04 gnorm: 0.81 [ 4:08:39<17:57:51] +[titan] 2025-07-09 17:43:39,473 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:43:40,259 - root - INFO - step: 18750 loss: 16.7086 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.1011 +[titan] 2025-07-09 17:43:40,259 - root - INFO - lr: 2.7914e-04 gnorm: 0.78 [ 4:08:43<17:57:47] +[titan] 2025-07-09 17:43:44,226 - root - INFO - step: 18755 loss: 16.9601 memory: 44.58GiB(31.99%) tps: 82,613 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.3071 +[titan] 2025-07-09 17:43:44,226 - root - INFO - lr: 2.7913e-04 gnorm: 0.81 [ 4:08:47<17:57:43] +[titan] 2025-07-09 17:43:48,142 - root - INFO - step: 18760 loss: 16.9243 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6375 global_avg_mtp_loss: 14.2867 +[titan] 2025-07-09 17:43:48,143 - root - INFO - lr: 2.7912e-04 gnorm: 0.85 [ 4:08:51<17:57:38] +[titan] 2025-07-09 17:43:52,057 - root - INFO - step: 18765 loss: 16.7978 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.1895 +[titan] 2025-07-09 17:43:52,057 - root - INFO - lr: 2.7911e-04 gnorm: 0.85 [ 4:08:55<17:57:34] +[titan] 2025-07-09 17:43:55,972 - root - INFO - step: 18770 loss: 16.6934 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.0917 +[titan] 2025-07-09 17:43:55,972 - root - INFO - lr: 2.7910e-04 gnorm: 0.87 [ 4:08:58<17:57:30] +[titan] 2025-07-09 17:43:59,912 - root - INFO - step: 18775 loss: 17.0862 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6738 global_avg_mtp_loss: 14.4124 +[titan] 2025-07-09 17:43:59,912 - root - INFO - lr: 2.7909e-04 gnorm: 0.78 [ 4:09:02<17:57:26] +[titan] 2025-07-09 17:44:03,847 - root - INFO - step: 18780 loss: 17.1848 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6989 global_avg_mtp_loss: 14.4859 +[titan] 2025-07-09 17:44:03,847 - root - INFO - lr: 2.7907e-04 gnorm: 0.84 [ 4:09:06<17:57:22] +[titan] 2025-07-09 17:44:07,759 - root - INFO - step: 18785 loss: 17.2027 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6970 global_avg_mtp_loss: 14.5057 +[titan] 2025-07-09 17:44:07,759 - root - INFO - lr: 2.7906e-04 gnorm: 0.83 [ 4:09:10<17:57:17] +[titan] 2025-07-09 17:44:11,684 - root - INFO - step: 18790 loss: 16.9684 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.6481 global_avg_mtp_loss: 14.3203 +[titan] 2025-07-09 17:44:11,684 - root - INFO - lr: 2.7905e-04 gnorm: 0.85 [ 4:09:14<17:57:13] +[titan] 2025-07-09 17:44:15,593 - root - INFO - step: 18795 loss: 17.0368 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.3854 +[titan] 2025-07-09 17:44:15,593 - root - INFO - lr: 2.7904e-04 gnorm: 0.83 [ 4:09:18<17:57:09] +[titan] 2025-07-09 17:44:18,727 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:44:19,510 - root - INFO - step: 18800 loss: 17.0620 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6545 global_avg_mtp_loss: 14.4075 +[titan] 2025-07-09 17:44:19,510 - root - INFO - lr: 2.7903e-04 gnorm: 0.84 [ 4:09:22<17:57:05] +[titan] 2025-07-09 17:44:23,430 - root - INFO - step: 18805 loss: 16.9343 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6345 global_avg_mtp_loss: 14.2998 +[titan] 2025-07-09 17:44:23,430 - root - INFO - lr: 2.7902e-04 gnorm: 0.84 [ 4:09:26<17:57:00] +[titan] 2025-07-09 17:44:27,324 - root - INFO - step: 18810 loss: 16.7498 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6020 global_avg_mtp_loss: 14.1478 +[titan] 2025-07-09 17:44:27,324 - root - INFO - lr: 2.7901e-04 gnorm: 0.80 [ 4:09:30<17:56:56] +[titan] 2025-07-09 17:44:31,269 - root - INFO - step: 18815 loss: 16.9681 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.6457 global_avg_mtp_loss: 14.3225 +[titan] 2025-07-09 17:44:31,270 - root - INFO - lr: 2.7899e-04 gnorm: 0.85 [ 4:09:34<17:56:52] +[titan] 2025-07-09 17:44:35,176 - root - INFO - step: 18820 loss: 16.9524 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.3181 +[titan] 2025-07-09 17:44:35,177 - root - INFO - lr: 2.7898e-04 gnorm: 0.92 [ 4:09:38<17:56:48] +[titan] 2025-07-09 17:44:39,098 - root - INFO - step: 18825 loss: 17.2447 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.7028 global_avg_mtp_loss: 14.5419 +[titan] 2025-07-09 17:44:39,098 - root - INFO - lr: 2.7897e-04 gnorm: 0.88 [ 4:09:42<17:56:43] +[titan] 2025-07-09 17:44:42,994 - root - INFO - step: 18830 loss: 17.0807 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6721 global_avg_mtp_loss: 14.4086 +[titan] 2025-07-09 17:44:42,994 - root - INFO - lr: 2.7896e-04 gnorm: 0.82 [ 4:09:45<17:56:39] +[titan] 2025-07-09 17:44:46,905 - root - INFO - step: 18835 loss: 16.9694 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.3279 +[titan] 2025-07-09 17:44:46,906 - root - INFO - lr: 2.7895e-04 gnorm: 0.84 [ 4:09:49<17:56:35] +[titan] 2025-07-09 17:44:50,821 - root - INFO - step: 18840 loss: 16.8517 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.2311 +[titan] 2025-07-09 17:44:50,821 - root - INFO - lr: 2.7894e-04 gnorm: 0.82 [ 4:09:53<17:56:30] +[titan] 2025-07-09 17:44:54,737 - root - INFO - step: 18845 loss: 17.1787 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6800 global_avg_mtp_loss: 14.4987 +[titan] 2025-07-09 17:44:54,737 - root - INFO - lr: 2.7892e-04 gnorm: 0.85 [ 4:09:57<17:56:26] +[titan] 2025-07-09 17:44:57,865 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:44:58,657 - root - INFO - step: 18850 loss: 17.1494 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6815 global_avg_mtp_loss: 14.4679 +[titan] 2025-07-09 17:44:58,657 - root - INFO - lr: 2.7891e-04 gnorm: 0.78 [ 4:10:01<17:56:22] +[titan] 2025-07-09 17:45:02,607 - root - INFO - step: 18855 loss: 17.2075 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6777 global_avg_mtp_loss: 14.5299 +[titan] 2025-07-09 17:45:02,608 - root - INFO - lr: 2.7890e-04 gnorm: 0.83 [ 4:10:05<17:56:18] +[titan] 2025-07-09 17:45:06,521 - root - INFO - step: 18860 loss: 16.8988 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6289 global_avg_mtp_loss: 14.2699 +[titan] 2025-07-09 17:45:06,521 - root - INFO - lr: 2.7889e-04 gnorm: 0.86 [ 4:10:09<17:56:14] +[titan] 2025-07-09 17:45:10,437 - root - INFO - step: 18865 loss: 17.0533 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6456 global_avg_mtp_loss: 14.4077 +[titan] 2025-07-09 17:45:10,437 - root - INFO - lr: 2.7888e-04 gnorm: 0.90 [ 4:10:13<17:56:09] +[titan] 2025-07-09 17:45:14,352 - root - INFO - step: 18870 loss: 16.8665 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6232 global_avg_mtp_loss: 14.2433 +[titan] 2025-07-09 17:45:14,352 - root - INFO - lr: 2.7887e-04 gnorm: 0.85 [ 4:10:17<17:56:05] +[titan] 2025-07-09 17:45:18,254 - root - INFO - step: 18875 loss: 16.9800 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6566 global_avg_mtp_loss: 14.3234 +[titan] 2025-07-09 17:45:18,254 - root - INFO - lr: 2.7886e-04 gnorm: 0.83 [ 4:10:21<17:56:01] +[titan] 2025-07-09 17:45:22,165 - root - INFO - step: 18880 loss: 16.9509 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.3175 +[titan] 2025-07-09 17:45:22,166 - root - INFO - lr: 2.7884e-04 gnorm: 0.77 [ 4:10:25<17:55:57] +[titan] 2025-07-09 17:45:26,091 - root - INFO - step: 18885 loss: 17.0293 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.3671 +[titan] 2025-07-09 17:45:26,091 - root - INFO - lr: 2.7883e-04 gnorm: 0.81 [ 4:10:29<17:55:52] +[titan] 2025-07-09 17:45:30,010 - root - INFO - step: 18890 loss: 16.8956 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6356 global_avg_mtp_loss: 14.2600 +[titan] 2025-07-09 17:45:30,010 - root - INFO - lr: 2.7882e-04 gnorm: 0.77 [ 4:10:32<17:55:48] +[titan] 2025-07-09 17:45:34,008 - root - INFO - step: 18895 loss: 17.1558 memory: 44.58GiB(31.99%) tps: 81,967 tflops: 282.88 mfu: 28.60% global_avg_ntp_loss: 2.6741 global_avg_mtp_loss: 14.4817 +[titan] 2025-07-09 17:45:34,008 - root - INFO - lr: 2.7881e-04 gnorm: 0.86 [ 4:10:36<17:55:44] +[titan] 2025-07-09 17:45:37,130 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:45:37,927 - root - INFO - step: 18900 loss: 17.0310 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6557 global_avg_mtp_loss: 14.3753 +[titan] 2025-07-09 17:45:37,928 - root - INFO - lr: 2.7880e-04 gnorm: 0.80 [ 4:10:40<17:55:40] +[titan] 2025-07-09 17:45:41,844 - root - INFO - step: 18905 loss: 17.1749 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6855 global_avg_mtp_loss: 14.4894 +[titan] 2025-07-09 17:45:41,844 - root - INFO - lr: 2.7879e-04 gnorm: 0.84 [ 4:10:44<17:55:36] +[titan] 2025-07-09 17:45:45,745 - root - INFO - step: 18910 loss: 17.0467 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6586 global_avg_mtp_loss: 14.3881 +[titan] 2025-07-09 17:45:45,745 - root - INFO - lr: 2.7878e-04 gnorm: 0.80 [ 4:10:48<17:55:31] +[titan] 2025-07-09 17:45:49,677 - root - INFO - step: 18915 loss: 17.0404 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.6565 global_avg_mtp_loss: 14.3839 +[titan] 2025-07-09 17:45:49,677 - root - INFO - lr: 2.7876e-04 gnorm: 0.89 [ 4:10:52<17:55:27] +[titan] 2025-07-09 17:45:53,601 - root - INFO - step: 18920 loss: 17.0116 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6539 global_avg_mtp_loss: 14.3577 +[titan] 2025-07-09 17:45:53,602 - root - INFO - lr: 2.7875e-04 gnorm: 0.76 [ 4:10:56<17:55:23] +[titan] 2025-07-09 17:45:57,551 - root - INFO - step: 18925 loss: 17.0255 memory: 44.58GiB(31.99%) tps: 82,968 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.6553 global_avg_mtp_loss: 14.3701 +[titan] 2025-07-09 17:45:57,552 - root - INFO - lr: 2.7874e-04 gnorm: 0.87 [ 4:11:00<17:55:19] +[titan] 2025-07-09 17:46:01,505 - root - INFO - step: 18930 loss: 16.9462 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.6348 global_avg_mtp_loss: 14.3113 +[titan] 2025-07-09 17:46:01,505 - root - INFO - lr: 2.7873e-04 gnorm: 0.88 [ 4:11:04<17:55:15] +[titan] 2025-07-09 17:46:05,416 - root - INFO - step: 18935 loss: 17.0272 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6525 global_avg_mtp_loss: 14.3747 +[titan] 2025-07-09 17:46:05,417 - root - INFO - lr: 2.7872e-04 gnorm: 0.85 [ 4:11:08<17:55:10] +[titan] 2025-07-09 17:46:09,356 - root - INFO - step: 18940 loss: 17.1495 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.6767 global_avg_mtp_loss: 14.4728 +[titan] 2025-07-09 17:46:09,357 - root - INFO - lr: 2.7871e-04 gnorm: 0.83 [ 4:11:12<17:55:06] +[titan] 2025-07-09 17:46:12,718 - root - INFO - Dumping profiler traces at step 18944 +[titan] 2025-07-09 17:46:12,750 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:46:13,553 - root - INFO - step: 18945 loss: 17.0972 memory: 44.58GiB(31.99%) tps: 78,085 tflops: 269.48 mfu: 27.25% global_avg_ntp_loss: 2.6680 global_avg_mtp_loss: 14.4292 +[titan] 2025-07-09 17:46:13,553 - root - INFO - lr: 2.7869e-04 gnorm: 0.78 [ 4:11:16<17:55:03] +[titan] 2025-07-09 17:46:16,697 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:46:17,481 - root - INFO - step: 18950 loss: 17.1201 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6580 global_avg_mtp_loss: 14.4621 +[titan] 2025-07-09 17:46:17,482 - root - INFO - lr: 2.7868e-04 gnorm: 0.80 [ 4:11:20<17:54:59] +[titan] 2025-07-09 17:46:21,443 - root - INFO - step: 18955 loss: 16.8682 memory: 44.58GiB(31.99%) tps: 82,716 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.6158 global_avg_mtp_loss: 14.2524 +[titan] 2025-07-09 17:46:21,443 - root - INFO - lr: 2.7867e-04 gnorm: 0.74 [ 4:11:24<17:54:55] +[titan] 2025-07-09 17:46:25,385 - root - INFO - step: 18960 loss: 17.1511 memory: 44.58GiB(31.99%) tps: 83,142 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.6861 global_avg_mtp_loss: 14.4650 +[titan] 2025-07-09 17:46:25,385 - root - INFO - lr: 2.7866e-04 gnorm: 0.77 [ 4:11:28<17:54:51] +[titan] 2025-07-09 17:46:29,321 - root - INFO - step: 18965 loss: 16.5098 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 13.9360 +[titan] 2025-07-09 17:46:29,321 - root - INFO - lr: 2.7865e-04 gnorm: 0.91 [ 4:11:32<17:54:47] +[titan] 2025-07-09 17:46:33,234 - root - INFO - step: 18970 loss: 16.9560 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6398 global_avg_mtp_loss: 14.3162 +[titan] 2025-07-09 17:46:33,234 - root - INFO - lr: 2.7864e-04 gnorm: 0.79 [ 4:11:36<17:54:42] +[titan] 2025-07-09 17:46:37,130 - root - INFO - step: 18975 loss: 17.0687 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6674 global_avg_mtp_loss: 14.4012 +[titan] 2025-07-09 17:46:37,130 - root - INFO - lr: 2.7863e-04 gnorm: 0.80 [ 4:11:40<17:54:38] +[titan] 2025-07-09 17:46:41,027 - root - INFO - step: 18980 loss: 16.8699 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.2273 +[titan] 2025-07-09 17:46:41,028 - root - INFO - lr: 2.7861e-04 gnorm: 0.81 [ 4:11:43<17:54:34] +[titan] 2025-07-09 17:46:44,967 - root - INFO - step: 18985 loss: 17.3121 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.7180 global_avg_mtp_loss: 14.5941 +[titan] 2025-07-09 17:46:44,967 - root - INFO - lr: 2.7860e-04 gnorm: 0.83 [ 4:11:47<17:54:30] +[titan] 2025-07-09 17:46:48,903 - root - INFO - step: 18990 loss: 17.0864 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6643 global_avg_mtp_loss: 14.4221 +[titan] 2025-07-09 17:46:48,903 - root - INFO - lr: 2.7859e-04 gnorm: 0.82 [ 4:11:51<17:54:25] +[titan] 2025-07-09 17:46:52,821 - root - INFO - step: 18995 loss: 16.6725 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5829 global_avg_mtp_loss: 14.0896 +[titan] 2025-07-09 17:46:52,821 - root - INFO - lr: 2.7858e-04 gnorm: 0.88 [ 4:11:55<17:54:21] +[titan] 2025-07-09 17:46:55,925 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:46:56,723 - root - INFO - step: 19000 loss: 17.0785 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6653 global_avg_mtp_loss: 14.4132 +[titan] 2025-07-09 17:46:56,724 - root - INFO - lr: 2.7857e-04 gnorm: 0.88 [ 4:11:59<17:54:17] +[titan] 2025-07-09 17:47:00,662 - root - INFO - step: 19005 loss: 16.6011 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5649 global_avg_mtp_loss: 14.0362 +[titan] 2025-07-09 17:47:00,662 - root - INFO - lr: 2.7856e-04 gnorm: 0.80 [ 4:12:03<17:54:13] +[titan] 2025-07-09 17:47:04,561 - root - INFO - step: 19010 loss: 17.1514 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6877 global_avg_mtp_loss: 14.4636 +[titan] 2025-07-09 17:47:04,562 - root - INFO - lr: 2.7854e-04 gnorm: 0.82 [ 4:12:07<17:54:08] +[titan] 2025-07-09 17:47:08,455 - root - INFO - step: 19015 loss: 16.8993 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2722 +[titan] 2025-07-09 17:47:08,456 - root - INFO - lr: 2.7853e-04 gnorm: 0.77 [ 4:12:11<17:54:04] +[titan] 2025-07-09 17:47:12,348 - root - INFO - step: 19020 loss: 16.9391 memory: 44.58GiB(31.99%) tps: 84,192 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.6353 global_avg_mtp_loss: 14.3039 +[titan] 2025-07-09 17:47:12,348 - root - INFO - lr: 2.7852e-04 gnorm: 0.90 [ 4:12:15<17:54:00] +[titan] 2025-07-09 17:47:16,273 - root - INFO - step: 19025 loss: 17.2092 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.6829 global_avg_mtp_loss: 14.5263 +[titan] 2025-07-09 17:47:16,273 - root - INFO - lr: 2.7851e-04 gnorm: 0.81 [ 4:12:19<17:53:56] +[titan] 2025-07-09 17:47:20,199 - root - INFO - step: 19030 loss: 16.9382 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.2945 +[titan] 2025-07-09 17:47:20,199 - root - INFO - lr: 2.7850e-04 gnorm: 0.83 [ 4:12:23<17:53:51] +[titan] 2025-07-09 17:47:24,110 - root - INFO - step: 19035 loss: 16.9121 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6268 global_avg_mtp_loss: 14.2853 +[titan] 2025-07-09 17:47:24,110 - root - INFO - lr: 2.7849e-04 gnorm: 0.77 [ 4:12:27<17:53:47] +[titan] 2025-07-09 17:47:28,055 - root - INFO - step: 19040 loss: 16.7693 memory: 44.58GiB(31.99%) tps: 83,076 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.1410 +[titan] 2025-07-09 17:47:28,055 - root - INFO - lr: 2.7847e-04 gnorm: 0.92 [ 4:12:30<17:53:43] +[titan] 2025-07-09 17:47:31,969 - root - INFO - step: 19045 loss: 17.1554 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6734 global_avg_mtp_loss: 14.4821 +[titan] 2025-07-09 17:47:31,969 - root - INFO - lr: 2.7846e-04 gnorm: 0.83 [ 4:12:34<17:53:39] +[titan] 2025-07-09 17:47:35,111 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:47:35,910 - root - INFO - step: 19050 loss: 16.9289 memory: 44.58GiB(31.99%) tps: 83,147 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.2799 +[titan] 2025-07-09 17:47:35,910 - root - INFO - lr: 2.7845e-04 gnorm: 1.63 [ 4:12:38<17:53:35] +[titan] 2025-07-09 17:47:39,818 - root - INFO - step: 19055 loss: 17.0155 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6511 global_avg_mtp_loss: 14.3645 +[titan] 2025-07-09 17:47:39,818 - root - INFO - lr: 2.7844e-04 gnorm: 0.84 [ 4:12:42<17:53:30] +[titan] 2025-07-09 17:47:43,763 - root - INFO - step: 19060 loss: 16.8142 memory: 44.58GiB(31.99%) tps: 83,068 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.6183 global_avg_mtp_loss: 14.1960 +[titan] 2025-07-09 17:47:43,763 - root - INFO - lr: 2.7843e-04 gnorm: 0.83 [ 4:12:46<17:53:26] +[titan] 2025-07-09 17:47:47,718 - root - INFO - step: 19065 loss: 17.0032 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.6462 global_avg_mtp_loss: 14.3570 +[titan] 2025-07-09 17:47:47,718 - root - INFO - lr: 2.7842e-04 gnorm: 0.85 [ 4:12:50<17:53:22] +[titan] 2025-07-09 17:47:51,621 - root - INFO - step: 19070 loss: 17.2509 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.7028 global_avg_mtp_loss: 14.5481 +[titan] 2025-07-09 17:47:51,621 - root - INFO - lr: 2.7840e-04 gnorm: 0.86 [ 4:12:54<17:53:18] +[titan] 2025-07-09 17:47:55,569 - root - INFO - step: 19075 loss: 16.8280 memory: 44.58GiB(31.99%) tps: 82,997 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.6146 global_avg_mtp_loss: 14.2134 +[titan] 2025-07-09 17:47:55,570 - root - INFO - lr: 2.7839e-04 gnorm: 0.81 [ 4:12:58<17:53:14] +[titan] 2025-07-09 17:47:59,470 - root - INFO - step: 19080 loss: 17.1191 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.6750 global_avg_mtp_loss: 14.4441 +[titan] 2025-07-09 17:47:59,470 - root - INFO - lr: 2.7838e-04 gnorm: 0.80 [ 4:13:02<17:53:09] +[titan] 2025-07-09 17:48:03,387 - root - INFO - step: 19085 loss: 16.9004 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.2727 +[titan] 2025-07-09 17:48:03,388 - root - INFO - lr: 2.7837e-04 gnorm: 0.81 [ 4:13:06<17:53:05] +[titan] 2025-07-09 17:48:07,319 - root - INFO - step: 19090 loss: 16.7515 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6116 global_avg_mtp_loss: 14.1399 +[titan] 2025-07-09 17:48:07,319 - root - INFO - lr: 2.7836e-04 gnorm: 0.76 [ 4:13:10<17:53:01] +[titan] 2025-07-09 17:48:11,276 - root - INFO - step: 19095 loss: 17.0783 memory: 44.58GiB(31.99%) tps: 82,810 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.6733 global_avg_mtp_loss: 14.4050 +[titan] 2025-07-09 17:48:11,277 - root - INFO - lr: 2.7835e-04 gnorm: 0.84 [ 4:13:14<17:52:57] +[titan] 2025-07-09 17:48:14,397 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:48:15,182 - root - INFO - step: 19100 loss: 17.1498 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6789 global_avg_mtp_loss: 14.4709 +[titan] 2025-07-09 17:48:15,183 - root - INFO - lr: 2.7834e-04 gnorm: 0.77 [ 4:13:18<17:52:53] +[titan] 2025-07-09 17:48:19,112 - root - INFO - step: 19105 loss: 17.2043 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.7013 global_avg_mtp_loss: 14.5030 +[titan] 2025-07-09 17:48:19,112 - root - INFO - lr: 2.7832e-04 gnorm: 0.83 [ 4:13:22<17:52:48] +[titan] 2025-07-09 17:48:23,108 - root - INFO - step: 19110 loss: 17.0739 memory: 44.58GiB(31.99%) tps: 82,002 tflops: 283.00 mfu: 28.61% global_avg_ntp_loss: 2.6602 global_avg_mtp_loss: 14.4137 +[titan] 2025-07-09 17:48:23,109 - root - INFO - lr: 2.7831e-04 gnorm: 0.79 [ 4:13:26<17:52:45] +[titan] 2025-07-09 17:48:27,005 - root - INFO - step: 19115 loss: 17.1179 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.6650 global_avg_mtp_loss: 14.4529 +[titan] 2025-07-09 17:48:27,005 - root - INFO - lr: 2.7830e-04 gnorm: 0.81 [ 4:13:29<17:52:40] +[titan] 2025-07-09 17:48:30,937 - root - INFO - step: 19120 loss: 17.2609 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.7153 global_avg_mtp_loss: 14.5455 +[titan] 2025-07-09 17:48:30,938 - root - INFO - lr: 2.7829e-04 gnorm: 0.86 [ 4:13:33<17:52:36] +[titan] 2025-07-09 17:48:34,837 - root - INFO - step: 19125 loss: 17.2961 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.7168 global_avg_mtp_loss: 14.5793 +[titan] 2025-07-09 17:48:34,837 - root - INFO - lr: 2.7828e-04 gnorm: 0.94 [ 4:13:37<17:52:32] +[titan] 2025-07-09 17:48:38,756 - root - INFO - step: 19130 loss: 16.6041 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.0137 +[titan] 2025-07-09 17:48:38,756 - root - INFO - lr: 2.7827e-04 gnorm: 0.95 [ 4:13:41<17:52:27] +[titan] 2025-07-09 17:48:42,656 - root - INFO - step: 19135 loss: 17.0495 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6614 global_avg_mtp_loss: 14.3881 +[titan] 2025-07-09 17:48:42,656 - root - INFO - lr: 2.7825e-04 gnorm: 0.82 [ 4:13:45<17:52:23] +[titan] 2025-07-09 17:48:46,559 - root - INFO - step: 19140 loss: 17.1871 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6976 global_avg_mtp_loss: 14.4895 +[titan] 2025-07-09 17:48:46,559 - root - INFO - lr: 2.7824e-04 gnorm: 0.82 [ 4:13:49<17:52:19] +[titan] 2025-07-09 17:48:50,460 - root - INFO - step: 19145 loss: 16.6871 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.0974 +[titan] 2025-07-09 17:48:50,460 - root - INFO - lr: 2.7823e-04 gnorm: 0.79 [ 4:13:53<17:52:15] +[titan] 2025-07-09 17:48:53,574 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:48:54,367 - root - INFO - step: 19150 loss: 17.1187 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.4522 +[titan] 2025-07-09 17:48:54,367 - root - INFO - lr: 2.7822e-04 gnorm: 0.76 [ 4:13:57<17:52:10] +[titan] 2025-07-09 17:48:58,352 - root - INFO - step: 19155 loss: 17.1258 memory: 44.58GiB(31.99%) tps: 82,236 tflops: 283.81 mfu: 28.70% global_avg_ntp_loss: 2.6886 global_avg_mtp_loss: 14.4372 +[titan] 2025-07-09 17:48:58,352 - root - INFO - lr: 2.7821e-04 gnorm: 0.86 [ 4:14:01<17:52:06] +[titan] 2025-07-09 17:49:02,263 - root - INFO - step: 19160 loss: 17.0154 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6593 global_avg_mtp_loss: 14.3561 +[titan] 2025-07-09 17:49:02,263 - root - INFO - lr: 2.7820e-04 gnorm: 0.79 [ 4:14:05<17:52:02] +[titan] 2025-07-09 17:49:06,172 - root - INFO - step: 19165 loss: 17.1231 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.6711 global_avg_mtp_loss: 14.4521 +[titan] 2025-07-09 17:49:06,173 - root - INFO - lr: 2.7818e-04 gnorm: 0.83 [ 4:14:09<17:51:58] +[titan] 2025-07-09 17:49:10,166 - root - INFO - step: 19170 loss: 16.9875 memory: 44.58GiB(31.99%) tps: 82,067 tflops: 283.23 mfu: 28.64% global_avg_ntp_loss: 2.6441 global_avg_mtp_loss: 14.3435 +[titan] 2025-07-09 17:49:10,166 - root - INFO - lr: 2.7817e-04 gnorm: 0.77 [ 4:14:13<17:51:54] +[titan] 2025-07-09 17:49:14,083 - root - INFO - step: 19175 loss: 17.0895 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6701 global_avg_mtp_loss: 14.4194 +[titan] 2025-07-09 17:49:14,083 - root - INFO - lr: 2.7816e-04 gnorm: 0.77 [ 4:14:16<17:51:50] +[titan] 2025-07-09 17:49:18,002 - root - INFO - step: 19180 loss: 17.0939 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6717 global_avg_mtp_loss: 14.4222 +[titan] 2025-07-09 17:49:18,002 - root - INFO - lr: 2.7815e-04 gnorm: 0.86 [ 4:14:20<17:51:45] +[titan] 2025-07-09 17:49:21,954 - root - INFO - step: 19185 loss: 16.8383 memory: 44.58GiB(31.99%) tps: 82,921 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.6218 global_avg_mtp_loss: 14.2164 +[titan] 2025-07-09 17:49:21,954 - root - INFO - lr: 2.7814e-04 gnorm: 0.83 [ 4:14:24<17:51:41] +[titan] 2025-07-09 17:49:25,864 - root - INFO - step: 19190 loss: 16.7874 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6219 global_avg_mtp_loss: 14.1655 +[titan] 2025-07-09 17:49:25,864 - root - INFO - lr: 2.7813e-04 gnorm: 0.86 [ 4:14:28<17:51:37] +[titan] 2025-07-09 17:49:29,785 - root - INFO - step: 19195 loss: 16.8701 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6244 global_avg_mtp_loss: 14.2457 +[titan] 2025-07-09 17:49:29,785 - root - INFO - lr: 2.7811e-04 gnorm: 0.79 [ 4:14:32<17:51:33] +[titan] 2025-07-09 17:49:32,892 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:49:33,680 - root - INFO - step: 19200 loss: 17.0324 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6623 global_avg_mtp_loss: 14.3701 +[titan] 2025-07-09 17:49:33,680 - root - INFO - lr: 2.7810e-04 gnorm: 0.79 [ 4:14:36<17:51:28] +[titan] 2025-07-09 17:49:37,577 - root - INFO - step: 19205 loss: 17.1722 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6828 global_avg_mtp_loss: 14.4894 +[titan] 2025-07-09 17:49:37,578 - root - INFO - lr: 2.7809e-04 gnorm: 0.76 [ 4:14:40<17:51:24] +[titan] 2025-07-09 17:49:41,475 - root - INFO - step: 19210 loss: 17.0156 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6417 global_avg_mtp_loss: 14.3739 +[titan] 2025-07-09 17:49:41,475 - root - INFO - lr: 2.7808e-04 gnorm: 0.85 [ 4:14:44<17:51:20] +[titan] 2025-07-09 17:49:45,381 - root - INFO - step: 19215 loss: 17.0473 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6713 global_avg_mtp_loss: 14.3759 +[titan] 2025-07-09 17:49:45,381 - root - INFO - lr: 2.7807e-04 gnorm: 0.85 [ 4:14:48<17:51:16] +[titan] 2025-07-09 17:49:49,278 - root - INFO - step: 19220 loss: 16.8750 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2510 +[titan] 2025-07-09 17:49:49,278 - root - INFO - lr: 2.7806e-04 gnorm: 0.78 [ 4:14:52<17:51:11] +[titan] 2025-07-09 17:49:53,191 - root - INFO - step: 19225 loss: 16.9094 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.2691 +[titan] 2025-07-09 17:49:53,192 - root - INFO - lr: 2.7804e-04 gnorm: 0.77 [ 4:14:56<17:51:07] +[titan] 2025-07-09 17:49:57,108 - root - INFO - step: 19230 loss: 17.0952 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6634 global_avg_mtp_loss: 14.4318 +[titan] 2025-07-09 17:49:57,109 - root - INFO - lr: 2.7803e-04 gnorm: 0.79 [ 4:15:00<17:51:03] +[titan] 2025-07-09 17:50:01,010 - root - INFO - step: 19235 loss: 16.9145 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.2826 +[titan] 2025-07-09 17:50:01,010 - root - INFO - lr: 2.7802e-04 gnorm: 0.80 [ 4:15:03<17:50:58] +[titan] 2025-07-09 17:50:04,939 - root - INFO - step: 19240 loss: 17.1655 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6846 global_avg_mtp_loss: 14.4809 +[titan] 2025-07-09 17:50:04,939 - root - INFO - lr: 2.7801e-04 gnorm: 0.73 [ 4:15:07<17:50:54] +[titan] 2025-07-09 17:50:08,860 - root - INFO - step: 19245 loss: 17.3149 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.7204 global_avg_mtp_loss: 14.5945 +[titan] 2025-07-09 17:50:08,861 - root - INFO - lr: 2.7800e-04 gnorm: 0.86 [ 4:15:11<17:50:50] +[titan] 2025-07-09 17:50:11,982 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:50:12,764 - root - INFO - step: 19250 loss: 17.0249 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6593 global_avg_mtp_loss: 14.3655 +[titan] 2025-07-09 17:50:12,764 - root - INFO - lr: 2.7798e-04 gnorm: 0.84 [ 4:15:15<17:50:46] +[titan] 2025-07-09 17:50:16,690 - root - INFO - step: 19255 loss: 17.0682 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.4041 +[titan] 2025-07-09 17:50:16,690 - root - INFO - lr: 2.7797e-04 gnorm: 0.82 [ 4:15:19<17:50:42] +[titan] 2025-07-09 17:50:20,597 - root - INFO - step: 19260 loss: 16.8542 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.2178 +[titan] 2025-07-09 17:50:20,598 - root - INFO - lr: 2.7796e-04 gnorm: 0.84 [ 4:15:23<17:50:37] +[titan] 2025-07-09 17:50:24,497 - root - INFO - step: 19265 loss: 17.2949 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.7144 global_avg_mtp_loss: 14.5805 +[titan] 2025-07-09 17:50:24,497 - root - INFO - lr: 2.7795e-04 gnorm: 0.84 [ 4:15:27<17:50:33] +[titan] 2025-07-09 17:50:28,419 - root - INFO - step: 19270 loss: 17.1731 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6735 global_avg_mtp_loss: 14.4996 +[titan] 2025-07-09 17:50:28,420 - root - INFO - lr: 2.7794e-04 gnorm: 0.77 [ 4:15:31<17:50:29] +[titan] 2025-07-09 17:50:32,377 - root - INFO - step: 19275 loss: 16.9411 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.3126 +[titan] 2025-07-09 17:50:32,378 - root - INFO - lr: 2.7793e-04 gnorm: 0.77 [ 4:15:35<17:50:25] +[titan] 2025-07-09 17:50:36,282 - root - INFO - step: 19280 loss: 16.7514 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.1506 +[titan] 2025-07-09 17:50:36,282 - root - INFO - lr: 2.7791e-04 gnorm: 0.84 [ 4:15:39<17:50:20] +[titan] 2025-07-09 17:50:40,207 - root - INFO - step: 19285 loss: 17.0631 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.6611 global_avg_mtp_loss: 14.4020 +[titan] 2025-07-09 17:50:40,207 - root - INFO - lr: 2.7790e-04 gnorm: 0.82 [ 4:15:43<17:50:16] +[titan] 2025-07-09 17:50:44,129 - root - INFO - step: 19290 loss: 17.0922 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6659 global_avg_mtp_loss: 14.4263 +[titan] 2025-07-09 17:50:44,130 - root - INFO - lr: 2.7789e-04 gnorm: 0.97 [ 4:15:47<17:50:12] +[titan] 2025-07-09 17:50:48,085 - root - INFO - step: 19295 loss: 17.0609 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.6611 global_avg_mtp_loss: 14.3999 +[titan] 2025-07-09 17:50:48,085 - root - INFO - lr: 2.7788e-04 gnorm: 0.80 [ 4:15:50<17:50:08] +[titan] 2025-07-09 17:50:51,197 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:50:52,004 - root - INFO - step: 19300 loss: 16.8447 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6346 global_avg_mtp_loss: 14.2101 +[titan] 2025-07-09 17:50:52,004 - root - INFO - lr: 2.7787e-04 gnorm: 0.77 [ 4:15:54<17:50:04] +[titan] 2025-07-09 17:50:55,937 - root - INFO - step: 19305 loss: 16.8270 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6313 global_avg_mtp_loss: 14.1957 +[titan] 2025-07-09 17:50:55,937 - root - INFO - lr: 2.7786e-04 gnorm: 0.78 [ 4:15:58<17:50:00] +[titan] 2025-07-09 17:50:59,849 - root - INFO - step: 19310 loss: 17.0300 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6484 global_avg_mtp_loss: 14.3816 +[titan] 2025-07-09 17:50:59,850 - root - INFO - lr: 2.7784e-04 gnorm: 0.80 [ 4:16:02<17:49:55] +[titan] 2025-07-09 17:51:03,755 - root - INFO - step: 19315 loss: 17.0252 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6737 global_avg_mtp_loss: 14.3515 +[titan] 2025-07-09 17:51:03,755 - root - INFO - lr: 2.7783e-04 gnorm: 0.78 [ 4:16:06<17:49:51] +[titan] 2025-07-09 17:51:07,649 - root - INFO - step: 19320 loss: 16.8582 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.2440 +[titan] 2025-07-09 17:51:07,649 - root - INFO - lr: 2.7782e-04 gnorm: 0.86 [ 4:16:10<17:49:47] +[titan] 2025-07-09 17:51:11,560 - root - INFO - step: 19325 loss: 16.9325 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6396 global_avg_mtp_loss: 14.2929 +[titan] 2025-07-09 17:51:11,560 - root - INFO - lr: 2.7781e-04 gnorm: 0.75 [ 4:16:14<17:49:42] +[titan] 2025-07-09 17:51:15,497 - root - INFO - step: 19330 loss: 16.7228 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 17:51:15,498 - root - INFO - lr: 2.7780e-04 gnorm: 0.76 [ 4:16:18<17:49:38] +[titan] 2025-07-09 17:51:19,418 - root - INFO - step: 19335 loss: 16.9474 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.3103 +[titan] 2025-07-09 17:51:19,418 - root - INFO - lr: 2.7779e-04 gnorm: 0.83 [ 4:16:22<17:49:34] +[titan] 2025-07-09 17:51:23,338 - root - INFO - step: 19340 loss: 17.2067 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.5222 +[titan] 2025-07-09 17:51:23,338 - root - INFO - lr: 2.7777e-04 gnorm: 0.81 [ 4:16:26<17:49:30] +[titan] 2025-07-09 17:51:27,234 - root - INFO - step: 19345 loss: 17.1713 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6706 global_avg_mtp_loss: 14.5006 +[titan] 2025-07-09 17:51:27,235 - root - INFO - lr: 2.7776e-04 gnorm: 0.81 [ 4:16:30<17:49:25] +[titan] 2025-07-09 17:51:30,362 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:51:31,151 - root - INFO - step: 19350 loss: 16.8897 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6439 global_avg_mtp_loss: 14.2458 +[titan] 2025-07-09 17:51:31,151 - root - INFO - lr: 2.7775e-04 gnorm: 0.83 [ 4:16:34<17:49:21] +[titan] 2025-07-09 17:51:35,054 - root - INFO - step: 19355 loss: 16.7457 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.6055 global_avg_mtp_loss: 14.1402 +[titan] 2025-07-09 17:51:35,054 - root - INFO - lr: 2.7774e-04 gnorm: 0.79 [ 4:16:37<17:49:17] +[titan] 2025-07-09 17:51:38,970 - root - INFO - step: 19360 loss: 16.8592 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6268 global_avg_mtp_loss: 14.2324 +[titan] 2025-07-09 17:51:38,971 - root - INFO - lr: 2.7773e-04 gnorm: 0.77 [ 4:16:41<17:49:13] +[titan] 2025-07-09 17:51:42,890 - root - INFO - step: 19365 loss: 16.9372 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.2951 +[titan] 2025-07-09 17:51:42,891 - root - INFO - lr: 2.7771e-04 gnorm: 0.84 [ 4:16:45<17:49:08] +[titan] 2025-07-09 17:51:46,812 - root - INFO - step: 19370 loss: 16.8405 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6157 global_avg_mtp_loss: 14.2247 +[titan] 2025-07-09 17:51:46,812 - root - INFO - lr: 2.7770e-04 gnorm: 0.90 [ 4:16:49<17:49:04] +[titan] 2025-07-09 17:51:50,715 - root - INFO - step: 19375 loss: 17.1481 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6769 global_avg_mtp_loss: 14.4712 +[titan] 2025-07-09 17:51:50,715 - root - INFO - lr: 2.7769e-04 gnorm: 0.76 [ 4:16:53<17:49:00] +[titan] 2025-07-09 17:51:54,623 - root - INFO - step: 19380 loss: 17.1280 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4524 +[titan] 2025-07-09 17:51:54,623 - root - INFO - lr: 2.7768e-04 gnorm: 0.79 [ 4:16:57<17:48:56] +[titan] 2025-07-09 17:51:58,533 - root - INFO - step: 19385 loss: 17.0032 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6635 global_avg_mtp_loss: 14.3397 +[titan] 2025-07-09 17:51:58,533 - root - INFO - lr: 2.7767e-04 gnorm: 0.82 [ 4:17:01<17:48:51] +[titan] 2025-07-09 17:52:02,432 - root - INFO - step: 19390 loss: 16.6365 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.0423 +[titan] 2025-07-09 17:52:02,432 - root - INFO - lr: 2.7766e-04 gnorm: 0.84 [ 4:17:05<17:48:47] +[titan] 2025-07-09 17:52:06,332 - root - INFO - step: 19395 loss: 16.8393 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2114 +[titan] 2025-07-09 17:52:06,333 - root - INFO - lr: 2.7764e-04 gnorm: 0.82 [ 4:17:09<17:48:43] +[titan] 2025-07-09 17:52:09,462 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:52:10,250 - root - INFO - step: 19400 loss: 16.9258 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6398 global_avg_mtp_loss: 14.2860 +[titan] 2025-07-09 17:52:10,250 - root - INFO - lr: 2.7763e-04 gnorm: 0.78 [ 4:17:13<17:48:39] +[titan] 2025-07-09 17:52:14,174 - root - INFO - step: 19405 loss: 16.9895 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6471 global_avg_mtp_loss: 14.3423 +[titan] 2025-07-09 17:52:14,174 - root - INFO - lr: 2.7762e-04 gnorm: 0.77 [ 4:17:17<17:48:34] +[titan] 2025-07-09 17:52:18,096 - root - INFO - step: 19410 loss: 17.0828 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6629 global_avg_mtp_loss: 14.4199 +[titan] 2025-07-09 17:52:18,096 - root - INFO - lr: 2.7761e-04 gnorm: 0.74 [ 4:17:20<17:48:30] +[titan] 2025-07-09 17:52:22,036 - root - INFO - step: 19415 loss: 16.9844 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6457 global_avg_mtp_loss: 14.3388 +[titan] 2025-07-09 17:52:22,036 - root - INFO - lr: 2.7760e-04 gnorm: 0.92 [ 4:17:24<17:48:26] +[titan] 2025-07-09 17:52:25,980 - root - INFO - step: 19420 loss: 17.0331 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.6624 global_avg_mtp_loss: 14.3707 +[titan] 2025-07-09 17:52:25,980 - root - INFO - lr: 2.7758e-04 gnorm: 0.84 [ 4:17:28<17:48:22] +[titan] 2025-07-09 17:52:29,913 - root - INFO - step: 19425 loss: 16.8400 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6162 global_avg_mtp_loss: 14.2238 +[titan] 2025-07-09 17:52:29,913 - root - INFO - lr: 2.7757e-04 gnorm: 0.83 [ 4:17:32<17:48:18] +[titan] 2025-07-09 17:52:33,827 - root - INFO - step: 19430 loss: 17.0609 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6657 global_avg_mtp_loss: 14.3951 +[titan] 2025-07-09 17:52:33,827 - root - INFO - lr: 2.7756e-04 gnorm: 0.84 [ 4:17:36<17:48:14] +[titan] 2025-07-09 17:52:37,753 - root - INFO - step: 19435 loss: 17.0948 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6659 global_avg_mtp_loss: 14.4289 +[titan] 2025-07-09 17:52:37,754 - root - INFO - lr: 2.7755e-04 gnorm: 0.89 [ 4:17:40<17:48:09] +[titan] 2025-07-09 17:52:41,674 - root - INFO - step: 19440 loss: 16.9265 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.2819 +[titan] 2025-07-09 17:52:41,674 - root - INFO - lr: 2.7754e-04 gnorm: 0.85 [ 4:17:44<17:48:05] +[titan] 2025-07-09 17:52:45,590 - root - INFO - step: 19445 loss: 16.9684 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6386 global_avg_mtp_loss: 14.3298 +[titan] 2025-07-09 17:52:45,590 - root - INFO - lr: 2.7753e-04 gnorm: 0.76 [ 4:17:48<17:48:01] +[titan] 2025-07-09 17:52:48,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:52:49,502 - root - INFO - step: 19450 loss: 17.1492 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6950 global_avg_mtp_loss: 14.4542 +[titan] 2025-07-09 17:52:49,502 - root - INFO - lr: 2.7751e-04 gnorm: 0.76 [ 4:17:52<17:47:57] +[titan] 2025-07-09 17:52:53,480 - root - INFO - step: 19455 loss: 17.0469 memory: 44.58GiB(31.99%) tps: 82,366 tflops: 284.26 mfu: 28.74% global_avg_ntp_loss: 2.6449 global_avg_mtp_loss: 14.4019 +[titan] 2025-07-09 17:52:53,481 - root - INFO - lr: 2.7750e-04 gnorm: 0.82 [ 4:17:56<17:47:53] +[titan] 2025-07-09 17:52:54,424 - root - INFO - Dumping profiler traces at step 19456 +[titan] 2025-07-09 17:52:54,456 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:52:57,603 - root - INFO - step: 19460 loss: 16.9388 memory: 44.58GiB(31.99%) tps: 79,497 tflops: 274.36 mfu: 27.74% global_avg_ntp_loss: 2.6339 global_avg_mtp_loss: 14.3049 +[titan] 2025-07-09 17:52:57,603 - root - INFO - lr: 2.7749e-04 gnorm: 0.77 [ 4:18:00<17:47:49] +[titan] 2025-07-09 17:53:01,516 - root - INFO - step: 19465 loss: 16.8901 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6293 global_avg_mtp_loss: 14.2609 +[titan] 2025-07-09 17:53:01,516 - root - INFO - lr: 2.7748e-04 gnorm: 0.78 [ 4:18:04<17:47:45] +[titan] 2025-07-09 17:53:05,420 - root - INFO - step: 19470 loss: 16.7534 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6036 global_avg_mtp_loss: 14.1498 +[titan] 2025-07-09 17:53:05,420 - root - INFO - lr: 2.7747e-04 gnorm: 0.76 [ 4:18:08<17:47:41] +[titan] 2025-07-09 17:53:09,376 - root - INFO - step: 19475 loss: 17.3296 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.7195 global_avg_mtp_loss: 14.6101 +[titan] 2025-07-09 17:53:09,377 - root - INFO - lr: 2.7745e-04 gnorm: 0.78 [ 4:18:12<17:47:37] +[titan] 2025-07-09 17:53:13,304 - root - INFO - step: 19480 loss: 17.1613 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6897 global_avg_mtp_loss: 14.4716 +[titan] 2025-07-09 17:53:13,304 - root - INFO - lr: 2.7744e-04 gnorm: 0.90 [ 4:18:16<17:47:33] +[titan] 2025-07-09 17:53:17,204 - root - INFO - step: 19485 loss: 16.8516 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2430 +[titan] 2025-07-09 17:53:17,204 - root - INFO - lr: 2.7743e-04 gnorm: 0.81 [ 4:18:20<17:47:28] +[titan] 2025-07-09 17:53:21,129 - root - INFO - step: 19490 loss: 17.0024 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.3558 +[titan] 2025-07-09 17:53:21,129 - root - INFO - lr: 2.7742e-04 gnorm: 0.87 [ 4:18:24<17:47:24] +[titan] 2025-07-09 17:53:25,058 - root - INFO - step: 19495 loss: 17.0048 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.3668 +[titan] 2025-07-09 17:53:25,058 - root - INFO - lr: 2.7741e-04 gnorm: 0.81 [ 4:18:27<17:47:20] +[titan] 2025-07-09 17:53:28,206 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:53:28,995 - root - INFO - step: 19500 loss: 17.0843 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.6481 global_avg_mtp_loss: 14.4362 +[titan] 2025-07-09 17:53:28,995 - root - INFO - lr: 2.7740e-04 gnorm: 0.86 [ 4:18:31<17:47:16] +[titan] 2025-07-09 17:53:32,904 - root - INFO - step: 19505 loss: 16.9975 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6318 global_avg_mtp_loss: 14.3657 +[titan] 2025-07-09 17:53:32,904 - root - INFO - lr: 2.7738e-04 gnorm: 0.79 [ 4:18:35<17:47:11] +[titan] 2025-07-09 17:53:36,792 - root - INFO - step: 19510 loss: 16.8371 memory: 44.58GiB(31.99%) tps: 84,275 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2112 +[titan] 2025-07-09 17:53:36,792 - root - INFO - lr: 2.7737e-04 gnorm: 0.76 [ 4:18:39<17:47:07] +[titan] 2025-07-09 17:53:40,686 - root - INFO - step: 19515 loss: 16.7364 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.6249 global_avg_mtp_loss: 14.1115 +[titan] 2025-07-09 17:53:40,687 - root - INFO - lr: 2.7736e-04 gnorm: 1.24 [ 4:18:43<17:47:03] +[titan] 2025-07-09 17:53:44,614 - root - INFO - step: 19520 loss: 17.0236 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.6642 global_avg_mtp_loss: 14.3593 +[titan] 2025-07-09 17:53:44,614 - root - INFO - lr: 2.7735e-04 gnorm: 0.83 [ 4:18:47<17:46:59] +[titan] 2025-07-09 17:53:48,532 - root - INFO - step: 19525 loss: 16.9430 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6327 global_avg_mtp_loss: 14.3103 +[titan] 2025-07-09 17:53:48,532 - root - INFO - lr: 2.7734e-04 gnorm: 0.77 [ 4:18:51<17:46:54] +[titan] 2025-07-09 17:53:52,426 - root - INFO - step: 19530 loss: 16.9690 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.6512 global_avg_mtp_loss: 14.3177 +[titan] 2025-07-09 17:53:52,426 - root - INFO - lr: 2.7732e-04 gnorm: 0.84 [ 4:18:55<17:46:50] +[titan] 2025-07-09 17:53:56,349 - root - INFO - step: 19535 loss: 17.2415 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6951 global_avg_mtp_loss: 14.5464 +[titan] 2025-07-09 17:53:56,349 - root - INFO - lr: 2.7731e-04 gnorm: 0.74 [ 4:18:59<17:46:46] +[titan] 2025-07-09 17:54:00,268 - root - INFO - step: 19540 loss: 17.1067 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6711 global_avg_mtp_loss: 14.4356 +[titan] 2025-07-09 17:54:00,268 - root - INFO - lr: 2.7730e-04 gnorm: 0.81 [ 4:19:03<17:46:42] +[titan] 2025-07-09 17:54:04,179 - root - INFO - step: 19545 loss: 17.0195 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.3665 +[titan] 2025-07-09 17:54:04,179 - root - INFO - lr: 2.7729e-04 gnorm: 0.76 [ 4:19:07<17:46:37] +[titan] 2025-07-09 17:54:07,284 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:54:08,074 - root - INFO - step: 19550 loss: 16.8740 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.6296 global_avg_mtp_loss: 14.2443 +[titan] 2025-07-09 17:54:08,074 - root - INFO - lr: 2.7728e-04 gnorm: 0.82 [ 4:19:10<17:46:33] +[titan] 2025-07-09 17:54:11,995 - root - INFO - step: 19555 loss: 16.8815 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6179 global_avg_mtp_loss: 14.2636 +[titan] 2025-07-09 17:54:11,995 - root - INFO - lr: 2.7726e-04 gnorm: 0.78 [ 4:19:14<17:46:29] +[titan] 2025-07-09 17:54:15,949 - root - INFO - step: 19560 loss: 17.2608 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.7024 global_avg_mtp_loss: 14.5584 +[titan] 2025-07-09 17:54:15,949 - root - INFO - lr: 2.7725e-04 gnorm: 0.85 [ 4:19:18<17:46:25] +[titan] 2025-07-09 17:54:19,866 - root - INFO - step: 19565 loss: 16.7299 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6028 global_avg_mtp_loss: 14.1272 +[titan] 2025-07-09 17:54:19,866 - root - INFO - lr: 2.7724e-04 gnorm: 0.80 [ 4:19:22<17:46:21] +[titan] 2025-07-09 17:54:23,823 - root - INFO - step: 19570 loss: 16.9514 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.3131 +[titan] 2025-07-09 17:54:23,823 - root - INFO - lr: 2.7723e-04 gnorm: 0.77 [ 4:19:26<17:46:16] +[titan] 2025-07-09 17:54:27,737 - root - INFO - step: 19575 loss: 16.9285 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.3024 +[titan] 2025-07-09 17:54:27,737 - root - INFO - lr: 2.7722e-04 gnorm: 0.86 [ 4:19:30<17:46:12] +[titan] 2025-07-09 17:54:31,673 - root - INFO - step: 19580 loss: 17.0865 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.4269 +[titan] 2025-07-09 17:54:31,673 - root - INFO - lr: 2.7720e-04 gnorm: 0.77 [ 4:19:34<17:46:08] +[titan] 2025-07-09 17:54:35,638 - root - INFO - step: 19585 loss: 17.0582 memory: 44.58GiB(31.99%) tps: 82,658 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.4023 +[titan] 2025-07-09 17:54:35,638 - root - INFO - lr: 2.7719e-04 gnorm: 0.79 [ 4:19:38<17:46:04] +[titan] 2025-07-09 17:54:39,554 - root - INFO - step: 19590 loss: 17.0933 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6598 global_avg_mtp_loss: 14.4336 +[titan] 2025-07-09 17:54:39,554 - root - INFO - lr: 2.7718e-04 gnorm: 0.77 [ 4:19:42<17:46:00] +[titan] 2025-07-09 17:54:43,463 - root - INFO - step: 19595 loss: 16.9118 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.2754 +[titan] 2025-07-09 17:54:43,464 - root - INFO - lr: 2.7717e-04 gnorm: 0.85 [ 4:19:46<17:45:56] +[titan] 2025-07-09 17:54:46,605 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:54:47,396 - root - INFO - step: 19600 loss: 17.1053 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6828 global_avg_mtp_loss: 14.4225 +[titan] 2025-07-09 17:54:47,396 - root - INFO - lr: 2.7716e-04 gnorm: 0.76 [ 4:19:50<17:45:51] +[titan] 2025-07-09 17:54:51,319 - root - INFO - step: 19605 loss: 16.9292 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.2866 +[titan] 2025-07-09 17:54:51,320 - root - INFO - lr: 2.7715e-04 gnorm: 0.80 [ 4:19:54<17:45:47] +[titan] 2025-07-09 17:54:55,263 - root - INFO - step: 19610 loss: 16.8393 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.2283 +[titan] 2025-07-09 17:54:55,263 - root - INFO - lr: 2.7713e-04 gnorm: 0.76 [ 4:19:58<17:45:43] +[titan] 2025-07-09 17:54:59,255 - root - INFO - step: 19615 loss: 17.1712 memory: 44.58GiB(31.99%) tps: 82,092 tflops: 283.31 mfu: 28.65% global_avg_ntp_loss: 2.6948 global_avg_mtp_loss: 14.4764 +[titan] 2025-07-09 17:54:59,255 - root - INFO - lr: 2.7712e-04 gnorm: 0.81 [ 4:20:02<17:45:39] +[titan] 2025-07-09 17:55:03,181 - root - INFO - step: 19620 loss: 17.0532 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.6579 global_avg_mtp_loss: 14.3953 +[titan] 2025-07-09 17:55:03,181 - root - INFO - lr: 2.7711e-04 gnorm: 0.78 [ 4:20:06<17:45:35] +[titan] 2025-07-09 17:55:07,103 - root - INFO - step: 19625 loss: 17.1253 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6650 global_avg_mtp_loss: 14.4603 +[titan] 2025-07-09 17:55:07,103 - root - INFO - lr: 2.7710e-04 gnorm: 0.76 [ 4:20:09<17:45:31] +[titan] 2025-07-09 17:55:11,042 - root - INFO - step: 19630 loss: 16.8090 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.6084 global_avg_mtp_loss: 14.2005 +[titan] 2025-07-09 17:55:11,042 - root - INFO - lr: 2.7709e-04 gnorm: 0.78 [ 4:20:13<17:45:27] +[titan] 2025-07-09 17:55:14,962 - root - INFO - step: 19635 loss: 17.0994 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6668 global_avg_mtp_loss: 14.4325 +[titan] 2025-07-09 17:55:14,962 - root - INFO - lr: 2.7707e-04 gnorm: 0.83 [ 4:20:17<17:45:22] +[titan] 2025-07-09 17:55:18,903 - root - INFO - step: 19640 loss: 17.0290 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.6610 global_avg_mtp_loss: 14.3679 +[titan] 2025-07-09 17:55:18,904 - root - INFO - lr: 2.7706e-04 gnorm: 0.80 [ 4:20:21<17:45:18] +[titan] 2025-07-09 17:55:22,804 - root - INFO - step: 19645 loss: 17.0787 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6646 global_avg_mtp_loss: 14.4141 +[titan] 2025-07-09 17:55:22,804 - root - INFO - lr: 2.7705e-04 gnorm: 0.77 [ 4:20:25<17:45:14] +[titan] 2025-07-09 17:55:25,920 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:55:26,705 - root - INFO - step: 19650 loss: 16.8142 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 17:55:26,705 - root - INFO - lr: 2.7704e-04 gnorm: 0.78 [ 4:20:29<17:45:10] +[titan] 2025-07-09 17:55:30,594 - root - INFO - step: 19655 loss: 17.0406 memory: 44.58GiB(31.99%) tps: 84,275 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.6560 global_avg_mtp_loss: 14.3846 +[titan] 2025-07-09 17:55:30,594 - root - INFO - lr: 2.7703e-04 gnorm: 0.87 [ 4:20:33<17:45:05] +[titan] 2025-07-09 17:55:34,522 - root - INFO - step: 19660 loss: 16.9818 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6536 global_avg_mtp_loss: 14.3282 +[titan] 2025-07-09 17:55:34,523 - root - INFO - lr: 2.7701e-04 gnorm: 0.71 [ 4:20:37<17:45:01] +[titan] 2025-07-09 17:55:38,441 - root - INFO - step: 19665 loss: 17.2609 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6953 global_avg_mtp_loss: 14.5656 +[titan] 2025-07-09 17:55:38,441 - root - INFO - lr: 2.7700e-04 gnorm: 0.79 [ 4:20:41<17:44:57] +[titan] 2025-07-09 17:55:42,392 - root - INFO - step: 19670 loss: 17.1723 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.4809 +[titan] 2025-07-09 17:55:42,392 - root - INFO - lr: 2.7699e-04 gnorm: 0.85 [ 4:20:45<17:44:53] +[titan] 2025-07-09 17:55:46,297 - root - INFO - step: 19675 loss: 17.0187 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6582 global_avg_mtp_loss: 14.3606 +[titan] 2025-07-09 17:55:46,297 - root - INFO - lr: 2.7698e-04 gnorm: 0.75 [ 4:20:49<17:44:49] +[titan] 2025-07-09 17:55:50,231 - root - INFO - step: 19680 loss: 17.2418 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6963 global_avg_mtp_loss: 14.5454 +[titan] 2025-07-09 17:55:50,231 - root - INFO - lr: 2.7697e-04 gnorm: 0.80 [ 4:20:53<17:44:44] +[titan] 2025-07-09 17:55:54,124 - root - INFO - step: 19685 loss: 16.6037 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0309 +[titan] 2025-07-09 17:55:54,125 - root - INFO - lr: 2.7695e-04 gnorm: 0.93 [ 4:20:56<17:44:40] +[titan] 2025-07-09 17:55:58,048 - root - INFO - step: 19690 loss: 16.7343 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1459 +[titan] 2025-07-09 17:55:58,048 - root - INFO - lr: 2.7694e-04 gnorm: 0.86 [ 4:21:00<17:44:36] +[titan] 2025-07-09 17:56:01,955 - root - INFO - step: 19695 loss: 16.9809 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3317 +[titan] 2025-07-09 17:56:01,956 - root - INFO - lr: 2.7693e-04 gnorm: 0.80 [ 4:21:04<17:44:32] +[titan] 2025-07-09 17:56:05,061 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:56:05,856 - root - INFO - step: 19700 loss: 17.0649 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6744 global_avg_mtp_loss: 14.3905 +[titan] 2025-07-09 17:56:05,857 - root - INFO - lr: 2.7692e-04 gnorm: 0.91 [ 4:21:08<17:44:27] +[titan] 2025-07-09 17:56:09,764 - root - INFO - step: 19705 loss: 16.7623 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.1619 +[titan] 2025-07-09 17:56:09,764 - root - INFO - lr: 2.7691e-04 gnorm: 0.80 [ 4:21:12<17:44:23] +[titan] 2025-07-09 17:56:13,670 - root - INFO - step: 19710 loss: 17.1072 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4316 +[titan] 2025-07-09 17:56:13,670 - root - INFO - lr: 2.7689e-04 gnorm: 0.81 [ 4:21:16<17:44:19] +[titan] 2025-07-09 17:56:17,579 - root - INFO - step: 19715 loss: 17.1524 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6824 global_avg_mtp_loss: 14.4700 +[titan] 2025-07-09 17:56:17,580 - root - INFO - lr: 2.7688e-04 gnorm: 0.80 [ 4:21:20<17:44:15] +[titan] 2025-07-09 17:56:21,492 - root - INFO - step: 19720 loss: 16.9822 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6548 global_avg_mtp_loss: 14.3274 +[titan] 2025-07-09 17:56:21,492 - root - INFO - lr: 2.7687e-04 gnorm: 0.73 [ 4:21:24<17:44:10] +[titan] 2025-07-09 17:56:25,412 - root - INFO - step: 19725 loss: 16.8135 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6219 global_avg_mtp_loss: 14.1916 +[titan] 2025-07-09 17:56:25,413 - root - INFO - lr: 2.7686e-04 gnorm: 0.74 [ 4:21:28<17:44:06] +[titan] 2025-07-09 17:56:29,322 - root - INFO - step: 19730 loss: 16.9345 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.2948 +[titan] 2025-07-09 17:56:29,322 - root - INFO - lr: 2.7685e-04 gnorm: 0.81 [ 4:21:32<17:44:02] +[titan] 2025-07-09 17:56:33,216 - root - INFO - step: 19735 loss: 16.9767 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.3294 +[titan] 2025-07-09 17:56:33,216 - root - INFO - lr: 2.7683e-04 gnorm: 0.96 [ 4:21:36<17:43:58] +[titan] 2025-07-09 17:56:37,109 - root - INFO - step: 19740 loss: 17.0036 memory: 44.58GiB(31.99%) tps: 84,181 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 2.6555 global_avg_mtp_loss: 14.3481 +[titan] 2025-07-09 17:56:37,109 - root - INFO - lr: 2.7682e-04 gnorm: 0.83 [ 4:21:39<17:43:53] +[titan] 2025-07-09 17:56:41,065 - root - INFO - step: 19745 loss: 16.8201 memory: 44.58GiB(31.99%) tps: 82,833 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.6366 global_avg_mtp_loss: 14.1835 +[titan] 2025-07-09 17:56:41,065 - root - INFO - lr: 2.7681e-04 gnorm: 0.81 [ 4:21:43<17:43:49] +[titan] 2025-07-09 17:56:44,185 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:56:44,974 - root - INFO - step: 19750 loss: 17.1935 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6729 global_avg_mtp_loss: 14.5206 +[titan] 2025-07-09 17:56:44,974 - root - INFO - lr: 2.7680e-04 gnorm: 0.90 [ 4:21:47<17:43:45] +[titan] 2025-07-09 17:56:48,877 - root - INFO - step: 19755 loss: 16.8843 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6368 global_avg_mtp_loss: 14.2475 +[titan] 2025-07-09 17:56:48,878 - root - INFO - lr: 2.7679e-04 gnorm: 0.84 [ 4:21:51<17:43:41] +[titan] 2025-07-09 17:56:52,811 - root - INFO - step: 19760 loss: 17.1830 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6733 global_avg_mtp_loss: 14.5097 +[titan] 2025-07-09 17:56:52,812 - root - INFO - lr: 2.7677e-04 gnorm: 0.77 [ 4:21:55<17:43:37] +[titan] 2025-07-09 17:56:56,724 - root - INFO - step: 19765 loss: 17.0619 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6695 global_avg_mtp_loss: 14.3924 +[titan] 2025-07-09 17:56:56,724 - root - INFO - lr: 2.7676e-04 gnorm: 0.77 [ 4:21:59<17:43:32] +[titan] 2025-07-09 17:57:00,643 - root - INFO - step: 19770 loss: 17.3556 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.7520 global_avg_mtp_loss: 14.6035 +[titan] 2025-07-09 17:57:00,643 - root - INFO - lr: 2.7675e-04 gnorm: 0.83 [ 4:22:03<17:43:28] +[titan] 2025-07-09 17:57:04,574 - root - INFO - step: 19775 loss: 16.7719 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.1381 +[titan] 2025-07-09 17:57:04,574 - root - INFO - lr: 2.7674e-04 gnorm: 0.80 [ 4:22:07<17:43:24] +[titan] 2025-07-09 17:57:08,524 - root - INFO - step: 19780 loss: 17.1748 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.6812 global_avg_mtp_loss: 14.4936 +[titan] 2025-07-09 17:57:08,524 - root - INFO - lr: 2.7673e-04 gnorm: 0.79 [ 4:22:11<17:43:20] +[titan] 2025-07-09 17:57:12,445 - root - INFO - step: 19785 loss: 16.9123 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6294 global_avg_mtp_loss: 14.2830 +[titan] 2025-07-09 17:57:12,446 - root - INFO - lr: 2.7671e-04 gnorm: 0.81 [ 4:22:15<17:43:16] +[titan] 2025-07-09 17:57:16,388 - root - INFO - step: 19790 loss: 17.0056 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.6609 global_avg_mtp_loss: 14.3447 +[titan] 2025-07-09 17:57:16,388 - root - INFO - lr: 2.7670e-04 gnorm: 0.78 [ 4:22:19<17:43:12] +[titan] 2025-07-09 17:57:20,296 - root - INFO - step: 19795 loss: 16.9028 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6349 global_avg_mtp_loss: 14.2678 +[titan] 2025-07-09 17:57:20,296 - root - INFO - lr: 2.7669e-04 gnorm: 0.80 [ 4:22:23<17:43:07] +[titan] 2025-07-09 17:57:23,400 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:57:24,187 - root - INFO - step: 19800 loss: 17.0138 memory: 44.58GiB(31.99%) tps: 84,218 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6474 global_avg_mtp_loss: 14.3664 +[titan] 2025-07-09 17:57:24,187 - root - INFO - lr: 2.7668e-04 gnorm: 0.86 [ 4:22:27<17:43:03] +[titan] 2025-07-09 17:57:28,130 - root - INFO - step: 19805 loss: 16.9310 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.2904 +[titan] 2025-07-09 17:57:28,130 - root - INFO - lr: 2.7667e-04 gnorm: 0.75 [ 4:22:30<17:42:59] +[titan] 2025-07-09 17:57:32,056 - root - INFO - step: 19810 loss: 16.8203 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.1925 +[titan] 2025-07-09 17:57:32,056 - root - INFO - lr: 2.7665e-04 gnorm: 0.82 [ 4:22:34<17:42:55] +[titan] 2025-07-09 17:57:35,999 - root - INFO - step: 19815 loss: 17.1492 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.6760 global_avg_mtp_loss: 14.4732 +[titan] 2025-07-09 17:57:36,000 - root - INFO - lr: 2.7664e-04 gnorm: 0.81 [ 4:22:38<17:42:51] +[titan] 2025-07-09 17:57:39,970 - root - INFO - step: 19820 loss: 16.9488 memory: 44.58GiB(31.99%) tps: 82,528 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.6408 global_avg_mtp_loss: 14.3080 +[titan] 2025-07-09 17:57:39,970 - root - INFO - lr: 2.7663e-04 gnorm: 0.80 [ 4:22:42<17:42:47] +[titan] 2025-07-09 17:57:43,934 - root - INFO - step: 19825 loss: 16.8503 memory: 44.58GiB(31.99%) tps: 82,673 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.6221 global_avg_mtp_loss: 14.2283 +[titan] 2025-07-09 17:57:43,934 - root - INFO - lr: 2.7662e-04 gnorm: 0.78 [ 4:22:46<17:42:43] +[titan] 2025-07-09 17:57:47,854 - root - INFO - step: 19830 loss: 16.8855 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.2658 +[titan] 2025-07-09 17:57:47,855 - root - INFO - lr: 2.7661e-04 gnorm: 0.82 [ 4:22:50<17:42:38] +[titan] 2025-07-09 17:57:51,775 - root - INFO - step: 19835 loss: 16.9385 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.3175 +[titan] 2025-07-09 17:57:51,775 - root - INFO - lr: 2.7659e-04 gnorm: 0.79 [ 4:22:54<17:42:34] +[titan] 2025-07-09 17:57:55,702 - root - INFO - step: 19840 loss: 17.0090 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6486 global_avg_mtp_loss: 14.3603 +[titan] 2025-07-09 17:57:55,702 - root - INFO - lr: 2.7658e-04 gnorm: 0.78 [ 4:22:58<17:42:30] +[titan] 2025-07-09 17:57:59,668 - root - INFO - step: 19845 loss: 16.9691 memory: 44.58GiB(31.99%) tps: 82,637 tflops: 285.19 mfu: 28.84% global_avg_ntp_loss: 2.6532 global_avg_mtp_loss: 14.3160 +[titan] 2025-07-09 17:57:59,668 - root - INFO - lr: 2.7657e-04 gnorm: 0.84 [ 4:23:02<17:42:26] +[titan] 2025-07-09 17:58:02,834 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:58:03,630 - root - INFO - step: 19850 loss: 16.8595 memory: 44.58GiB(31.99%) tps: 82,717 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.6198 global_avg_mtp_loss: 14.2397 +[titan] 2025-07-09 17:58:03,630 - root - INFO - lr: 2.7656e-04 gnorm: 0.85 [ 4:23:06<17:42:22] +[titan] 2025-07-09 17:58:07,546 - root - INFO - step: 19855 loss: 16.9955 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6438 global_avg_mtp_loss: 14.3517 +[titan] 2025-07-09 17:58:07,546 - root - INFO - lr: 2.7655e-04 gnorm: 0.85 [ 4:23:10<17:42:18] +[titan] 2025-07-09 17:58:11,477 - root - INFO - step: 19860 loss: 16.6129 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5958 global_avg_mtp_loss: 14.0171 +[titan] 2025-07-09 17:58:11,477 - root - INFO - lr: 2.7653e-04 gnorm: 0.85 [ 4:23:14<17:42:13] +[titan] 2025-07-09 17:58:15,393 - root - INFO - step: 19865 loss: 17.1274 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6752 global_avg_mtp_loss: 14.4522 +[titan] 2025-07-09 17:58:15,394 - root - INFO - lr: 2.7652e-04 gnorm: 0.84 [ 4:23:18<17:42:09] +[titan] 2025-07-09 17:58:19,299 - root - INFO - step: 19870 loss: 17.0205 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6454 global_avg_mtp_loss: 14.3751 +[titan] 2025-07-09 17:58:19,299 - root - INFO - lr: 2.7651e-04 gnorm: 0.79 [ 4:23:22<17:42:05] +[titan] 2025-07-09 17:58:23,266 - root - INFO - step: 19875 loss: 16.7029 memory: 44.58GiB(31.99%) tps: 82,604 tflops: 285.08 mfu: 28.83% global_avg_ntp_loss: 2.6037 global_avg_mtp_loss: 14.0993 +[titan] 2025-07-09 17:58:23,266 - root - INFO - lr: 2.7650e-04 gnorm: 0.71 [ 4:23:26<17:42:01] +[titan] 2025-07-09 17:58:27,189 - root - INFO - step: 19880 loss: 16.8652 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2374 +[titan] 2025-07-09 17:58:27,189 - root - INFO - lr: 2.7649e-04 gnorm: 0.78 [ 4:23:30<17:41:57] +[titan] 2025-07-09 17:58:31,101 - root - INFO - step: 19885 loss: 16.9156 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6302 global_avg_mtp_loss: 14.2854 +[titan] 2025-07-09 17:58:31,102 - root - INFO - lr: 2.7647e-04 gnorm: 0.89 [ 4:23:33<17:41:53] +[titan] 2025-07-09 17:58:35,013 - root - INFO - step: 19890 loss: 17.0685 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6837 global_avg_mtp_loss: 14.3847 +[titan] 2025-07-09 17:58:35,014 - root - INFO - lr: 2.7646e-04 gnorm: 0.89 [ 4:23:37<17:41:48] +[titan] 2025-07-09 17:58:38,933 - root - INFO - step: 19895 loss: 16.8173 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6142 global_avg_mtp_loss: 14.2032 +[titan] 2025-07-09 17:58:38,934 - root - INFO - lr: 2.7645e-04 gnorm: 0.81 [ 4:23:41<17:41:44] +[titan] 2025-07-09 17:58:42,089 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:58:42,878 - root - INFO - step: 19900 loss: 17.0701 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.4002 +[titan] 2025-07-09 17:58:42,878 - root - INFO - lr: 2.7644e-04 gnorm: 0.84 [ 4:23:45<17:41:40] +[titan] 2025-07-09 17:58:46,785 - root - INFO - step: 19905 loss: 17.0319 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6476 global_avg_mtp_loss: 14.3843 +[titan] 2025-07-09 17:58:46,785 - root - INFO - lr: 2.7642e-04 gnorm: 0.82 [ 4:23:49<17:41:36] +[titan] 2025-07-09 17:58:50,709 - root - INFO - step: 19910 loss: 17.0101 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3682 +[titan] 2025-07-09 17:58:50,710 - root - INFO - lr: 2.7641e-04 gnorm: 0.82 [ 4:23:53<17:41:32] +[titan] 2025-07-09 17:58:54,638 - root - INFO - step: 19915 loss: 16.7382 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5988 global_avg_mtp_loss: 14.1394 +[titan] 2025-07-09 17:58:54,638 - root - INFO - lr: 2.7640e-04 gnorm: 0.85 [ 4:23:57<17:41:27] +[titan] 2025-07-09 17:58:58,534 - root - INFO - step: 19920 loss: 16.8739 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6135 global_avg_mtp_loss: 14.2604 +[titan] 2025-07-09 17:58:58,534 - root - INFO - lr: 2.7639e-04 gnorm: 0.90 [ 4:24:01<17:41:23] +[titan] 2025-07-09 17:59:02,465 - root - INFO - step: 19925 loss: 16.8343 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2194 +[titan] 2025-07-09 17:59:02,466 - root - INFO - lr: 2.7638e-04 gnorm: 0.86 [ 4:24:05<17:41:19] +[titan] 2025-07-09 17:59:06,386 - root - INFO - step: 19930 loss: 16.8490 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6287 global_avg_mtp_loss: 14.2202 +[titan] 2025-07-09 17:59:06,386 - root - INFO - lr: 2.7636e-04 gnorm: 0.86 [ 4:24:09<17:41:15] +[titan] 2025-07-09 17:59:10,300 - root - INFO - step: 19935 loss: 16.9906 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6443 global_avg_mtp_loss: 14.3463 +[titan] 2025-07-09 17:59:10,301 - root - INFO - lr: 2.7635e-04 gnorm: 0.85 [ 4:24:13<17:41:11] +[titan] 2025-07-09 17:59:14,214 - root - INFO - step: 19940 loss: 16.9824 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6452 global_avg_mtp_loss: 14.3372 +[titan] 2025-07-09 17:59:14,214 - root - INFO - lr: 2.7634e-04 gnorm: 0.85 [ 4:24:17<17:41:06] +[titan] 2025-07-09 17:59:18,114 - root - INFO - step: 19945 loss: 16.7483 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.1501 +[titan] 2025-07-09 17:59:18,114 - root - INFO - lr: 2.7633e-04 gnorm: 0.73 [ 4:24:20<17:41:02] +[titan] 2025-07-09 17:59:21,242 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 17:59:22,036 - root - INFO - step: 19950 loss: 17.1592 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6890 global_avg_mtp_loss: 14.4702 +[titan] 2025-07-09 17:59:22,036 - root - INFO - lr: 2.7632e-04 gnorm: 0.91 [ 4:24:24<17:40:58] +[titan] 2025-07-09 17:59:25,969 - root - INFO - step: 19955 loss: 17.1487 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6794 global_avg_mtp_loss: 14.4693 +[titan] 2025-07-09 17:59:25,970 - root - INFO - lr: 2.7630e-04 gnorm: 0.90 [ 4:24:28<17:40:54] +[titan] 2025-07-09 17:59:29,896 - root - INFO - step: 19960 loss: 16.8701 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.6369 global_avg_mtp_loss: 14.2332 +[titan] 2025-07-09 17:59:29,896 - root - INFO - lr: 2.7629e-04 gnorm: 0.79 [ 4:24:32<17:40:49] +[titan] 2025-07-09 17:59:33,890 - root - INFO - step: 19965 loss: 17.0022 memory: 44.58GiB(31.99%) tps: 82,060 tflops: 283.20 mfu: 28.64% global_avg_ntp_loss: 2.6539 global_avg_mtp_loss: 14.3482 +[titan] 2025-07-09 17:59:33,890 - root - INFO - lr: 2.7628e-04 gnorm: 0.75 [ 4:24:36<17:40:46] +[titan] 2025-07-09 17:59:36,407 - root - INFO - Dumping profiler traces at step 19968 +[titan] 2025-07-09 17:59:36,438 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 17:59:38,016 - root - INFO - step: 19970 loss: 17.1860 memory: 44.58GiB(31.99%) tps: 79,419 tflops: 274.09 mfu: 27.71% global_avg_ntp_loss: 2.7125 global_avg_mtp_loss: 14.4735 +[titan] 2025-07-09 17:59:38,016 - root - INFO - lr: 2.7627e-04 gnorm: 0.81 [ 4:24:40<17:40:42] +[titan] 2025-07-09 17:59:41,920 - root - INFO - step: 19975 loss: 17.1799 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6984 global_avg_mtp_loss: 14.4815 +[titan] 2025-07-09 17:59:41,920 - root - INFO - lr: 2.7626e-04 gnorm: 0.78 [ 4:24:44<17:40:38] +[titan] 2025-07-09 17:59:45,824 - root - INFO - step: 19980 loss: 17.0417 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6587 global_avg_mtp_loss: 14.3830 +[titan] 2025-07-09 17:59:45,824 - root - INFO - lr: 2.7624e-04 gnorm: 0.82 [ 4:24:48<17:40:34] +[titan] 2025-07-09 17:59:49,745 - root - INFO - step: 19985 loss: 16.9509 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.6382 global_avg_mtp_loss: 14.3127 +[titan] 2025-07-09 17:59:49,745 - root - INFO - lr: 2.7623e-04 gnorm: 0.86 [ 4:24:52<17:40:29] +[titan] 2025-07-09 17:59:53,647 - root - INFO - step: 19990 loss: 16.9138 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.2800 +[titan] 2025-07-09 17:59:53,647 - root - INFO - lr: 2.7622e-04 gnorm: 0.80 [ 4:24:56<17:40:25] +[titan] 2025-07-09 17:59:57,555 - root - INFO - step: 19995 loss: 17.2363 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6936 global_avg_mtp_loss: 14.5427 +[titan] 2025-07-09 17:59:57,556 - root - INFO - lr: 2.7621e-04 gnorm: 0.87 [ 4:25:00<17:40:21] +[titan] 2025-07-09 18:00:00,666 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:00:01,460 - root - INFO - step: 20000 loss: 16.9875 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.6536 global_avg_mtp_loss: 14.3339 +[titan] 2025-07-09 18:00:01,461 - root - INFO - lr: 2.7619e-04 gnorm: 0.84 [ 4:25:04<17:40:17] +[titan] 2025-07-09 18:00:01,461 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-09 18:00:03,711 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-09 18:00:03,711 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.25 seconds. +[titan] 2025-07-09 18:01:02,511 - root - INFO - step: 20005 loss: 16.8314 memory: 44.58GiB(31.99%) tps: 5,367 tflops: 18.52 mfu: 1.87% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2092 +[titan] 2025-07-09 18:01:02,511 - root - INFO - lr: 2.7618e-04 gnorm: 0.81 [ 4:26:05<17:44:01] +[titan] 2025-07-09 18:01:06,450 - root - INFO - step: 20010 loss: 16.7648 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.1577 +[titan] 2025-07-09 18:01:06,450 - root - INFO - lr: 2.7617e-04 gnorm: 0.74 [ 4:26:09<17:43:57] +[titan] 2025-07-09 18:01:10,363 - root - INFO - step: 20015 loss: 16.9016 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.2697 +[titan] 2025-07-09 18:01:10,364 - root - INFO - lr: 2.7616e-04 gnorm: 0.80 [ 4:26:13<17:43:52] +[titan] 2025-07-09 18:01:14,326 - root - INFO - step: 20020 loss: 17.0164 memory: 44.58GiB(31.99%) tps: 82,697 tflops: 285.40 mfu: 28.86% global_avg_ntp_loss: 2.6487 global_avg_mtp_loss: 14.3677 +[titan] 2025-07-09 18:01:14,327 - root - INFO - lr: 2.7615e-04 gnorm: 0.85 [ 4:26:17<17:43:48] +[titan] 2025-07-09 18:01:18,280 - root - INFO - step: 20025 loss: 17.0748 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.6641 global_avg_mtp_loss: 14.4107 +[titan] 2025-07-09 18:01:18,280 - root - INFO - lr: 2.7613e-04 gnorm: 0.78 [ 4:26:21<17:43:44] +[titan] 2025-07-09 18:01:22,249 - root - INFO - step: 20030 loss: 16.7637 memory: 44.58GiB(31.99%) tps: 82,563 tflops: 284.94 mfu: 28.81% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.1615 +[titan] 2025-07-09 18:01:22,250 - root - INFO - lr: 2.7612e-04 gnorm: 0.77 [ 4:26:25<17:43:40] +[titan] 2025-07-09 18:01:26,192 - root - INFO - step: 20035 loss: 17.1922 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.6912 global_avg_mtp_loss: 14.5010 +[titan] 2025-07-09 18:01:26,192 - root - INFO - lr: 2.7611e-04 gnorm: 0.82 [ 4:26:29<17:43:36] +[titan] 2025-07-09 18:01:30,092 - root - INFO - step: 20040 loss: 16.8558 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2323 +[titan] 2025-07-09 18:01:30,092 - root - INFO - lr: 2.7610e-04 gnorm: 0.79 [ 4:26:32<17:43:32] +[titan] 2025-07-09 18:01:34,005 - root - INFO - step: 20045 loss: 17.0982 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6725 global_avg_mtp_loss: 14.4256 +[titan] 2025-07-09 18:01:34,005 - root - INFO - lr: 2.7608e-04 gnorm: 0.75 [ 4:26:36<17:43:27] +[titan] 2025-07-09 18:01:37,150 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:01:37,952 - root - INFO - step: 20050 loss: 17.1159 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.6730 global_avg_mtp_loss: 14.4429 +[titan] 2025-07-09 18:01:37,953 - root - INFO - lr: 2.7607e-04 gnorm: 0.78 [ 4:26:40<17:43:23] +[titan] 2025-07-09 18:01:41,985 - root - INFO - step: 20055 loss: 16.9757 memory: 44.58GiB(31.99%) tps: 81,256 tflops: 280.43 mfu: 28.35% global_avg_ntp_loss: 2.6482 global_avg_mtp_loss: 14.3275 +[titan] 2025-07-09 18:01:41,986 - root - INFO - lr: 2.7606e-04 gnorm: 0.77 [ 4:26:44<17:43:19] +[titan] 2025-07-09 18:01:45,909 - root - INFO - step: 20060 loss: 16.8315 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6230 global_avg_mtp_loss: 14.2085 +[titan] 2025-07-09 18:01:45,909 - root - INFO - lr: 2.7605e-04 gnorm: 0.74 [ 4:26:48<17:43:15] +[titan] 2025-07-09 18:01:49,834 - root - INFO - step: 20065 loss: 17.1634 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6876 global_avg_mtp_loss: 14.4758 +[titan] 2025-07-09 18:01:49,834 - root - INFO - lr: 2.7604e-04 gnorm: 0.76 [ 4:26:52<17:43:11] +[titan] 2025-07-09 18:01:53,823 - root - INFO - step: 20070 loss: 17.5865 memory: 44.58GiB(31.99%) tps: 82,135 tflops: 283.46 mfu: 28.66% global_avg_ntp_loss: 2.7772 global_avg_mtp_loss: 14.8092 +[titan] 2025-07-09 18:01:53,824 - root - INFO - lr: 2.7602e-04 gnorm: 0.86 [ 4:26:56<17:43:07] +[titan] 2025-07-09 18:01:57,808 - root - INFO - step: 20075 loss: 17.0942 memory: 44.58GiB(31.99%) tps: 82,241 tflops: 283.83 mfu: 28.70% global_avg_ntp_loss: 2.6498 global_avg_mtp_loss: 14.4444 +[titan] 2025-07-09 18:01:57,808 - root - INFO - lr: 2.7601e-04 gnorm: 0.82 [ 4:27:00<17:43:03] +[titan] 2025-07-09 18:02:01,709 - root - INFO - step: 20080 loss: 16.9988 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6509 global_avg_mtp_loss: 14.3479 +[titan] 2025-07-09 18:02:01,709 - root - INFO - lr: 2.7600e-04 gnorm: 0.83 [ 4:27:04<17:42:58] +[titan] 2025-07-09 18:02:05,680 - root - INFO - step: 20085 loss: 17.1672 memory: 44.58GiB(31.99%) tps: 82,519 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.6978 global_avg_mtp_loss: 14.4694 +[titan] 2025-07-09 18:02:05,681 - root - INFO - lr: 2.7599e-04 gnorm: 0.81 [ 4:27:08<17:42:54] +[titan] 2025-07-09 18:02:09,613 - root - INFO - step: 20090 loss: 16.9465 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6246 global_avg_mtp_loss: 14.3219 +[titan] 2025-07-09 18:02:09,613 - root - INFO - lr: 2.7598e-04 gnorm: 0.74 [ 4:27:12<17:42:50] +[titan] 2025-07-09 18:02:13,553 - root - INFO - step: 20095 loss: 17.1349 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6762 global_avg_mtp_loss: 14.4587 +[titan] 2025-07-09 18:02:13,553 - root - INFO - lr: 2.7596e-04 gnorm: 0.86 [ 4:27:16<17:42:46] +[titan] 2025-07-09 18:02:16,711 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:02:17,506 - root - INFO - step: 20100 loss: 17.0700 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.6591 global_avg_mtp_loss: 14.4109 +[titan] 2025-07-09 18:02:17,506 - root - INFO - lr: 2.7595e-04 gnorm: 0.84 [ 4:27:20<17:42:42] +[titan] 2025-07-09 18:02:21,438 - root - INFO - step: 20105 loss: 16.8117 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.6097 global_avg_mtp_loss: 14.2021 +[titan] 2025-07-09 18:02:21,438 - root - INFO - lr: 2.7594e-04 gnorm: 0.79 [ 4:27:24<17:42:38] +[titan] 2025-07-09 18:02:25,346 - root - INFO - step: 20110 loss: 16.8328 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2180 +[titan] 2025-07-09 18:02:25,346 - root - INFO - lr: 2.7593e-04 gnorm: 0.81 [ 4:27:28<17:42:33] +[titan] 2025-07-09 18:02:29,307 - root - INFO - step: 20115 loss: 17.1235 memory: 44.58GiB(31.99%) tps: 82,736 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.6759 global_avg_mtp_loss: 14.4476 +[titan] 2025-07-09 18:02:29,307 - root - INFO - lr: 2.7591e-04 gnorm: 0.83 [ 4:27:32<17:42:29] +[titan] 2025-07-09 18:02:33,216 - root - INFO - step: 20120 loss: 16.8334 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.6064 global_avg_mtp_loss: 14.2270 +[titan] 2025-07-09 18:02:33,216 - root - INFO - lr: 2.7590e-04 gnorm: 0.84 [ 4:27:36<17:42:25] +[titan] 2025-07-09 18:02:37,178 - root - INFO - step: 20125 loss: 17.0026 memory: 44.58GiB(31.99%) tps: 82,699 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3535 +[titan] 2025-07-09 18:02:37,179 - root - INFO - lr: 2.7589e-04 gnorm: 0.86 [ 4:27:39<17:42:21] +[titan] 2025-07-09 18:02:41,107 - root - INFO - step: 20130 loss: 16.7775 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6055 global_avg_mtp_loss: 14.1720 +[titan] 2025-07-09 18:02:41,108 - root - INFO - lr: 2.7588e-04 gnorm: 0.82 [ 4:27:43<17:42:17] +[titan] 2025-07-09 18:02:45,029 - root - INFO - step: 20135 loss: 17.1575 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6783 global_avg_mtp_loss: 14.4792 +[titan] 2025-07-09 18:02:45,029 - root - INFO - lr: 2.7587e-04 gnorm: 0.84 [ 4:27:47<17:42:12] +[titan] 2025-07-09 18:02:48,935 - root - INFO - step: 20140 loss: 16.9848 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.3466 +[titan] 2025-07-09 18:02:48,935 - root - INFO - lr: 2.7585e-04 gnorm: 0.82 [ 4:27:51<17:42:08] +[titan] 2025-07-09 18:02:52,824 - root - INFO - step: 20145 loss: 16.9802 memory: 44.58GiB(31.99%) tps: 84,258 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.6444 global_avg_mtp_loss: 14.3358 +[titan] 2025-07-09 18:02:52,824 - root - INFO - lr: 2.7584e-04 gnorm: 0.78 [ 4:27:55<17:42:04] +[titan] 2025-07-09 18:02:55,961 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:02:56,751 - root - INFO - step: 20150 loss: 16.7597 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6019 global_avg_mtp_loss: 14.1579 +[titan] 2025-07-09 18:02:56,752 - root - INFO - lr: 2.7583e-04 gnorm: 0.85 [ 4:27:59<17:41:59] +[titan] 2025-07-09 18:03:00,677 - root - INFO - step: 20155 loss: 17.1195 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6754 global_avg_mtp_loss: 14.4441 +[titan] 2025-07-09 18:03:00,677 - root - INFO - lr: 2.7582e-04 gnorm: 0.82 [ 4:28:03<17:41:55] +[titan] 2025-07-09 18:03:04,588 - root - INFO - step: 20160 loss: 17.0908 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6630 global_avg_mtp_loss: 14.4278 +[titan] 2025-07-09 18:03:04,588 - root - INFO - lr: 2.7580e-04 gnorm: 0.87 [ 4:28:07<17:41:51] +[titan] 2025-07-09 18:03:08,490 - root - INFO - step: 20165 loss: 17.1293 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6587 global_avg_mtp_loss: 14.4706 +[titan] 2025-07-09 18:03:08,490 - root - INFO - lr: 2.7579e-04 gnorm: 0.84 [ 4:28:11<17:41:46] +[titan] 2025-07-09 18:03:12,393 - root - INFO - step: 20170 loss: 17.0206 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6551 global_avg_mtp_loss: 14.3655 +[titan] 2025-07-09 18:03:12,393 - root - INFO - lr: 2.7578e-04 gnorm: 0.87 [ 4:28:15<17:41:42] +[titan] 2025-07-09 18:03:16,285 - root - INFO - step: 20175 loss: 16.9251 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.2925 +[titan] 2025-07-09 18:03:16,286 - root - INFO - lr: 2.7577e-04 gnorm: 0.80 [ 4:28:19<17:41:38] +[titan] 2025-07-09 18:03:20,203 - root - INFO - step: 20180 loss: 16.8755 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.2353 +[titan] 2025-07-09 18:03:20,203 - root - INFO - lr: 2.7576e-04 gnorm: 0.75 [ 4:28:23<17:41:33] +[titan] 2025-07-09 18:03:24,130 - root - INFO - step: 20185 loss: 16.7889 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.1730 +[titan] 2025-07-09 18:03:24,130 - root - INFO - lr: 2.7574e-04 gnorm: 0.80 [ 4:28:26<17:41:29] +[titan] 2025-07-09 18:03:28,060 - root - INFO - step: 20190 loss: 17.0212 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6546 global_avg_mtp_loss: 14.3666 +[titan] 2025-07-09 18:03:28,060 - root - INFO - lr: 2.7573e-04 gnorm: 0.78 [ 4:28:30<17:41:25] +[titan] 2025-07-09 18:03:31,961 - root - INFO - step: 20195 loss: 16.6761 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-09 18:03:31,961 - root - INFO - lr: 2.7572e-04 gnorm: 0.82 [ 4:28:34<17:41:21] +[titan] 2025-07-09 18:03:35,073 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:03:35,866 - root - INFO - step: 20200 loss: 16.9197 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6263 global_avg_mtp_loss: 14.2934 +[titan] 2025-07-09 18:03:35,866 - root - INFO - lr: 2.7571e-04 gnorm: 0.77 [ 4:28:38<17:41:16] +[titan] 2025-07-09 18:03:39,756 - root - INFO - step: 20205 loss: 16.8692 memory: 44.58GiB(31.99%) tps: 84,255 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2542 +[titan] 2025-07-09 18:03:39,756 - root - INFO - lr: 2.7569e-04 gnorm: 0.77 [ 4:28:42<17:41:12] +[titan] 2025-07-09 18:03:43,650 - root - INFO - step: 20210 loss: 16.6953 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5954 global_avg_mtp_loss: 14.1000 +[titan] 2025-07-09 18:03:43,651 - root - INFO - lr: 2.7568e-04 gnorm: 0.82 [ 4:28:46<17:41:08] +[titan] 2025-07-09 18:03:47,569 - root - INFO - step: 20215 loss: 17.2188 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6890 global_avg_mtp_loss: 14.5299 +[titan] 2025-07-09 18:03:47,570 - root - INFO - lr: 2.7567e-04 gnorm: 0.87 [ 4:28:50<17:41:03] +[titan] 2025-07-09 18:03:51,470 - root - INFO - step: 20220 loss: 16.8770 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6281 global_avg_mtp_loss: 14.2489 +[titan] 2025-07-09 18:03:51,470 - root - INFO - lr: 2.7566e-04 gnorm: 0.81 [ 4:28:54<17:40:59] +[titan] 2025-07-09 18:03:55,379 - root - INFO - step: 20225 loss: 17.2075 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6816 global_avg_mtp_loss: 14.5259 +[titan] 2025-07-09 18:03:55,380 - root - INFO - lr: 2.7564e-04 gnorm: 0.76 [ 4:28:58<17:40:55] +[titan] 2025-07-09 18:03:59,292 - root - INFO - step: 20230 loss: 17.2797 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.7040 global_avg_mtp_loss: 14.5757 +[titan] 2025-07-09 18:03:59,293 - root - INFO - lr: 2.7563e-04 gnorm: 0.85 [ 4:29:02<17:40:50] +[titan] 2025-07-09 18:04:03,214 - root - INFO - step: 20235 loss: 17.1673 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6745 global_avg_mtp_loss: 14.4928 +[titan] 2025-07-09 18:04:03,214 - root - INFO - lr: 2.7562e-04 gnorm: 0.86 [ 4:29:06<17:40:46] +[titan] 2025-07-09 18:04:07,120 - root - INFO - step: 20240 loss: 16.8630 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2310 +[titan] 2025-07-09 18:04:07,120 - root - INFO - lr: 2.7561e-04 gnorm: 0.79 [ 4:29:09<17:40:42] +[titan] 2025-07-09 18:04:11,053 - root - INFO - step: 20245 loss: 16.8943 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.2546 +[titan] 2025-07-09 18:04:11,053 - root - INFO - lr: 2.7560e-04 gnorm: 0.84 [ 4:29:13<17:40:38] +[titan] 2025-07-09 18:04:14,169 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:04:14,964 - root - INFO - step: 20250 loss: 16.6300 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5803 global_avg_mtp_loss: 14.0497 +[titan] 2025-07-09 18:04:14,964 - root - INFO - lr: 2.7558e-04 gnorm: 0.84 [ 4:29:17<17:40:33] +[titan] 2025-07-09 18:04:18,878 - root - INFO - step: 20255 loss: 16.9794 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.3301 +[titan] 2025-07-09 18:04:18,879 - root - INFO - lr: 2.7557e-04 gnorm: 0.81 [ 4:29:21<17:40:29] +[titan] 2025-07-09 18:04:22,777 - root - INFO - step: 20260 loss: 16.8038 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6228 global_avg_mtp_loss: 14.1811 +[titan] 2025-07-09 18:04:22,777 - root - INFO - lr: 2.7556e-04 gnorm: 0.76 [ 4:29:25<17:40:25] +[titan] 2025-07-09 18:04:26,679 - root - INFO - step: 20265 loss: 17.0027 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6551 global_avg_mtp_loss: 14.3476 +[titan] 2025-07-09 18:04:26,679 - root - INFO - lr: 2.7555e-04 gnorm: 0.80 [ 4:29:29<17:40:20] +[titan] 2025-07-09 18:04:30,601 - root - INFO - step: 20270 loss: 17.0422 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6591 global_avg_mtp_loss: 14.3831 +[titan] 2025-07-09 18:04:30,601 - root - INFO - lr: 2.7553e-04 gnorm: 0.75 [ 4:29:33<17:40:16] +[titan] 2025-07-09 18:04:34,520 - root - INFO - step: 20275 loss: 16.8931 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2743 +[titan] 2025-07-09 18:04:34,521 - root - INFO - lr: 2.7552e-04 gnorm: 0.79 [ 4:29:37<17:40:12] +[titan] 2025-07-09 18:04:38,418 - root - INFO - step: 20280 loss: 16.8784 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6239 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 18:04:38,419 - root - INFO - lr: 2.7551e-04 gnorm: 0.75 [ 4:29:41<17:40:07] +[titan] 2025-07-09 18:04:42,316 - root - INFO - step: 20285 loss: 16.9287 memory: 44.58GiB(31.99%) tps: 84,084 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6526 global_avg_mtp_loss: 14.2762 +[titan] 2025-07-09 18:04:42,316 - root - INFO - lr: 2.7550e-04 gnorm: 0.77 [ 4:29:45<17:40:03] +[titan] 2025-07-09 18:04:46,234 - root - INFO - step: 20290 loss: 16.8763 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6359 global_avg_mtp_loss: 14.2405 +[titan] 2025-07-09 18:04:46,235 - root - INFO - lr: 2.7549e-04 gnorm: 0.75 [ 4:29:49<17:39:59] +[titan] 2025-07-09 18:04:50,139 - root - INFO - step: 20295 loss: 17.0497 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6501 global_avg_mtp_loss: 14.3996 +[titan] 2025-07-09 18:04:50,139 - root - INFO - lr: 2.7547e-04 gnorm: 0.76 [ 4:29:52<17:39:54] +[titan] 2025-07-09 18:04:53,272 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:04:54,057 - root - INFO - step: 20300 loss: 16.9378 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6365 global_avg_mtp_loss: 14.3014 +[titan] 2025-07-09 18:04:54,057 - root - INFO - lr: 2.7546e-04 gnorm: 0.83 [ 4:29:56<17:39:50] +[titan] 2025-07-09 18:04:57,971 - root - INFO - step: 20305 loss: 16.9046 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.2803 +[titan] 2025-07-09 18:04:57,972 - root - INFO - lr: 2.7545e-04 gnorm: 0.85 [ 4:30:00<17:39:46] +[titan] 2025-07-09 18:05:01,884 - root - INFO - step: 20310 loss: 16.6226 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5933 global_avg_mtp_loss: 14.0292 +[titan] 2025-07-09 18:05:01,885 - root - INFO - lr: 2.7544e-04 gnorm: 1.72 [ 4:30:04<17:39:42] +[titan] 2025-07-09 18:05:05,802 - root - INFO - step: 20315 loss: 16.8627 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6346 global_avg_mtp_loss: 14.2281 +[titan] 2025-07-09 18:05:05,802 - root - INFO - lr: 2.7542e-04 gnorm: 0.78 [ 4:30:08<17:39:37] +[titan] 2025-07-09 18:05:09,713 - root - INFO - step: 20320 loss: 16.8624 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6236 global_avg_mtp_loss: 14.2388 +[titan] 2025-07-09 18:05:09,714 - root - INFO - lr: 2.7541e-04 gnorm: 0.86 [ 4:30:12<17:39:33] +[titan] 2025-07-09 18:05:13,640 - root - INFO - step: 20325 loss: 17.0139 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3680 +[titan] 2025-07-09 18:05:13,641 - root - INFO - lr: 2.7540e-04 gnorm: 0.79 [ 4:30:16<17:39:29] +[titan] 2025-07-09 18:05:17,602 - root - INFO - step: 20330 loss: 17.1106 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6912 global_avg_mtp_loss: 14.4194 +[titan] 2025-07-09 18:05:17,602 - root - INFO - lr: 2.7539e-04 gnorm: 0.84 [ 4:30:20<17:39:25] +[titan] 2025-07-09 18:05:21,531 - root - INFO - step: 20335 loss: 16.9735 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3179 +[titan] 2025-07-09 18:05:21,532 - root - INFO - lr: 2.7537e-04 gnorm: 0.82 [ 4:30:24<17:39:20] +[titan] 2025-07-09 18:05:25,434 - root - INFO - step: 20340 loss: 16.5292 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 13.9512 +[titan] 2025-07-09 18:05:25,435 - root - INFO - lr: 2.7536e-04 gnorm: 0.85 [ 4:30:28<17:39:16] +[titan] 2025-07-09 18:05:29,341 - root - INFO - step: 20345 loss: 16.5852 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5759 global_avg_mtp_loss: 14.0094 +[titan] 2025-07-09 18:05:29,341 - root - INFO - lr: 2.7535e-04 gnorm: 0.83 [ 4:30:32<17:39:12] +[titan] 2025-07-09 18:05:32,460 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:05:33,247 - root - INFO - step: 20350 loss: 16.9241 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6599 global_avg_mtp_loss: 14.2642 +[titan] 2025-07-09 18:05:33,247 - root - INFO - lr: 2.7534e-04 gnorm: 0.77 [ 4:30:36<17:39:07] +[titan] 2025-07-09 18:05:37,155 - root - INFO - step: 20355 loss: 17.0197 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.3663 +[titan] 2025-07-09 18:05:37,155 - root - INFO - lr: 2.7532e-04 gnorm: 0.78 [ 4:30:39<17:39:03] +[titan] 2025-07-09 18:05:41,055 - root - INFO - step: 20360 loss: 17.1387 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6878 global_avg_mtp_loss: 14.4509 +[titan] 2025-07-09 18:05:41,055 - root - INFO - lr: 2.7531e-04 gnorm: 0.74 [ 4:30:43<17:38:59] +[titan] 2025-07-09 18:05:44,966 - root - INFO - step: 20365 loss: 17.0109 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6572 global_avg_mtp_loss: 14.3537 +[titan] 2025-07-09 18:05:44,966 - root - INFO - lr: 2.7530e-04 gnorm: 0.79 [ 4:30:47<17:38:55] +[titan] 2025-07-09 18:05:48,853 - root - INFO - step: 20370 loss: 17.0177 memory: 44.58GiB(31.99%) tps: 84,308 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.6675 global_avg_mtp_loss: 14.3502 +[titan] 2025-07-09 18:05:48,853 - root - INFO - lr: 2.7529e-04 gnorm: 0.74 [ 4:30:51<17:38:50] +[titan] 2025-07-09 18:05:52,754 - root - INFO - step: 20375 loss: 17.0691 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6631 global_avg_mtp_loss: 14.4059 +[titan] 2025-07-09 18:05:52,755 - root - INFO - lr: 2.7528e-04 gnorm: 0.79 [ 4:30:55<17:38:46] +[titan] 2025-07-09 18:05:56,651 - root - INFO - step: 20380 loss: 17.2594 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.6898 global_avg_mtp_loss: 14.5696 +[titan] 2025-07-09 18:05:56,652 - root - INFO - lr: 2.7526e-04 gnorm: 0.77 [ 4:30:59<17:38:41] +[titan] 2025-07-09 18:06:00,562 - root - INFO - step: 20385 loss: 17.2107 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.7060 global_avg_mtp_loss: 14.5047 +[titan] 2025-07-09 18:06:00,562 - root - INFO - lr: 2.7525e-04 gnorm: 0.88 [ 4:31:03<17:38:37] +[titan] 2025-07-09 18:06:04,464 - root - INFO - step: 20390 loss: 16.7597 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.1530 +[titan] 2025-07-09 18:06:04,465 - root - INFO - lr: 2.7524e-04 gnorm: 0.74 [ 4:31:07<17:38:33] +[titan] 2025-07-09 18:06:08,369 - root - INFO - step: 20395 loss: 17.0421 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6444 global_avg_mtp_loss: 14.3977 +[titan] 2025-07-09 18:06:08,369 - root - INFO - lr: 2.7523e-04 gnorm: 0.80 [ 4:31:11<17:38:29] +[titan] 2025-07-09 18:06:11,485 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:06:12,295 - root - INFO - step: 20400 loss: 16.9475 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6313 global_avg_mtp_loss: 14.3162 +[titan] 2025-07-09 18:06:12,295 - root - INFO - lr: 2.7521e-04 gnorm: 0.84 [ 4:31:15<17:38:24] +[titan] 2025-07-09 18:06:16,194 - root - INFO - step: 20405 loss: 17.1684 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6745 global_avg_mtp_loss: 14.4938 +[titan] 2025-07-09 18:06:16,194 - root - INFO - lr: 2.7520e-04 gnorm: 0.80 [ 4:31:18<17:38:20] +[titan] 2025-07-09 18:06:20,099 - root - INFO - step: 20410 loss: 16.8354 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6230 global_avg_mtp_loss: 14.2124 +[titan] 2025-07-09 18:06:20,099 - root - INFO - lr: 2.7519e-04 gnorm: 0.80 [ 4:31:22<17:38:16] +[titan] 2025-07-09 18:06:24,013 - root - INFO - step: 20415 loss: 17.0576 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6563 global_avg_mtp_loss: 14.4013 +[titan] 2025-07-09 18:06:24,013 - root - INFO - lr: 2.7518e-04 gnorm: 0.80 [ 4:31:26<17:38:11] +[titan] 2025-07-09 18:06:27,921 - root - INFO - step: 20420 loss: 16.9393 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6442 global_avg_mtp_loss: 14.2952 +[titan] 2025-07-09 18:06:27,921 - root - INFO - lr: 2.7516e-04 gnorm: 0.76 [ 4:31:30<17:38:07] +[titan] 2025-07-09 18:06:31,870 - root - INFO - step: 20425 loss: 16.9304 memory: 44.58GiB(31.99%) tps: 82,973 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.6322 global_avg_mtp_loss: 14.2982 +[titan] 2025-07-09 18:06:31,871 - root - INFO - lr: 2.7515e-04 gnorm: 0.77 [ 4:31:34<17:38:03] +[titan] 2025-07-09 18:06:35,774 - root - INFO - step: 20430 loss: 17.0673 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6682 global_avg_mtp_loss: 14.3991 +[titan] 2025-07-09 18:06:35,774 - root - INFO - lr: 2.7514e-04 gnorm: 0.77 [ 4:31:38<17:37:59] +[titan] 2025-07-09 18:06:39,696 - root - INFO - step: 20435 loss: 17.1319 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6778 global_avg_mtp_loss: 14.4541 +[titan] 2025-07-09 18:06:39,696 - root - INFO - lr: 2.7513e-04 gnorm: 0.80 [ 4:31:42<17:37:54] +[titan] 2025-07-09 18:06:43,599 - root - INFO - step: 20440 loss: 16.8874 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2564 +[titan] 2025-07-09 18:06:43,600 - root - INFO - lr: 2.7511e-04 gnorm: 0.77 [ 4:31:46<17:37:50] +[titan] 2025-07-09 18:06:47,496 - root - INFO - step: 20445 loss: 17.1952 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.6976 global_avg_mtp_loss: 14.4977 +[titan] 2025-07-09 18:06:47,496 - root - INFO - lr: 2.7510e-04 gnorm: 0.75 [ 4:31:50<17:37:46] +[titan] 2025-07-09 18:06:50,605 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:06:51,402 - root - INFO - step: 20450 loss: 16.9258 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6257 global_avg_mtp_loss: 14.3001 +[titan] 2025-07-09 18:06:51,403 - root - INFO - lr: 2.7509e-04 gnorm: 0.86 [ 4:31:54<17:37:41] +[titan] 2025-07-09 18:06:55,298 - root - INFO - step: 20455 loss: 17.3989 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.7502 global_avg_mtp_loss: 14.6487 +[titan] 2025-07-09 18:06:55,298 - root - INFO - lr: 2.7508e-04 gnorm: 0.79 [ 4:31:58<17:37:37] +[titan] 2025-07-09 18:06:59,219 - root - INFO - step: 20460 loss: 16.9095 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.2831 +[titan] 2025-07-09 18:06:59,219 - root - INFO - lr: 2.7507e-04 gnorm: 0.79 [ 4:32:02<17:37:33] +[titan] 2025-07-09 18:07:03,112 - root - INFO - step: 20465 loss: 17.1672 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.6902 global_avg_mtp_loss: 14.4770 +[titan] 2025-07-09 18:07:03,112 - root - INFO - lr: 2.7505e-04 gnorm: 0.88 [ 4:32:05<17:37:28] +[titan] 2025-07-09 18:07:07,022 - root - INFO - step: 20470 loss: 16.7447 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6140 global_avg_mtp_loss: 14.1307 +[titan] 2025-07-09 18:07:07,022 - root - INFO - lr: 2.7504e-04 gnorm: 0.82 [ 4:32:09<17:37:24] +[titan] 2025-07-09 18:07:10,910 - root - INFO - step: 20475 loss: 16.9160 memory: 44.58GiB(31.99%) tps: 84,285 tflops: 290.88 mfu: 29.41% global_avg_ntp_loss: 2.6400 global_avg_mtp_loss: 14.2760 +[titan] 2025-07-09 18:07:10,910 - root - INFO - lr: 2.7503e-04 gnorm: 0.83 [ 4:32:13<17:37:20] +[titan] 2025-07-09 18:07:14,931 - root - INFO - step: 20480 loss: 16.9960 memory: 44.58GiB(31.99%) tps: 81,512 tflops: 281.31 mfu: 28.44% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.3534 +[titan] 2025-07-09 18:07:14,931 - root - INFO - lr: 2.7502e-04 gnorm: 0.84 [ 4:32:17<17:37:16] +[titan] 2025-07-09 18:07:15,086 - root - INFO - Dumping profiler traces at step 20480 +[titan] 2025-07-09 18:07:15,120 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:07:19,051 - root - INFO - step: 20485 loss: 16.9419 memory: 44.58GiB(31.99%) tps: 79,530 tflops: 274.47 mfu: 27.75% global_avg_ntp_loss: 2.6427 global_avg_mtp_loss: 14.2992 +[titan] 2025-07-09 18:07:19,051 - root - INFO - lr: 2.7500e-04 gnorm: 0.79 [ 4:32:21<17:37:12] +[titan] 2025-07-09 18:07:22,961 - root - INFO - step: 20490 loss: 17.0383 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6599 global_avg_mtp_loss: 14.3784 +[titan] 2025-07-09 18:07:22,962 - root - INFO - lr: 2.7499e-04 gnorm: 0.73 [ 4:32:25<17:37:08] +[titan] 2025-07-09 18:07:26,859 - root - INFO - step: 20495 loss: 16.8633 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.2522 +[titan] 2025-07-09 18:07:26,859 - root - INFO - lr: 2.7498e-04 gnorm: 0.81 [ 4:32:29<17:37:04] +[titan] 2025-07-09 18:07:29,976 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:07:30,769 - root - INFO - step: 20500 loss: 17.0300 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.3770 +[titan] 2025-07-09 18:07:30,769 - root - INFO - lr: 2.7497e-04 gnorm: 0.80 [ 4:32:33<17:36:59] +[titan] 2025-07-09 18:07:34,686 - root - INFO - step: 20505 loss: 17.0556 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6669 global_avg_mtp_loss: 14.3887 +[titan] 2025-07-09 18:07:34,686 - root - INFO - lr: 2.7495e-04 gnorm: 0.79 [ 4:32:37<17:36:55] +[titan] 2025-07-09 18:07:38,601 - root - INFO - step: 20510 loss: 16.8723 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.2483 +[titan] 2025-07-09 18:07:38,601 - root - INFO - lr: 2.7494e-04 gnorm: 0.78 [ 4:32:41<17:36:51] +[titan] 2025-07-09 18:07:42,539 - root - INFO - step: 20515 loss: 17.1802 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.6734 global_avg_mtp_loss: 14.5069 +[titan] 2025-07-09 18:07:42,539 - root - INFO - lr: 2.7493e-04 gnorm: 0.81 [ 4:32:45<17:36:47] +[titan] 2025-07-09 18:07:46,458 - root - INFO - step: 20520 loss: 16.8245 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.2241 +[titan] 2025-07-09 18:07:46,458 - root - INFO - lr: 2.7492e-04 gnorm: 0.76 [ 4:32:49<17:36:42] +[titan] 2025-07-09 18:07:50,390 - root - INFO - step: 20525 loss: 17.1361 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6826 global_avg_mtp_loss: 14.4535 +[titan] 2025-07-09 18:07:50,391 - root - INFO - lr: 2.7490e-04 gnorm: 0.75 [ 4:32:53<17:36:38] +[titan] 2025-07-09 18:07:54,314 - root - INFO - step: 20530 loss: 16.9164 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.2793 +[titan] 2025-07-09 18:07:54,314 - root - INFO - lr: 2.7489e-04 gnorm: 0.80 [ 4:32:57<17:36:34] +[titan] 2025-07-09 18:07:58,222 - root - INFO - step: 20535 loss: 16.9627 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3213 +[titan] 2025-07-09 18:07:58,222 - root - INFO - lr: 2.7488e-04 gnorm: 0.78 [ 4:33:01<17:36:30] +[titan] 2025-07-09 18:08:02,127 - root - INFO - step: 20540 loss: 17.0284 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6537 global_avg_mtp_loss: 14.3747 +[titan] 2025-07-09 18:08:02,128 - root - INFO - lr: 2.7487e-04 gnorm: 0.78 [ 4:33:04<17:36:25] +[titan] 2025-07-09 18:08:06,032 - root - INFO - step: 20545 loss: 17.0779 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6668 global_avg_mtp_loss: 14.4110 +[titan] 2025-07-09 18:08:06,033 - root - INFO - lr: 2.7485e-04 gnorm: 0.81 [ 4:33:08<17:36:21] +[titan] 2025-07-09 18:08:09,154 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:08:09,944 - root - INFO - step: 20550 loss: 16.8786 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.2469 +[titan] 2025-07-09 18:08:09,944 - root - INFO - lr: 2.7484e-04 gnorm: 0.76 [ 4:33:12<17:36:17] +[titan] 2025-07-09 18:08:13,868 - root - INFO - step: 20555 loss: 17.0040 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6508 global_avg_mtp_loss: 14.3532 +[titan] 2025-07-09 18:08:13,869 - root - INFO - lr: 2.7483e-04 gnorm: 0.80 [ 4:33:16<17:36:12] +[titan] 2025-07-09 18:08:17,803 - root - INFO - step: 20560 loss: 16.9417 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6314 global_avg_mtp_loss: 14.3103 +[titan] 2025-07-09 18:08:17,803 - root - INFO - lr: 2.7482e-04 gnorm: 0.81 [ 4:33:20<17:36:08] +[titan] 2025-07-09 18:08:21,708 - root - INFO - step: 20565 loss: 16.9961 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6435 global_avg_mtp_loss: 14.3526 +[titan] 2025-07-09 18:08:21,709 - root - INFO - lr: 2.7480e-04 gnorm: 0.78 [ 4:33:24<17:36:04] +[titan] 2025-07-09 18:08:25,663 - root - INFO - step: 20570 loss: 17.0603 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.6547 global_avg_mtp_loss: 14.4056 +[titan] 2025-07-09 18:08:25,664 - root - INFO - lr: 2.7479e-04 gnorm: 0.75 [ 4:33:28<17:36:00] +[titan] 2025-07-09 18:08:29,612 - root - INFO - step: 20575 loss: 17.1260 memory: 44.58GiB(31.99%) tps: 82,987 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.6827 global_avg_mtp_loss: 14.4433 +[titan] 2025-07-09 18:08:29,612 - root - INFO - lr: 2.7478e-04 gnorm: 0.79 [ 4:33:32<17:35:56] +[titan] 2025-07-09 18:08:33,521 - root - INFO - step: 20580 loss: 16.6868 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1011 +[titan] 2025-07-09 18:08:33,522 - root - INFO - lr: 2.7477e-04 gnorm: 0.83 [ 4:33:36<17:35:51] +[titan] 2025-07-09 18:08:37,423 - root - INFO - step: 20585 loss: 17.1130 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6792 global_avg_mtp_loss: 14.4338 +[titan] 2025-07-09 18:08:37,423 - root - INFO - lr: 2.7475e-04 gnorm: 0.82 [ 4:33:40<17:35:47] +[titan] 2025-07-09 18:08:41,332 - root - INFO - step: 20590 loss: 16.8683 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6266 global_avg_mtp_loss: 14.2417 +[titan] 2025-07-09 18:08:41,332 - root - INFO - lr: 2.7474e-04 gnorm: 0.76 [ 4:33:44<17:35:43] +[titan] 2025-07-09 18:08:45,263 - root - INFO - step: 20595 loss: 17.1936 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.6871 global_avg_mtp_loss: 14.5065 +[titan] 2025-07-09 18:08:45,263 - root - INFO - lr: 2.7473e-04 gnorm: 1.01 [ 4:33:48<17:35:39] +[titan] 2025-07-09 18:08:48,371 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:08:49,160 - root - INFO - step: 20600 loss: 16.9940 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6404 global_avg_mtp_loss: 14.3536 +[titan] 2025-07-09 18:08:49,160 - root - INFO - lr: 2.7472e-04 gnorm: 0.80 [ 4:33:51<17:35:34] +[titan] 2025-07-09 18:08:53,081 - root - INFO - step: 20605 loss: 17.1063 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.4435 +[titan] 2025-07-09 18:08:53,081 - root - INFO - lr: 2.7470e-04 gnorm: 0.86 [ 4:33:55<17:35:30] +[titan] 2025-07-09 18:08:57,046 - root - INFO - step: 20610 loss: 17.1236 memory: 44.58GiB(31.99%) tps: 82,651 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.4770 +[titan] 2025-07-09 18:08:57,046 - root - INFO - lr: 2.7469e-04 gnorm: 0.78 [ 4:33:59<17:35:26] +[titan] 2025-07-09 18:09:00,966 - root - INFO - step: 20615 loss: 16.7460 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.1471 +[titan] 2025-07-09 18:09:00,966 - root - INFO - lr: 2.7468e-04 gnorm: 0.74 [ 4:34:03<17:35:22] +[titan] 2025-07-09 18:09:04,886 - root - INFO - step: 20620 loss: 17.0920 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6793 global_avg_mtp_loss: 14.4126 +[titan] 2025-07-09 18:09:04,886 - root - INFO - lr: 2.7467e-04 gnorm: 0.85 [ 4:34:07<17:35:17] +[titan] 2025-07-09 18:09:08,824 - root - INFO - step: 20625 loss: 17.1129 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6696 global_avg_mtp_loss: 14.4433 +[titan] 2025-07-09 18:09:08,824 - root - INFO - lr: 2.7465e-04 gnorm: 0.72 [ 4:34:11<17:35:13] +[titan] 2025-07-09 18:09:12,734 - root - INFO - step: 20630 loss: 17.0237 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6595 global_avg_mtp_loss: 14.3642 +[titan] 2025-07-09 18:09:12,734 - root - INFO - lr: 2.7464e-04 gnorm: 0.90 [ 4:34:15<17:35:09] +[titan] 2025-07-09 18:09:16,663 - root - INFO - step: 20635 loss: 16.8687 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6220 global_avg_mtp_loss: 14.2467 +[titan] 2025-07-09 18:09:16,663 - root - INFO - lr: 2.7463e-04 gnorm: 0.84 [ 4:34:19<17:35:05] +[titan] 2025-07-09 18:09:20,574 - root - INFO - step: 20640 loss: 17.0722 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6526 global_avg_mtp_loss: 14.4195 +[titan] 2025-07-09 18:09:20,574 - root - INFO - lr: 2.7462e-04 gnorm: 0.79 [ 4:34:23<17:35:00] +[titan] 2025-07-09 18:09:24,490 - root - INFO - step: 20645 loss: 16.8739 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6218 global_avg_mtp_loss: 14.2521 +[titan] 2025-07-09 18:09:24,490 - root - INFO - lr: 2.7460e-04 gnorm: 0.76 [ 4:34:27<17:34:56] +[titan] 2025-07-09 18:09:27,642 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:09:28,440 - root - INFO - step: 20650 loss: 17.0997 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.6653 global_avg_mtp_loss: 14.4344 +[titan] 2025-07-09 18:09:28,441 - root - INFO - lr: 2.7459e-04 gnorm: 0.78 [ 4:34:31<17:34:52] +[titan] 2025-07-09 18:09:32,343 - root - INFO - step: 20655 loss: 16.7124 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.1055 +[titan] 2025-07-09 18:09:32,343 - root - INFO - lr: 2.7458e-04 gnorm: 0.91 [ 4:34:35<17:34:48] +[titan] 2025-07-09 18:09:36,249 - root - INFO - step: 20660 loss: 16.9062 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6336 global_avg_mtp_loss: 14.2726 +[titan] 2025-07-09 18:09:36,249 - root - INFO - lr: 2.7457e-04 gnorm: 0.81 [ 4:34:39<17:34:43] +[titan] 2025-07-09 18:09:40,223 - root - INFO - step: 20665 loss: 16.9889 memory: 44.58GiB(31.99%) tps: 82,447 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.6467 global_avg_mtp_loss: 14.3422 +[titan] 2025-07-09 18:09:40,224 - root - INFO - lr: 2.7455e-04 gnorm: 0.79 [ 4:34:42<17:34:39] +[titan] 2025-07-09 18:09:44,173 - root - INFO - step: 20670 loss: 17.2366 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.6914 global_avg_mtp_loss: 14.5452 +[titan] 2025-07-09 18:09:44,173 - root - INFO - lr: 2.7454e-04 gnorm: 0.79 [ 4:34:46<17:34:35] +[titan] 2025-07-09 18:09:48,093 - root - INFO - step: 20675 loss: 17.0390 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6586 global_avg_mtp_loss: 14.3804 +[titan] 2025-07-09 18:09:48,093 - root - INFO - lr: 2.7453e-04 gnorm: 0.83 [ 4:34:50<17:34:31] +[titan] 2025-07-09 18:09:52,028 - root - INFO - step: 20680 loss: 17.1365 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.6949 global_avg_mtp_loss: 14.4416 +[titan] 2025-07-09 18:09:52,028 - root - INFO - lr: 2.7452e-04 gnorm: 0.79 [ 4:34:54<17:34:27] +[titan] 2025-07-09 18:09:55,954 - root - INFO - step: 20685 loss: 16.7726 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1803 +[titan] 2025-07-09 18:09:55,954 - root - INFO - lr: 2.7450e-04 gnorm: 0.90 [ 4:34:58<17:34:23] +[titan] 2025-07-09 18:09:59,904 - root - INFO - step: 20690 loss: 17.1087 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.6983 global_avg_mtp_loss: 14.4104 +[titan] 2025-07-09 18:09:59,904 - root - INFO - lr: 2.7449e-04 gnorm: 0.80 [ 4:35:02<17:34:18] +[titan] 2025-07-09 18:10:03,802 - root - INFO - step: 20695 loss: 16.7583 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6020 global_avg_mtp_loss: 14.1564 +[titan] 2025-07-09 18:10:03,802 - root - INFO - lr: 2.7448e-04 gnorm: 0.79 [ 4:35:06<17:34:14] +[titan] 2025-07-09 18:10:06,927 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:10:07,722 - root - INFO - step: 20700 loss: 17.0729 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6632 global_avg_mtp_loss: 14.4097 +[titan] 2025-07-09 18:10:07,722 - root - INFO - lr: 2.7447e-04 gnorm: 0.71 [ 4:35:10<17:34:10] +[titan] 2025-07-09 18:10:11,685 - root - INFO - step: 20705 loss: 17.1248 memory: 44.58GiB(31.99%) tps: 82,687 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.6799 global_avg_mtp_loss: 14.4449 +[titan] 2025-07-09 18:10:11,686 - root - INFO - lr: 2.7445e-04 gnorm: 0.83 [ 4:35:14<17:34:06] +[titan] 2025-07-09 18:10:15,578 - root - INFO - step: 20710 loss: 16.7724 memory: 44.58GiB(31.99%) tps: 84,185 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.1639 +[titan] 2025-07-09 18:10:15,578 - root - INFO - lr: 2.7444e-04 gnorm: 0.78 [ 4:35:18<17:34:01] +[titan] 2025-07-09 18:10:19,520 - root - INFO - step: 20715 loss: 17.0960 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.6614 global_avg_mtp_loss: 14.4346 +[titan] 2025-07-09 18:10:19,521 - root - INFO - lr: 2.7443e-04 gnorm: 0.83 [ 4:35:22<17:33:57] +[titan] 2025-07-09 18:10:23,444 - root - INFO - step: 20720 loss: 16.7911 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.1790 +[titan] 2025-07-09 18:10:23,444 - root - INFO - lr: 2.7442e-04 gnorm: 0.80 [ 4:35:26<17:33:53] +[titan] 2025-07-09 18:10:27,347 - root - INFO - step: 20725 loss: 16.9322 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6290 global_avg_mtp_loss: 14.3032 +[titan] 2025-07-09 18:10:27,347 - root - INFO - lr: 2.7440e-04 gnorm: 0.78 [ 4:35:30<17:33:49] +[titan] 2025-07-09 18:10:31,245 - root - INFO - step: 20730 loss: 17.0686 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.4127 +[titan] 2025-07-09 18:10:31,245 - root - INFO - lr: 2.7439e-04 gnorm: 0.78 [ 4:35:34<17:33:44] +[titan] 2025-07-09 18:10:35,168 - root - INFO - step: 20735 loss: 16.7605 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6054 global_avg_mtp_loss: 14.1551 +[titan] 2025-07-09 18:10:35,169 - root - INFO - lr: 2.7438e-04 gnorm: 0.82 [ 4:35:37<17:33:40] +[titan] 2025-07-09 18:10:39,096 - root - INFO - step: 20740 loss: 17.1060 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6668 global_avg_mtp_loss: 14.4391 +[titan] 2025-07-09 18:10:39,096 - root - INFO - lr: 2.7437e-04 gnorm: 0.83 [ 4:35:41<17:33:36] +[titan] 2025-07-09 18:10:43,006 - root - INFO - step: 20745 loss: 16.8765 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6356 global_avg_mtp_loss: 14.2410 +[titan] 2025-07-09 18:10:43,006 - root - INFO - lr: 2.7435e-04 gnorm: 0.77 [ 4:35:45<17:33:32] +[titan] 2025-07-09 18:10:46,109 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:10:46,898 - root - INFO - step: 20750 loss: 16.7880 memory: 44.58GiB(31.99%) tps: 84,207 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.1767 +[titan] 2025-07-09 18:10:46,898 - root - INFO - lr: 2.7434e-04 gnorm: 0.83 [ 4:35:49<17:33:27] +[titan] 2025-07-09 18:10:50,802 - root - INFO - step: 20755 loss: 17.1178 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4473 +[titan] 2025-07-09 18:10:50,802 - root - INFO - lr: 2.7433e-04 gnorm: 0.79 [ 4:35:53<17:33:23] +[titan] 2025-07-09 18:10:54,737 - root - INFO - step: 20760 loss: 17.0931 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.6648 global_avg_mtp_loss: 14.4284 +[titan] 2025-07-09 18:10:54,737 - root - INFO - lr: 2.7432e-04 gnorm: 0.80 [ 4:35:57<17:33:19] +[titan] 2025-07-09 18:10:58,666 - root - INFO - step: 20765 loss: 16.8945 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6372 global_avg_mtp_loss: 14.2573 +[titan] 2025-07-09 18:10:58,666 - root - INFO - lr: 2.7430e-04 gnorm: 0.74 [ 4:36:01<17:33:15] +[titan] 2025-07-09 18:11:02,570 - root - INFO - step: 20770 loss: 16.8710 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2529 +[titan] 2025-07-09 18:11:02,570 - root - INFO - lr: 2.7429e-04 gnorm: 0.80 [ 4:36:05<17:33:10] +[titan] 2025-07-09 18:11:06,471 - root - INFO - step: 20775 loss: 16.6829 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.0895 +[titan] 2025-07-09 18:11:06,471 - root - INFO - lr: 2.7428e-04 gnorm: 0.83 [ 4:36:09<17:33:06] +[titan] 2025-07-09 18:11:10,372 - root - INFO - step: 20780 loss: 16.8878 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6293 global_avg_mtp_loss: 14.2585 +[titan] 2025-07-09 18:11:10,372 - root - INFO - lr: 2.7427e-04 gnorm: 0.85 [ 4:36:13<17:33:02] +[titan] 2025-07-09 18:11:14,283 - root - INFO - step: 20785 loss: 16.9414 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.3136 +[titan] 2025-07-09 18:11:14,283 - root - INFO - lr: 2.7425e-04 gnorm: 0.78 [ 4:36:17<17:32:57] +[titan] 2025-07-09 18:11:18,184 - root - INFO - step: 20790 loss: 16.9784 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.3378 +[titan] 2025-07-09 18:11:18,185 - root - INFO - lr: 2.7424e-04 gnorm: 0.80 [ 4:36:20<17:32:53] +[titan] 2025-07-09 18:11:22,098 - root - INFO - step: 20795 loss: 17.0430 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6651 global_avg_mtp_loss: 14.3779 +[titan] 2025-07-09 18:11:22,098 - root - INFO - lr: 2.7423e-04 gnorm: 0.88 [ 4:36:24<17:32:49] +[titan] 2025-07-09 18:11:25,224 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:11:26,009 - root - INFO - step: 20800 loss: 17.1161 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4456 +[titan] 2025-07-09 18:11:26,009 - root - INFO - lr: 2.7422e-04 gnorm: 0.72 [ 4:36:28<17:32:44] +[titan] 2025-07-09 18:11:29,919 - root - INFO - step: 20805 loss: 16.8136 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.2011 +[titan] 2025-07-09 18:11:29,919 - root - INFO - lr: 2.7420e-04 gnorm: 0.74 [ 4:36:32<17:32:40] +[titan] 2025-07-09 18:11:33,845 - root - INFO - step: 20810 loss: 16.9674 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6428 global_avg_mtp_loss: 14.3246 +[titan] 2025-07-09 18:11:33,845 - root - INFO - lr: 2.7419e-04 gnorm: 0.77 [ 4:36:36<17:32:36] +[titan] 2025-07-09 18:11:37,767 - root - INFO - step: 20815 loss: 17.0199 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6527 global_avg_mtp_loss: 14.3672 +[titan] 2025-07-09 18:11:37,767 - root - INFO - lr: 2.7418e-04 gnorm: 0.86 [ 4:36:40<17:32:32] +[titan] 2025-07-09 18:11:41,675 - root - INFO - step: 20820 loss: 17.1522 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.7071 global_avg_mtp_loss: 14.4451 +[titan] 2025-07-09 18:11:41,675 - root - INFO - lr: 2.7416e-04 gnorm: 0.85 [ 4:36:44<17:32:27] +[titan] 2025-07-09 18:11:45,588 - root - INFO - step: 20825 loss: 16.8818 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6384 global_avg_mtp_loss: 14.2434 +[titan] 2025-07-09 18:11:45,589 - root - INFO - lr: 2.7415e-04 gnorm: 0.78 [ 4:36:48<17:32:23] +[titan] 2025-07-09 18:11:49,504 - root - INFO - step: 20830 loss: 16.9291 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.3054 +[titan] 2025-07-09 18:11:49,504 - root - INFO - lr: 2.7414e-04 gnorm: 0.75 [ 4:36:52<17:32:19] +[titan] 2025-07-09 18:11:53,403 - root - INFO - step: 20835 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.0629 +[titan] 2025-07-09 18:11:53,403 - root - INFO - lr: 2.7413e-04 gnorm: 0.79 [ 4:36:56<17:32:15] +[titan] 2025-07-09 18:11:57,318 - root - INFO - step: 20840 loss: 16.8939 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6372 global_avg_mtp_loss: 14.2567 +[titan] 2025-07-09 18:11:57,319 - root - INFO - lr: 2.7411e-04 gnorm: 0.83 [ 4:37:00<17:32:10] +[titan] 2025-07-09 18:12:01,238 - root - INFO - step: 20845 loss: 17.2611 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.7156 global_avg_mtp_loss: 14.5455 +[titan] 2025-07-09 18:12:01,239 - root - INFO - lr: 2.7410e-04 gnorm: 0.78 [ 4:37:03<17:32:06] +[titan] 2025-07-09 18:12:04,344 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:12:05,137 - root - INFO - step: 20850 loss: 16.7466 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5911 global_avg_mtp_loss: 14.1556 +[titan] 2025-07-09 18:12:05,138 - root - INFO - lr: 2.7409e-04 gnorm: 0.78 [ 4:37:07<17:32:02] +[titan] 2025-07-09 18:12:09,041 - root - INFO - step: 20855 loss: 17.0789 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.6615 global_avg_mtp_loss: 14.4174 +[titan] 2025-07-09 18:12:09,041 - root - INFO - lr: 2.7408e-04 gnorm: 0.88 [ 4:37:11<17:31:57] +[titan] 2025-07-09 18:12:12,930 - root - INFO - step: 20860 loss: 16.8000 memory: 44.58GiB(31.99%) tps: 84,256 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.1879 +[titan] 2025-07-09 18:12:12,930 - root - INFO - lr: 2.7406e-04 gnorm: 0.79 [ 4:37:15<17:31:53] +[titan] 2025-07-09 18:12:16,829 - root - INFO - step: 20865 loss: 16.9388 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6277 global_avg_mtp_loss: 14.3111 +[titan] 2025-07-09 18:12:16,830 - root - INFO - lr: 2.7405e-04 gnorm: 0.82 [ 4:37:19<17:31:49] +[titan] 2025-07-09 18:12:20,721 - root - INFO - step: 20870 loss: 16.6676 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.5792 global_avg_mtp_loss: 14.0884 +[titan] 2025-07-09 18:12:20,722 - root - INFO - lr: 2.7404e-04 gnorm: 0.84 [ 4:37:23<17:31:44] +[titan] 2025-07-09 18:12:24,625 - root - INFO - step: 20875 loss: 16.7749 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.1667 +[titan] 2025-07-09 18:12:24,625 - root - INFO - lr: 2.7403e-04 gnorm: 0.82 [ 4:37:27<17:31:40] +[titan] 2025-07-09 18:12:28,518 - root - INFO - step: 20880 loss: 16.7865 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.1826 +[titan] 2025-07-09 18:12:28,518 - root - INFO - lr: 2.7401e-04 gnorm: 0.89 [ 4:37:31<17:31:36] +[titan] 2025-07-09 18:12:32,436 - root - INFO - step: 20885 loss: 16.9611 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6357 global_avg_mtp_loss: 14.3254 +[titan] 2025-07-09 18:12:32,436 - root - INFO - lr: 2.7400e-04 gnorm: 0.76 [ 4:37:35<17:31:31] +[titan] 2025-07-09 18:12:36,372 - root - INFO - step: 20890 loss: 16.6835 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.0906 +[titan] 2025-07-09 18:12:36,372 - root - INFO - lr: 2.7399e-04 gnorm: 0.77 [ 4:37:39<17:31:27] +[titan] 2025-07-09 18:12:40,324 - root - INFO - step: 20895 loss: 17.0745 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.6541 global_avg_mtp_loss: 14.4204 +[titan] 2025-07-09 18:12:40,324 - root - INFO - lr: 2.7398e-04 gnorm: 0.78 [ 4:37:43<17:31:23] +[titan] 2025-07-09 18:12:43,443 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:12:44,228 - root - INFO - step: 20900 loss: 17.0720 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6734 global_avg_mtp_loss: 14.3986 +[titan] 2025-07-09 18:12:44,228 - root - INFO - lr: 2.7396e-04 gnorm: 0.80 [ 4:37:46<17:31:19] +[titan] 2025-07-09 18:12:48,118 - root - INFO - step: 20905 loss: 16.9090 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2779 +[titan] 2025-07-09 18:12:48,119 - root - INFO - lr: 2.7395e-04 gnorm: 0.76 [ 4:37:50<17:31:14] +[titan] 2025-07-09 18:12:52,005 - root - INFO - step: 20910 loss: 17.0899 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.6607 global_avg_mtp_loss: 14.4292 +[titan] 2025-07-09 18:12:52,006 - root - INFO - lr: 2.7394e-04 gnorm: 0.84 [ 4:37:54<17:31:10] +[titan] 2025-07-09 18:12:55,888 - root - INFO - step: 20915 loss: 16.7237 memory: 44.58GiB(31.99%) tps: 84,401 tflops: 291.28 mfu: 29.45% global_avg_ntp_loss: 2.5990 global_avg_mtp_loss: 14.1247 +[titan] 2025-07-09 18:12:55,889 - root - INFO - lr: 2.7392e-04 gnorm: 0.78 [ 4:37:58<17:31:06] +[titan] 2025-07-09 18:12:59,794 - root - INFO - step: 20920 loss: 17.0742 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6659 global_avg_mtp_loss: 14.4084 +[titan] 2025-07-09 18:12:59,794 - root - INFO - lr: 2.7391e-04 gnorm: 0.83 [ 4:38:02<17:31:01] +[titan] 2025-07-09 18:13:03,684 - root - INFO - step: 20925 loss: 16.7511 memory: 44.58GiB(31.99%) tps: 84,253 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.6232 global_avg_mtp_loss: 14.1279 +[titan] 2025-07-09 18:13:03,684 - root - INFO - lr: 2.7390e-04 gnorm: 0.74 [ 4:38:06<17:30:57] +[titan] 2025-07-09 18:13:07,609 - root - INFO - step: 20930 loss: 17.1305 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6685 global_avg_mtp_loss: 14.4620 +[titan] 2025-07-09 18:13:07,609 - root - INFO - lr: 2.7389e-04 gnorm: 0.87 [ 4:38:10<17:30:53] +[titan] 2025-07-09 18:13:11,501 - root - INFO - step: 20935 loss: 17.0227 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3709 +[titan] 2025-07-09 18:13:11,501 - root - INFO - lr: 2.7387e-04 gnorm: 0.78 [ 4:38:14<17:30:48] +[titan] 2025-07-09 18:13:15,392 - root - INFO - step: 20940 loss: 17.1114 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.6692 global_avg_mtp_loss: 14.4422 +[titan] 2025-07-09 18:13:15,392 - root - INFO - lr: 2.7386e-04 gnorm: 0.79 [ 4:38:18<17:30:44] +[titan] 2025-07-09 18:13:19,290 - root - INFO - step: 20945 loss: 16.8318 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2153 +[titan] 2025-07-09 18:13:19,290 - root - INFO - lr: 2.7385e-04 gnorm: 0.80 [ 4:38:22<17:30:40] +[titan] 2025-07-09 18:13:22,403 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:13:23,196 - root - INFO - step: 20950 loss: 17.1095 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6742 global_avg_mtp_loss: 14.4352 +[titan] 2025-07-09 18:13:23,196 - root - INFO - lr: 2.7384e-04 gnorm: 0.89 [ 4:38:25<17:30:36] +[titan] 2025-07-09 18:13:27,180 - root - INFO - step: 20955 loss: 16.8710 memory: 44.58GiB(31.99%) tps: 82,244 tflops: 283.84 mfu: 28.70% global_avg_ntp_loss: 2.6251 global_avg_mtp_loss: 14.2460 +[titan] 2025-07-09 18:13:27,181 - root - INFO - lr: 2.7382e-04 gnorm: 0.83 [ 4:38:29<17:30:32] +[titan] 2025-07-09 18:13:31,085 - root - INFO - step: 20960 loss: 16.6501 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.0603 +[titan] 2025-07-09 18:13:31,085 - root - INFO - lr: 2.7381e-04 gnorm: 0.82 [ 4:38:33<17:30:27] +[titan] 2025-07-09 18:13:35,002 - root - INFO - step: 20965 loss: 16.8117 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6238 global_avg_mtp_loss: 14.1879 +[titan] 2025-07-09 18:13:35,002 - root - INFO - lr: 2.7380e-04 gnorm: 0.74 [ 4:38:37<17:30:23] +[titan] 2025-07-09 18:13:38,917 - root - INFO - step: 20970 loss: 17.0466 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.6552 global_avg_mtp_loss: 14.3914 +[titan] 2025-07-09 18:13:38,917 - root - INFO - lr: 2.7379e-04 gnorm: 0.77 [ 4:38:41<17:30:19] +[titan] 2025-07-09 18:13:42,853 - root - INFO - step: 20975 loss: 16.9985 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6587 global_avg_mtp_loss: 14.3398 +[titan] 2025-07-09 18:13:42,853 - root - INFO - lr: 2.7377e-04 gnorm: 0.80 [ 4:38:45<17:30:15] +[titan] 2025-07-09 18:13:46,777 - root - INFO - step: 20980 loss: 16.5984 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.0223 +[titan] 2025-07-09 18:13:46,778 - root - INFO - lr: 2.7376e-04 gnorm: 0.80 [ 4:38:49<17:30:10] +[titan] 2025-07-09 18:13:50,725 - root - INFO - step: 20985 loss: 16.9578 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3141 +[titan] 2025-07-09 18:13:50,725 - root - INFO - lr: 2.7375e-04 gnorm: 0.80 [ 4:38:53<17:30:06] +[titan] 2025-07-09 18:13:54,718 - root - INFO - step: 20990 loss: 16.8598 memory: 44.58GiB(31.99%) tps: 82,065 tflops: 283.22 mfu: 28.64% global_avg_ntp_loss: 2.6265 global_avg_mtp_loss: 14.2333 +[titan] 2025-07-09 18:13:54,718 - root - INFO - lr: 2.7373e-04 gnorm: 0.79 [ 4:38:57<17:30:02] +[titan] 2025-07-09 18:13:56,435 - root - INFO - Dumping profiler traces at step 20992 +[titan] 2025-07-09 18:13:56,469 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:13:58,821 - root - INFO - step: 20995 loss: 16.9300 memory: 44.58GiB(31.99%) tps: 79,878 tflops: 275.67 mfu: 27.87% global_avg_ntp_loss: 2.6406 global_avg_mtp_loss: 14.2894 +[titan] 2025-07-09 18:13:58,821 - root - INFO - lr: 2.7372e-04 gnorm: 0.83 [ 4:39:01<17:29:59] +[titan] 2025-07-09 18:14:01,924 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:14:02,711 - root - INFO - step: 21000 loss: 17.1624 memory: 44.58GiB(31.99%) tps: 84,242 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.6699 global_avg_mtp_loss: 14.4925 +[titan] 2025-07-09 18:14:02,711 - root - INFO - lr: 2.7371e-04 gnorm: 0.76 [ 4:39:05<17:29:54] +[titan] 2025-07-09 18:14:06,628 - root - INFO - step: 21005 loss: 16.8771 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2606 +[titan] 2025-07-09 18:14:06,629 - root - INFO - lr: 2.7370e-04 gnorm: 0.78 [ 4:39:09<17:29:50] +[titan] 2025-07-09 18:14:10,512 - root - INFO - step: 21010 loss: 16.9922 memory: 44.58GiB(31.99%) tps: 84,382 tflops: 291.21 mfu: 29.45% global_avg_ntp_loss: 2.6432 global_avg_mtp_loss: 14.3490 +[titan] 2025-07-09 18:14:10,512 - root - INFO - lr: 2.7368e-04 gnorm: 0.75 [ 4:39:13<17:29:46] +[titan] 2025-07-09 18:14:14,406 - root - INFO - step: 21015 loss: 17.1421 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.6755 global_avg_mtp_loss: 14.4666 +[titan] 2025-07-09 18:14:14,406 - root - INFO - lr: 2.7367e-04 gnorm: 0.78 [ 4:39:17<17:29:41] +[titan] 2025-07-09 18:14:18,342 - root - INFO - step: 21020 loss: 16.7393 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5932 global_avg_mtp_loss: 14.1461 +[titan] 2025-07-09 18:14:18,342 - root - INFO - lr: 2.7366e-04 gnorm: 0.85 [ 4:39:21<17:29:37] +[titan] 2025-07-09 18:14:22,240 - root - INFO - step: 21025 loss: 17.1933 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6810 global_avg_mtp_loss: 14.5123 +[titan] 2025-07-09 18:14:22,240 - root - INFO - lr: 2.7365e-04 gnorm: 0.81 [ 4:39:24<17:29:33] +[titan] 2025-07-09 18:14:26,161 - root - INFO - step: 21030 loss: 16.9002 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.2710 +[titan] 2025-07-09 18:14:26,161 - root - INFO - lr: 2.7363e-04 gnorm: 0.81 [ 4:39:28<17:29:29] +[titan] 2025-07-09 18:14:30,097 - root - INFO - step: 21035 loss: 16.7525 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.1465 +[titan] 2025-07-09 18:14:30,097 - root - INFO - lr: 2.7362e-04 gnorm: 0.75 [ 4:39:32<17:29:24] +[titan] 2025-07-09 18:14:34,006 - root - INFO - step: 21040 loss: 17.1079 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6673 global_avg_mtp_loss: 14.4406 +[titan] 2025-07-09 18:14:34,006 - root - INFO - lr: 2.7361e-04 gnorm: 0.75 [ 4:39:36<17:29:20] +[titan] 2025-07-09 18:14:37,910 - root - INFO - step: 21045 loss: 16.8816 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6124 global_avg_mtp_loss: 14.2693 +[titan] 2025-07-09 18:14:37,910 - root - INFO - lr: 2.7360e-04 gnorm: 0.80 [ 4:39:40<17:29:16] +[titan] 2025-07-09 18:14:41,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:14:41,833 - root - INFO - step: 21050 loss: 16.6278 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0570 +[titan] 2025-07-09 18:14:41,833 - root - INFO - lr: 2.7358e-04 gnorm: 0.82 [ 4:39:44<17:29:12] +[titan] 2025-07-09 18:14:45,762 - root - INFO - step: 21055 loss: 16.7893 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6080 global_avg_mtp_loss: 14.1813 +[titan] 2025-07-09 18:14:45,762 - root - INFO - lr: 2.7357e-04 gnorm: 0.79 [ 4:39:48<17:29:07] +[titan] 2025-07-09 18:14:49,676 - root - INFO - step: 21060 loss: 17.0558 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6507 global_avg_mtp_loss: 14.4051 +[titan] 2025-07-09 18:14:49,676 - root - INFO - lr: 2.7356e-04 gnorm: 0.75 [ 4:39:52<17:29:03] +[titan] 2025-07-09 18:14:53,612 - root - INFO - step: 21065 loss: 17.1644 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.6755 global_avg_mtp_loss: 14.4889 +[titan] 2025-07-09 18:14:53,613 - root - INFO - lr: 2.7354e-04 gnorm: 0.82 [ 4:39:56<17:28:59] +[titan] 2025-07-09 18:14:57,521 - root - INFO - step: 21070 loss: 17.1106 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.6702 global_avg_mtp_loss: 14.4404 +[titan] 2025-07-09 18:14:57,521 - root - INFO - lr: 2.7353e-04 gnorm: 0.76 [ 4:40:00<17:28:55] +[titan] 2025-07-09 18:15:01,443 - root - INFO - step: 21075 loss: 16.8401 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.2270 +[titan] 2025-07-09 18:15:01,443 - root - INFO - lr: 2.7352e-04 gnorm: 0.75 [ 4:40:04<17:28:50] +[titan] 2025-07-09 18:15:05,382 - root - INFO - step: 21080 loss: 16.8660 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6344 global_avg_mtp_loss: 14.2316 +[titan] 2025-07-09 18:15:05,383 - root - INFO - lr: 2.7351e-04 gnorm: 0.86 [ 4:40:08<17:28:46] +[titan] 2025-07-09 18:15:09,297 - root - INFO - step: 21085 loss: 17.0582 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6681 global_avg_mtp_loss: 14.3901 +[titan] 2025-07-09 18:15:09,298 - root - INFO - lr: 2.7349e-04 gnorm: 0.83 [ 4:40:12<17:28:42] +[titan] 2025-07-09 18:15:13,225 - root - INFO - step: 21090 loss: 16.9806 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.6491 global_avg_mtp_loss: 14.3315 +[titan] 2025-07-09 18:15:13,225 - root - INFO - lr: 2.7348e-04 gnorm: 0.87 [ 4:40:15<17:28:38] +[titan] 2025-07-09 18:15:17,107 - root - INFO - step: 21095 loss: 16.8841 memory: 44.58GiB(31.99%) tps: 84,427 tflops: 291.37 mfu: 29.46% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2615 +[titan] 2025-07-09 18:15:17,107 - root - INFO - lr: 2.7347e-04 gnorm: 0.77 [ 4:40:19<17:28:33] +[titan] 2025-07-09 18:15:20,240 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:15:21,035 - root - INFO - step: 21100 loss: 16.9212 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6507 global_avg_mtp_loss: 14.2705 +[titan] 2025-07-09 18:15:21,036 - root - INFO - lr: 2.7345e-04 gnorm: 0.75 [ 4:40:23<17:28:29] +[titan] 2025-07-09 18:15:24,955 - root - INFO - step: 21105 loss: 17.1096 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6617 global_avg_mtp_loss: 14.4479 +[titan] 2025-07-09 18:15:24,955 - root - INFO - lr: 2.7344e-04 gnorm: 0.74 [ 4:40:27<17:28:25] +[titan] 2025-07-09 18:15:28,853 - root - INFO - step: 21110 loss: 16.4059 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 13.8748 +[titan] 2025-07-09 18:15:28,853 - root - INFO - lr: 2.7343e-04 gnorm: 0.77 [ 4:40:31<17:28:21] +[titan] 2025-07-09 18:15:32,852 - root - INFO - step: 21115 loss: 16.9821 memory: 44.58GiB(31.99%) tps: 81,951 tflops: 282.83 mfu: 28.60% global_avg_ntp_loss: 2.6500 global_avg_mtp_loss: 14.3321 +[titan] 2025-07-09 18:15:32,852 - root - INFO - lr: 2.7342e-04 gnorm: 0.82 [ 4:40:35<17:28:17] +[titan] 2025-07-09 18:15:36,754 - root - INFO - step: 21120 loss: 16.7248 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5979 global_avg_mtp_loss: 14.1269 +[titan] 2025-07-09 18:15:36,754 - root - INFO - lr: 2.7340e-04 gnorm: 0.85 [ 4:40:39<17:28:12] +[titan] 2025-07-09 18:15:40,688 - root - INFO - step: 21125 loss: 16.9241 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6436 global_avg_mtp_loss: 14.2805 +[titan] 2025-07-09 18:15:40,688 - root - INFO - lr: 2.7339e-04 gnorm: 0.84 [ 4:40:43<17:28:08] +[titan] 2025-07-09 18:15:44,602 - root - INFO - step: 21130 loss: 17.0011 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.3477 +[titan] 2025-07-09 18:15:44,602 - root - INFO - lr: 2.7338e-04 gnorm: 0.79 [ 4:40:47<17:28:04] +[titan] 2025-07-09 18:15:48,579 - root - INFO - step: 21135 loss: 16.7414 memory: 44.58GiB(31.99%) tps: 82,386 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.6062 global_avg_mtp_loss: 14.1352 +[titan] 2025-07-09 18:15:48,580 - root - INFO - lr: 2.7337e-04 gnorm: 0.85 [ 4:40:51<17:28:00] +[titan] 2025-07-09 18:15:52,468 - root - INFO - step: 21140 loss: 16.9094 memory: 44.58GiB(31.99%) tps: 84,277 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.6478 global_avg_mtp_loss: 14.2616 +[titan] 2025-07-09 18:15:52,468 - root - INFO - lr: 2.7335e-04 gnorm: 0.74 [ 4:40:55<17:27:56] +[titan] 2025-07-09 18:15:56,392 - root - INFO - step: 21145 loss: 16.9115 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6282 global_avg_mtp_loss: 14.2834 +[titan] 2025-07-09 18:15:56,393 - root - INFO - lr: 2.7334e-04 gnorm: 0.89 [ 4:40:59<17:27:51] +[titan] 2025-07-09 18:15:59,504 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:16:00,302 - root - INFO - step: 21150 loss: 16.5222 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-09 18:16:00,303 - root - INFO - lr: 2.7333e-04 gnorm: 0.77 [ 4:41:03<17:27:47] +[titan] 2025-07-09 18:16:04,215 - root - INFO - step: 21155 loss: 16.8746 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2524 +[titan] 2025-07-09 18:16:04,215 - root - INFO - lr: 2.7331e-04 gnorm: 0.85 [ 4:41:06<17:27:43] +[titan] 2025-07-09 18:16:08,133 - root - INFO - step: 21160 loss: 16.9305 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6352 global_avg_mtp_loss: 14.2953 +[titan] 2025-07-09 18:16:08,133 - root - INFO - lr: 2.7330e-04 gnorm: 0.77 [ 4:41:10<17:27:39] +[titan] 2025-07-09 18:16:12,060 - root - INFO - step: 21165 loss: 16.8823 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6147 global_avg_mtp_loss: 14.2676 +[titan] 2025-07-09 18:16:12,060 - root - INFO - lr: 2.7329e-04 gnorm: 0.84 [ 4:41:14<17:27:34] +[titan] 2025-07-09 18:16:15,970 - root - INFO - step: 21170 loss: 16.7033 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.1009 +[titan] 2025-07-09 18:16:15,970 - root - INFO - lr: 2.7328e-04 gnorm: 0.77 [ 4:41:18<17:27:30] +[titan] 2025-07-09 18:16:19,863 - root - INFO - step: 21175 loss: 16.8380 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.2001 +[titan] 2025-07-09 18:16:19,863 - root - INFO - lr: 2.7326e-04 gnorm: 1.18 [ 4:41:22<17:27:26] +[titan] 2025-07-09 18:16:23,772 - root - INFO - step: 21180 loss: 16.9402 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6513 global_avg_mtp_loss: 14.2889 +[titan] 2025-07-09 18:16:23,772 - root - INFO - lr: 2.7325e-04 gnorm: 0.93 [ 4:41:26<17:27:22] +[titan] 2025-07-09 18:16:27,667 - root - INFO - step: 21185 loss: 17.2125 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6999 global_avg_mtp_loss: 14.5126 +[titan] 2025-07-09 18:16:27,667 - root - INFO - lr: 2.7324e-04 gnorm: 0.89 [ 4:41:30<17:27:17] +[titan] 2025-07-09 18:16:31,573 - root - INFO - step: 21190 loss: 17.0073 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3636 +[titan] 2025-07-09 18:16:31,574 - root - INFO - lr: 2.7322e-04 gnorm: 0.78 [ 4:41:34<17:27:13] +[titan] 2025-07-09 18:16:35,507 - root - INFO - step: 21195 loss: 16.9266 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6304 global_avg_mtp_loss: 14.2961 +[titan] 2025-07-09 18:16:35,508 - root - INFO - lr: 2.7321e-04 gnorm: 0.84 [ 4:41:38<17:27:09] +[titan] 2025-07-09 18:16:38,623 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:16:39,407 - root - INFO - step: 21200 loss: 16.6140 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0412 +[titan] 2025-07-09 18:16:39,408 - root - INFO - lr: 2.7320e-04 gnorm: 0.77 [ 4:41:42<17:27:04] +[titan] 2025-07-09 18:16:43,309 - root - INFO - step: 21205 loss: 17.0530 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6578 global_avg_mtp_loss: 14.3952 +[titan] 2025-07-09 18:16:43,309 - root - INFO - lr: 2.7319e-04 gnorm: 0.76 [ 4:41:46<17:27:00] +[titan] 2025-07-09 18:16:47,226 - root - INFO - step: 21210 loss: 16.8448 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6167 global_avg_mtp_loss: 14.2281 +[titan] 2025-07-09 18:16:47,226 - root - INFO - lr: 2.7317e-04 gnorm: 0.74 [ 4:41:49<17:26:56] +[titan] 2025-07-09 18:16:51,147 - root - INFO - step: 21215 loss: 16.9061 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6282 global_avg_mtp_loss: 14.2779 +[titan] 2025-07-09 18:16:51,148 - root - INFO - lr: 2.7316e-04 gnorm: 0.71 [ 4:41:53<17:26:52] +[titan] 2025-07-09 18:16:55,084 - root - INFO - step: 21220 loss: 16.8501 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.2436 +[titan] 2025-07-09 18:16:55,085 - root - INFO - lr: 2.7315e-04 gnorm: 0.77 [ 4:41:57<17:26:47] +[titan] 2025-07-09 18:16:58,991 - root - INFO - step: 21225 loss: 16.8502 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.2242 +[titan] 2025-07-09 18:16:58,991 - root - INFO - lr: 2.7314e-04 gnorm: 0.82 [ 4:42:01<17:26:43] +[titan] 2025-07-09 18:17:02,885 - root - INFO - step: 21230 loss: 16.8298 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.1981 +[titan] 2025-07-09 18:17:02,885 - root - INFO - lr: 2.7312e-04 gnorm: 0.76 [ 4:42:05<17:26:39] +[titan] 2025-07-09 18:17:06,794 - root - INFO - step: 21235 loss: 16.9571 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6511 global_avg_mtp_loss: 14.3060 +[titan] 2025-07-09 18:17:06,795 - root - INFO - lr: 2.7311e-04 gnorm: 0.79 [ 4:42:09<17:26:35] +[titan] 2025-07-09 18:17:10,694 - root - INFO - step: 21240 loss: 16.9743 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6366 global_avg_mtp_loss: 14.3377 +[titan] 2025-07-09 18:17:10,694 - root - INFO - lr: 2.7310e-04 gnorm: 0.78 [ 4:42:13<17:26:30] +[titan] 2025-07-09 18:17:14,578 - root - INFO - step: 21245 loss: 17.2071 memory: 44.58GiB(31.99%) tps: 84,382 tflops: 291.22 mfu: 29.45% global_avg_ntp_loss: 2.6916 global_avg_mtp_loss: 14.5155 +[titan] 2025-07-09 18:17:14,578 - root - INFO - lr: 2.7308e-04 gnorm: 0.81 [ 4:42:17<17:26:26] +[titan] 2025-07-09 18:17:17,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:17:18,498 - root - INFO - step: 21250 loss: 16.9829 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6421 global_avg_mtp_loss: 14.3407 +[titan] 2025-07-09 18:17:18,498 - root - INFO - lr: 2.7307e-04 gnorm: 0.77 [ 4:42:21<17:26:22] +[titan] 2025-07-09 18:17:22,449 - root - INFO - step: 21255 loss: 16.7797 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1682 +[titan] 2025-07-09 18:17:22,449 - root - INFO - lr: 2.7306e-04 gnorm: 0.75 [ 4:42:25<17:26:18] +[titan] 2025-07-09 18:17:26,398 - root - INFO - step: 21260 loss: 16.9369 memory: 44.58GiB(31.99%) tps: 82,997 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.3058 +[titan] 2025-07-09 18:17:26,398 - root - INFO - lr: 2.7305e-04 gnorm: 0.79 [ 4:42:29<17:26:13] +[titan] 2025-07-09 18:17:30,333 - root - INFO - step: 21265 loss: 16.8159 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.1974 +[titan] 2025-07-09 18:17:30,333 - root - INFO - lr: 2.7303e-04 gnorm: 0.84 [ 4:42:33<17:26:09] +[titan] 2025-07-09 18:17:34,259 - root - INFO - step: 21270 loss: 16.8848 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.2572 +[titan] 2025-07-09 18:17:34,259 - root - INFO - lr: 2.7302e-04 gnorm: 0.84 [ 4:42:36<17:26:05] +[titan] 2025-07-09 18:17:38,205 - root - INFO - step: 21275 loss: 17.0552 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.6529 global_avg_mtp_loss: 14.4023 +[titan] 2025-07-09 18:17:38,205 - root - INFO - lr: 2.7301e-04 gnorm: 0.82 [ 4:42:40<17:26:01] +[titan] 2025-07-09 18:17:42,145 - root - INFO - step: 21280 loss: 16.9466 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6392 global_avg_mtp_loss: 14.3074 +[titan] 2025-07-09 18:17:42,145 - root - INFO - lr: 2.7299e-04 gnorm: 0.80 [ 4:42:44<17:25:57] +[titan] 2025-07-09 18:17:46,075 - root - INFO - step: 21285 loss: 17.0600 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6600 global_avg_mtp_loss: 14.4000 +[titan] 2025-07-09 18:17:46,075 - root - INFO - lr: 2.7298e-04 gnorm: 0.74 [ 4:42:48<17:25:53] +[titan] 2025-07-09 18:17:49,993 - root - INFO - step: 21290 loss: 17.0984 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6728 global_avg_mtp_loss: 14.4256 +[titan] 2025-07-09 18:17:49,993 - root - INFO - lr: 2.7297e-04 gnorm: 0.80 [ 4:42:52<17:25:48] +[titan] 2025-07-09 18:17:53,907 - root - INFO - step: 21295 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6455 global_avg_mtp_loss: 14.2945 +[titan] 2025-07-09 18:17:53,908 - root - INFO - lr: 2.7296e-04 gnorm: 0.83 [ 4:42:56<17:25:44] +[titan] 2025-07-09 18:17:57,023 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:17:57,811 - root - INFO - step: 21300 loss: 16.8928 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6229 global_avg_mtp_loss: 14.2699 +[titan] 2025-07-09 18:17:57,811 - root - INFO - lr: 2.7294e-04 gnorm: 0.91 [ 4:43:00<17:25:40] +[titan] 2025-07-09 18:18:01,728 - root - INFO - step: 21305 loss: 16.9634 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.3314 +[titan] 2025-07-09 18:18:01,728 - root - INFO - lr: 2.7293e-04 gnorm: 0.81 [ 4:43:04<17:25:36] +[titan] 2025-07-09 18:18:05,645 - root - INFO - step: 21310 loss: 16.8790 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6183 global_avg_mtp_loss: 14.2606 +[titan] 2025-07-09 18:18:05,645 - root - INFO - lr: 2.7292e-04 gnorm: 0.79 [ 4:43:08<17:25:31] +[titan] 2025-07-09 18:18:09,555 - root - INFO - step: 21315 loss: 16.7760 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.1619 +[titan] 2025-07-09 18:18:09,555 - root - INFO - lr: 2.7290e-04 gnorm: 0.77 [ 4:43:12<17:25:27] +[titan] 2025-07-09 18:18:13,469 - root - INFO - step: 21320 loss: 16.9397 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.3001 +[titan] 2025-07-09 18:18:13,469 - root - INFO - lr: 2.7289e-04 gnorm: 0.86 [ 4:43:16<17:25:23] +[titan] 2025-07-09 18:18:17,419 - root - INFO - step: 21325 loss: 16.7328 memory: 44.58GiB(31.99%) tps: 82,962 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5973 global_avg_mtp_loss: 14.1355 +[titan] 2025-07-09 18:18:17,419 - root - INFO - lr: 2.7288e-04 gnorm: 0.80 [ 4:43:20<17:25:19] +[titan] 2025-07-09 18:18:21,438 - root - INFO - step: 21330 loss: 17.1017 memory: 44.58GiB(31.99%) tps: 81,534 tflops: 281.39 mfu: 28.45% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.4319 +[titan] 2025-07-09 18:18:21,439 - root - INFO - lr: 2.7287e-04 gnorm: 0.88 [ 4:43:24<17:25:15] +[titan] 2025-07-09 18:18:25,356 - root - INFO - step: 21335 loss: 16.9076 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6352 global_avg_mtp_loss: 14.2724 +[titan] 2025-07-09 18:18:25,356 - root - INFO - lr: 2.7285e-04 gnorm: 0.90 [ 4:43:28<17:25:11] +[titan] 2025-07-09 18:18:29,328 - root - INFO - step: 21340 loss: 16.8794 memory: 44.58GiB(31.99%) tps: 82,495 tflops: 284.70 mfu: 28.79% global_avg_ntp_loss: 2.6288 global_avg_mtp_loss: 14.2507 +[titan] 2025-07-09 18:18:29,329 - root - INFO - lr: 2.7284e-04 gnorm: 0.79 [ 4:43:32<17:25:06] +[titan] 2025-07-09 18:18:33,386 - root - INFO - step: 21345 loss: 16.9760 memory: 44.58GiB(31.99%) tps: 80,766 tflops: 278.74 mfu: 28.18% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.3372 +[titan] 2025-07-09 18:18:33,386 - root - INFO - lr: 2.7283e-04 gnorm: 0.81 [ 4:43:36<17:25:03] +[titan] 2025-07-09 18:18:36,517 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:18:37,309 - root - INFO - step: 21350 loss: 16.8137 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.2070 +[titan] 2025-07-09 18:18:37,309 - root - INFO - lr: 2.7281e-04 gnorm: 0.77 [ 4:43:40<17:24:59] +[titan] 2025-07-09 18:18:41,236 - root - INFO - step: 21355 loss: 16.8639 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.6303 global_avg_mtp_loss: 14.2336 +[titan] 2025-07-09 18:18:41,236 - root - INFO - lr: 2.7280e-04 gnorm: 0.85 [ 4:43:43<17:24:54] +[titan] 2025-07-09 18:18:45,140 - root - INFO - step: 21360 loss: 17.0651 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6586 global_avg_mtp_loss: 14.4066 +[titan] 2025-07-09 18:18:45,140 - root - INFO - lr: 2.7279e-04 gnorm: 0.80 [ 4:43:47<17:24:50] +[titan] 2025-07-09 18:18:49,072 - root - INFO - step: 21365 loss: 16.8780 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6183 global_avg_mtp_loss: 14.2597 +[titan] 2025-07-09 18:18:49,072 - root - INFO - lr: 2.7278e-04 gnorm: 0.74 [ 4:43:51<17:24:46] +[titan] 2025-07-09 18:18:52,994 - root - INFO - step: 21370 loss: 16.8800 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6219 global_avg_mtp_loss: 14.2581 +[titan] 2025-07-09 18:18:52,994 - root - INFO - lr: 2.7276e-04 gnorm: 0.80 [ 4:43:55<17:24:42] +[titan] 2025-07-09 18:18:56,914 - root - INFO - step: 21375 loss: 16.9075 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6236 global_avg_mtp_loss: 14.2839 +[titan] 2025-07-09 18:18:56,914 - root - INFO - lr: 2.7275e-04 gnorm: 0.81 [ 4:43:59<17:24:37] +[titan] 2025-07-09 18:19:00,840 - root - INFO - step: 21380 loss: 16.7551 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.1463 +[titan] 2025-07-09 18:19:00,840 - root - INFO - lr: 2.7274e-04 gnorm: 0.75 [ 4:44:03<17:24:33] +[titan] 2025-07-09 18:19:04,760 - root - INFO - step: 21385 loss: 17.0324 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6501 global_avg_mtp_loss: 14.3823 +[titan] 2025-07-09 18:19:04,760 - root - INFO - lr: 2.7272e-04 gnorm: 0.81 [ 4:44:07<17:24:29] +[titan] 2025-07-09 18:19:08,684 - root - INFO - step: 21390 loss: 17.0703 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6769 global_avg_mtp_loss: 14.3934 +[titan] 2025-07-09 18:19:08,684 - root - INFO - lr: 2.7271e-04 gnorm: 0.83 [ 4:44:11<17:24:25] +[titan] 2025-07-09 18:19:12,601 - root - INFO - step: 21395 loss: 17.0189 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6499 global_avg_mtp_loss: 14.3690 +[titan] 2025-07-09 18:19:12,601 - root - INFO - lr: 2.7270e-04 gnorm: 0.88 [ 4:44:15<17:24:21] +[titan] 2025-07-09 18:19:15,717 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:19:16,507 - root - INFO - step: 21400 loss: 17.0689 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.6693 global_avg_mtp_loss: 14.3996 +[titan] 2025-07-09 18:19:16,507 - root - INFO - lr: 2.7268e-04 gnorm: 2.17 [ 4:44:19<17:24:16] +[titan] 2025-07-09 18:19:20,424 - root - INFO - step: 21405 loss: 16.9232 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2897 +[titan] 2025-07-09 18:19:20,424 - root - INFO - lr: 2.7267e-04 gnorm: 0.91 [ 4:44:23<17:24:12] +[titan] 2025-07-09 18:19:24,330 - root - INFO - step: 21410 loss: 16.7938 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6162 global_avg_mtp_loss: 14.1776 +[titan] 2025-07-09 18:19:24,331 - root - INFO - lr: 2.7266e-04 gnorm: 0.84 [ 4:44:27<17:24:08] +[titan] 2025-07-09 18:19:28,252 - root - INFO - step: 21415 loss: 17.0116 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6490 global_avg_mtp_loss: 14.3626 +[titan] 2025-07-09 18:19:28,252 - root - INFO - lr: 2.7265e-04 gnorm: 0.97 [ 4:44:30<17:24:03] +[titan] 2025-07-09 18:19:32,148 - root - INFO - step: 21420 loss: 16.8974 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6171 global_avg_mtp_loss: 14.2803 +[titan] 2025-07-09 18:19:32,148 - root - INFO - lr: 2.7263e-04 gnorm: 0.81 [ 4:44:34<17:23:59] +[titan] 2025-07-09 18:19:36,093 - root - INFO - step: 21425 loss: 17.0366 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3810 +[titan] 2025-07-09 18:19:36,093 - root - INFO - lr: 2.7262e-04 gnorm: 0.80 [ 4:44:38<17:23:55] +[titan] 2025-07-09 18:19:40,005 - root - INFO - step: 21430 loss: 17.1002 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6753 global_avg_mtp_loss: 14.4249 +[titan] 2025-07-09 18:19:40,005 - root - INFO - lr: 2.7261e-04 gnorm: 0.81 [ 4:44:42<17:23:51] +[titan] 2025-07-09 18:19:43,913 - root - INFO - step: 21435 loss: 16.9063 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6254 global_avg_mtp_loss: 14.2809 +[titan] 2025-07-09 18:19:43,913 - root - INFO - lr: 2.7259e-04 gnorm: 0.76 [ 4:44:46<17:23:47] +[titan] 2025-07-09 18:19:47,840 - root - INFO - step: 21440 loss: 16.7767 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6148 global_avg_mtp_loss: 14.1619 +[titan] 2025-07-09 18:19:47,840 - root - INFO - lr: 2.7258e-04 gnorm: 0.77 [ 4:44:50<17:23:42] +[titan] 2025-07-09 18:19:51,760 - root - INFO - step: 21445 loss: 16.9462 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6462 global_avg_mtp_loss: 14.3001 +[titan] 2025-07-09 18:19:51,760 - root - INFO - lr: 2.7257e-04 gnorm: 0.83 [ 4:44:54<17:23:38] +[titan] 2025-07-09 18:19:54,924 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:19:55,715 - root - INFO - step: 21450 loss: 17.0410 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.6640 global_avg_mtp_loss: 14.3769 +[titan] 2025-07-09 18:19:55,715 - root - INFO - lr: 2.7256e-04 gnorm: 0.77 [ 4:44:58<17:23:34] +[titan] 2025-07-09 18:19:59,614 - root - INFO - step: 21455 loss: 16.9714 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.3234 +[titan] 2025-07-09 18:19:59,614 - root - INFO - lr: 2.7254e-04 gnorm: 0.74 [ 4:45:02<17:23:30] +[titan] 2025-07-09 18:20:03,529 - root - INFO - step: 21460 loss: 17.1011 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6683 global_avg_mtp_loss: 14.4328 +[titan] 2025-07-09 18:20:03,529 - root - INFO - lr: 2.7253e-04 gnorm: 0.83 [ 4:45:06<17:23:25] +[titan] 2025-07-09 18:20:07,427 - root - INFO - step: 21465 loss: 17.0284 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6478 global_avg_mtp_loss: 14.3806 +[titan] 2025-07-09 18:20:07,427 - root - INFO - lr: 2.7252e-04 gnorm: 0.74 [ 4:45:10<17:23:21] +[titan] 2025-07-09 18:20:11,337 - root - INFO - step: 21470 loss: 16.7190 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1283 +[titan] 2025-07-09 18:20:11,337 - root - INFO - lr: 2.7250e-04 gnorm: 0.77 [ 4:45:14<17:23:17] +[titan] 2025-07-09 18:20:15,255 - root - INFO - step: 21475 loss: 17.0059 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.3639 +[titan] 2025-07-09 18:20:15,255 - root - INFO - lr: 2.7249e-04 gnorm: 0.76 [ 4:45:17<17:23:13] +[titan] 2025-07-09 18:20:19,170 - root - INFO - step: 21480 loss: 17.1497 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.4715 +[titan] 2025-07-09 18:20:19,170 - root - INFO - lr: 2.7248e-04 gnorm: 0.85 [ 4:45:21<17:23:08] +[titan] 2025-07-09 18:20:23,087 - root - INFO - step: 21485 loss: 16.7543 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.1474 +[titan] 2025-07-09 18:20:23,088 - root - INFO - lr: 2.7246e-04 gnorm: 0.82 [ 4:45:25<17:23:04] +[titan] 2025-07-09 18:20:27,006 - root - INFO - step: 21490 loss: 17.1135 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6629 global_avg_mtp_loss: 14.4506 +[titan] 2025-07-09 18:20:27,006 - root - INFO - lr: 2.7245e-04 gnorm: 0.83 [ 4:45:29<17:23:00] +[titan] 2025-07-09 18:20:30,968 - root - INFO - step: 21495 loss: 17.0722 memory: 44.58GiB(31.99%) tps: 82,716 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.6648 global_avg_mtp_loss: 14.4073 +[titan] 2025-07-09 18:20:30,968 - root - INFO - lr: 2.7244e-04 gnorm: 0.79 [ 4:45:33<17:22:56] +[titan] 2025-07-09 18:20:34,077 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:20:34,874 - root - INFO - step: 21500 loss: 16.6799 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.0984 +[titan] 2025-07-09 18:20:34,875 - root - INFO - lr: 2.7243e-04 gnorm: 0.74 [ 4:45:37<17:22:52] +[titan] 2025-07-09 18:20:38,264 - root - INFO - Dumping profiler traces at step 21504 +[titan] 2025-07-09 18:20:38,295 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:20:39,115 - root - INFO - step: 21505 loss: 16.8662 memory: 44.58GiB(31.99%) tps: 77,269 tflops: 266.67 mfu: 26.96% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.2447 +[titan] 2025-07-09 18:20:39,116 - root - INFO - lr: 2.7241e-04 gnorm: 0.81 [ 4:45:41<17:22:49] +[titan] 2025-07-09 18:20:43,020 - root - INFO - step: 21510 loss: 16.9095 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6355 global_avg_mtp_loss: 14.2739 +[titan] 2025-07-09 18:20:43,020 - root - INFO - lr: 2.7240e-04 gnorm: 0.81 [ 4:45:45<17:22:44] +[titan] 2025-07-09 18:20:46,936 - root - INFO - step: 21515 loss: 16.8576 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6156 global_avg_mtp_loss: 14.2420 +[titan] 2025-07-09 18:20:46,936 - root - INFO - lr: 2.7239e-04 gnorm: 0.77 [ 4:45:49<17:22:40] +[titan] 2025-07-09 18:20:50,860 - root - INFO - step: 21520 loss: 17.0412 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6427 global_avg_mtp_loss: 14.3985 +[titan] 2025-07-09 18:20:50,860 - root - INFO - lr: 2.7237e-04 gnorm: 0.77 [ 4:45:53<17:22:36] +[titan] 2025-07-09 18:20:54,740 - root - INFO - step: 21525 loss: 16.8910 memory: 44.58GiB(31.99%) tps: 84,461 tflops: 291.49 mfu: 29.47% global_avg_ntp_loss: 2.6297 global_avg_mtp_loss: 14.2613 +[titan] 2025-07-09 18:20:54,740 - root - INFO - lr: 2.7236e-04 gnorm: 0.76 [ 4:45:57<17:22:31] +[titan] 2025-07-09 18:20:58,652 - root - INFO - step: 21530 loss: 17.0995 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6712 global_avg_mtp_loss: 14.4283 +[titan] 2025-07-09 18:20:58,652 - root - INFO - lr: 2.7235e-04 gnorm: 0.77 [ 4:46:01<17:22:27] +[titan] 2025-07-09 18:21:02,571 - root - INFO - step: 21535 loss: 16.7699 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5935 global_avg_mtp_loss: 14.1764 +[titan] 2025-07-09 18:21:02,571 - root - INFO - lr: 2.7233e-04 gnorm: 0.80 [ 4:46:05<17:22:23] +[titan] 2025-07-09 18:21:06,486 - root - INFO - step: 21540 loss: 17.1224 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6764 global_avg_mtp_loss: 14.4460 +[titan] 2025-07-09 18:21:06,486 - root - INFO - lr: 2.7232e-04 gnorm: 0.81 [ 4:46:09<17:22:19] +[titan] 2025-07-09 18:21:10,390 - root - INFO - step: 21545 loss: 16.9881 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3388 +[titan] 2025-07-09 18:21:10,390 - root - INFO - lr: 2.7231e-04 gnorm: 0.92 [ 4:46:13<17:22:14] +[titan] 2025-07-09 18:21:13,516 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:21:14,304 - root - INFO - step: 21550 loss: 16.8678 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6334 global_avg_mtp_loss: 14.2345 +[titan] 2025-07-09 18:21:14,305 - root - INFO - lr: 2.7230e-04 gnorm: 0.87 [ 4:46:16<17:22:10] +[titan] 2025-07-09 18:21:18,264 - root - INFO - step: 21555 loss: 16.8932 memory: 44.58GiB(31.99%) tps: 82,759 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.6313 global_avg_mtp_loss: 14.2619 +[titan] 2025-07-09 18:21:18,265 - root - INFO - lr: 2.7228e-04 gnorm: 0.82 [ 4:46:20<17:22:06] +[titan] 2025-07-09 18:21:22,187 - root - INFO - step: 21560 loss: 16.5979 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.0240 +[titan] 2025-07-09 18:21:22,187 - root - INFO - lr: 2.7227e-04 gnorm: 0.88 [ 4:46:24<17:22:02] +[titan] 2025-07-09 18:21:26,109 - root - INFO - step: 21565 loss: 17.0521 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.6636 global_avg_mtp_loss: 14.3885 +[titan] 2025-07-09 18:21:26,110 - root - INFO - lr: 2.7226e-04 gnorm: 0.86 [ 4:46:28<17:21:58] +[titan] 2025-07-09 18:21:30,043 - root - INFO - step: 21570 loss: 17.0052 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6477 global_avg_mtp_loss: 14.3575 +[titan] 2025-07-09 18:21:30,043 - root - INFO - lr: 2.7224e-04 gnorm: 0.77 [ 4:46:32<17:21:53] +[titan] 2025-07-09 18:21:33,964 - root - INFO - step: 21575 loss: 16.9474 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.3173 +[titan] 2025-07-09 18:21:33,964 - root - INFO - lr: 2.7223e-04 gnorm: 0.82 [ 4:46:36<17:21:49] +[titan] 2025-07-09 18:21:37,876 - root - INFO - step: 21580 loss: 16.6223 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.0336 +[titan] 2025-07-09 18:21:37,876 - root - INFO - lr: 2.7222e-04 gnorm: 0.75 [ 4:46:40<17:21:45] +[titan] 2025-07-09 18:21:41,790 - root - INFO - step: 21585 loss: 16.7804 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1839 +[titan] 2025-07-09 18:21:41,790 - root - INFO - lr: 2.7220e-04 gnorm: 0.74 [ 4:46:44<17:21:41] +[titan] 2025-07-09 18:21:45,708 - root - INFO - step: 21590 loss: 17.0792 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6573 global_avg_mtp_loss: 14.4219 +[titan] 2025-07-09 18:21:45,708 - root - INFO - lr: 2.7219e-04 gnorm: 0.77 [ 4:46:48<17:21:37] +[titan] 2025-07-09 18:21:49,604 - root - INFO - step: 21595 loss: 16.9974 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6595 global_avg_mtp_loss: 14.3380 +[titan] 2025-07-09 18:21:49,605 - root - INFO - lr: 2.7218e-04 gnorm: 0.76 [ 4:46:52<17:21:32] +[titan] 2025-07-09 18:21:52,718 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:21:53,503 - root - INFO - step: 21600 loss: 16.9550 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6470 global_avg_mtp_loss: 14.3080 +[titan] 2025-07-09 18:21:53,504 - root - INFO - lr: 2.7217e-04 gnorm: 0.79 [ 4:46:56<17:21:28] +[titan] 2025-07-09 18:21:57,403 - root - INFO - step: 21605 loss: 16.8654 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2481 +[titan] 2025-07-09 18:21:57,404 - root - INFO - lr: 2.7215e-04 gnorm: 0.84 [ 4:47:00<17:21:24] +[titan] 2025-07-09 18:22:01,310 - root - INFO - step: 21610 loss: 16.9724 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3231 +[titan] 2025-07-09 18:22:01,310 - root - INFO - lr: 2.7214e-04 gnorm: 0.77 [ 4:47:03<17:21:19] +[titan] 2025-07-09 18:22:05,215 - root - INFO - step: 21615 loss: 16.7853 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6084 global_avg_mtp_loss: 14.1769 +[titan] 2025-07-09 18:22:05,216 - root - INFO - lr: 2.7213e-04 gnorm: 0.80 [ 4:47:07<17:21:15] +[titan] 2025-07-09 18:22:09,142 - root - INFO - step: 21620 loss: 16.8845 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6318 global_avg_mtp_loss: 14.2527 +[titan] 2025-07-09 18:22:09,142 - root - INFO - lr: 2.7211e-04 gnorm: 0.76 [ 4:47:11<17:21:11] +[titan] 2025-07-09 18:22:13,044 - root - INFO - step: 21625 loss: 16.6944 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.5957 global_avg_mtp_loss: 14.0988 +[titan] 2025-07-09 18:22:13,044 - root - INFO - lr: 2.7210e-04 gnorm: 0.77 [ 4:47:15<17:21:07] +[titan] 2025-07-09 18:22:16,966 - root - INFO - step: 21630 loss: 16.9277 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.2811 +[titan] 2025-07-09 18:22:16,966 - root - INFO - lr: 2.7209e-04 gnorm: 0.75 [ 4:47:19<17:21:02] +[titan] 2025-07-09 18:22:20,865 - root - INFO - step: 21635 loss: 16.9992 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.3462 +[titan] 2025-07-09 18:22:20,866 - root - INFO - lr: 2.7207e-04 gnorm: 0.77 [ 4:47:23<17:20:58] +[titan] 2025-07-09 18:22:24,782 - root - INFO - step: 21640 loss: 16.8186 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6248 global_avg_mtp_loss: 14.1937 +[titan] 2025-07-09 18:22:24,782 - root - INFO - lr: 2.7206e-04 gnorm: 0.84 [ 4:47:27<17:20:54] +[titan] 2025-07-09 18:22:28,684 - root - INFO - step: 21645 loss: 16.9702 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6384 global_avg_mtp_loss: 14.3318 +[titan] 2025-07-09 18:22:28,684 - root - INFO - lr: 2.7205e-04 gnorm: 0.86 [ 4:47:31<17:20:50] +[titan] 2025-07-09 18:22:31,794 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:22:32,582 - root - INFO - step: 21650 loss: 16.9561 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.3194 +[titan] 2025-07-09 18:22:32,583 - root - INFO - lr: 2.7204e-04 gnorm: 0.95 [ 4:47:35<17:20:45] +[titan] 2025-07-09 18:22:36,487 - root - INFO - step: 21655 loss: 16.8887 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2706 +[titan] 2025-07-09 18:22:36,487 - root - INFO - lr: 2.7202e-04 gnorm: 0.79 [ 4:47:39<17:20:41] +[titan] 2025-07-09 18:22:40,387 - root - INFO - step: 21660 loss: 17.0173 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.6526 global_avg_mtp_loss: 14.3646 +[titan] 2025-07-09 18:22:40,388 - root - INFO - lr: 2.7201e-04 gnorm: 0.89 [ 4:47:43<17:20:37] +[titan] 2025-07-09 18:22:44,327 - root - INFO - step: 21665 loss: 16.9465 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.6489 global_avg_mtp_loss: 14.2976 +[titan] 2025-07-09 18:22:44,328 - root - INFO - lr: 2.7200e-04 gnorm: 0.80 [ 4:47:47<17:20:33] +[titan] 2025-07-09 18:22:48,250 - root - INFO - step: 21670 loss: 16.7687 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.1679 +[titan] 2025-07-09 18:22:48,250 - root - INFO - lr: 2.7198e-04 gnorm: 0.80 [ 4:47:50<17:20:28] +[titan] 2025-07-09 18:22:52,235 - root - INFO - step: 21675 loss: 17.1365 memory: 44.58GiB(31.99%) tps: 82,245 tflops: 283.84 mfu: 28.70% global_avg_ntp_loss: 2.6648 global_avg_mtp_loss: 14.4717 +[titan] 2025-07-09 18:22:52,235 - root - INFO - lr: 2.7197e-04 gnorm: 0.87 [ 4:47:54<17:20:24] +[titan] 2025-07-09 18:22:56,136 - root - INFO - step: 21680 loss: 16.9814 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6400 global_avg_mtp_loss: 14.3414 +[titan] 2025-07-09 18:22:56,136 - root - INFO - lr: 2.7196e-04 gnorm: 0.74 [ 4:47:58<17:20:20] +[titan] 2025-07-09 18:23:00,051 - root - INFO - step: 21685 loss: 16.7207 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1265 +[titan] 2025-07-09 18:23:00,051 - root - INFO - lr: 2.7194e-04 gnorm: 0.83 [ 4:48:02<17:20:16] +[titan] 2025-07-09 18:23:03,989 - root - INFO - step: 21690 loss: 16.9486 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.6229 global_avg_mtp_loss: 14.3257 +[titan] 2025-07-09 18:23:03,989 - root - INFO - lr: 2.7193e-04 gnorm: 0.77 [ 4:48:06<17:20:12] +[titan] 2025-07-09 18:23:07,903 - root - INFO - step: 21695 loss: 17.1172 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6705 global_avg_mtp_loss: 14.4467 +[titan] 2025-07-09 18:23:07,903 - root - INFO - lr: 2.7192e-04 gnorm: 0.80 [ 4:48:10<17:20:07] +[titan] 2025-07-09 18:23:11,021 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:23:11,853 - root - INFO - step: 21700 loss: 17.0857 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.6579 global_avg_mtp_loss: 14.4278 +[titan] 2025-07-09 18:23:11,853 - root - INFO - lr: 2.7190e-04 gnorm: 0.77 [ 4:48:14<17:20:03] +[titan] 2025-07-09 18:23:15,789 - root - INFO - step: 21705 loss: 17.1616 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.6857 global_avg_mtp_loss: 14.4759 +[titan] 2025-07-09 18:23:15,790 - root - INFO - lr: 2.7189e-04 gnorm: 0.79 [ 4:48:18<17:19:59] +[titan] 2025-07-09 18:23:19,701 - root - INFO - step: 21710 loss: 16.8459 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2225 +[titan] 2025-07-09 18:23:19,701 - root - INFO - lr: 2.7188e-04 gnorm: 0.78 [ 4:48:22<17:19:55] +[titan] 2025-07-09 18:23:23,623 - root - INFO - step: 21715 loss: 17.0134 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6539 global_avg_mtp_loss: 14.3595 +[titan] 2025-07-09 18:23:23,624 - root - INFO - lr: 2.7187e-04 gnorm: 0.83 [ 4:48:26<17:19:51] +[titan] 2025-07-09 18:23:27,575 - root - INFO - step: 21720 loss: 16.9967 memory: 44.58GiB(31.99%) tps: 82,941 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.6379 global_avg_mtp_loss: 14.3588 +[titan] 2025-07-09 18:23:27,575 - root - INFO - lr: 2.7185e-04 gnorm: 0.79 [ 4:48:30<17:19:47] +[titan] 2025-07-09 18:23:31,502 - root - INFO - step: 21725 loss: 17.0503 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6523 global_avg_mtp_loss: 14.3980 +[titan] 2025-07-09 18:23:31,503 - root - INFO - lr: 2.7184e-04 gnorm: 0.75 [ 4:48:34<17:19:42] +[titan] 2025-07-09 18:23:35,423 - root - INFO - step: 21730 loss: 16.7648 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6019 global_avg_mtp_loss: 14.1629 +[titan] 2025-07-09 18:23:35,423 - root - INFO - lr: 2.7183e-04 gnorm: 0.84 [ 4:48:38<17:19:38] +[titan] 2025-07-09 18:23:39,351 - root - INFO - step: 21735 loss: 16.8838 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6304 global_avg_mtp_loss: 14.2535 +[titan] 2025-07-09 18:23:39,351 - root - INFO - lr: 2.7181e-04 gnorm: 0.78 [ 4:48:42<17:19:34] +[titan] 2025-07-09 18:23:43,255 - root - INFO - step: 21740 loss: 16.8974 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.2444 +[titan] 2025-07-09 18:23:43,255 - root - INFO - lr: 2.7180e-04 gnorm: 0.80 [ 4:48:45<17:19:30] +[titan] 2025-07-09 18:23:47,180 - root - INFO - step: 21745 loss: 16.8397 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2216 +[titan] 2025-07-09 18:23:47,180 - root - INFO - lr: 2.7179e-04 gnorm: 0.71 [ 4:48:49<17:19:25] +[titan] 2025-07-09 18:23:50,302 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:23:51,088 - root - INFO - step: 21750 loss: 16.7777 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.1537 +[titan] 2025-07-09 18:23:51,089 - root - INFO - lr: 2.7177e-04 gnorm: 0.76 [ 4:48:53<17:19:21] +[titan] 2025-07-09 18:23:54,994 - root - INFO - step: 21755 loss: 16.9322 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.2940 +[titan] 2025-07-09 18:23:54,994 - root - INFO - lr: 2.7176e-04 gnorm: 0.82 [ 4:48:57<17:19:17] +[titan] 2025-07-09 18:23:58,910 - root - INFO - step: 21760 loss: 17.1805 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6828 global_avg_mtp_loss: 14.4977 +[titan] 2025-07-09 18:23:58,910 - root - INFO - lr: 2.7175e-04 gnorm: 0.83 [ 4:49:01<17:19:13] +[titan] 2025-07-09 18:24:02,848 - root - INFO - step: 21765 loss: 17.0401 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6509 global_avg_mtp_loss: 14.3892 +[titan] 2025-07-09 18:24:02,848 - root - INFO - lr: 2.7173e-04 gnorm: 0.76 [ 4:49:05<17:19:09] +[titan] 2025-07-09 18:24:06,750 - root - INFO - step: 21770 loss: 16.7985 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.1893 +[titan] 2025-07-09 18:24:06,750 - root - INFO - lr: 2.7172e-04 gnorm: 0.77 [ 4:49:09<17:19:04] +[titan] 2025-07-09 18:24:10,715 - root - INFO - step: 21775 loss: 17.1302 memory: 44.58GiB(31.99%) tps: 82,645 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.6791 global_avg_mtp_loss: 14.4511 +[titan] 2025-07-09 18:24:10,716 - root - INFO - lr: 2.7171e-04 gnorm: 0.77 [ 4:49:13<17:19:00] +[titan] 2025-07-09 18:24:14,629 - root - INFO - step: 21780 loss: 16.9018 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.6309 global_avg_mtp_loss: 14.2709 +[titan] 2025-07-09 18:24:14,630 - root - INFO - lr: 2.7170e-04 gnorm: 0.80 [ 4:49:17<17:18:56] +[titan] 2025-07-09 18:24:18,550 - root - INFO - step: 21785 loss: 17.2212 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6915 global_avg_mtp_loss: 14.5297 +[titan] 2025-07-09 18:24:18,550 - root - INFO - lr: 2.7168e-04 gnorm: 0.81 [ 4:49:21<17:18:52] +[titan] 2025-07-09 18:24:22,449 - root - INFO - step: 21790 loss: 16.9692 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6444 global_avg_mtp_loss: 14.3248 +[titan] 2025-07-09 18:24:22,449 - root - INFO - lr: 2.7167e-04 gnorm: 0.74 [ 4:49:25<17:18:47] +[titan] 2025-07-09 18:24:26,374 - root - INFO - step: 21795 loss: 16.9010 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2690 +[titan] 2025-07-09 18:24:26,374 - root - INFO - lr: 2.7166e-04 gnorm: 0.82 [ 4:49:29<17:18:43] +[titan] 2025-07-09 18:24:29,507 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:24:30,301 - root - INFO - step: 21800 loss: 16.9292 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.2928 +[titan] 2025-07-09 18:24:30,302 - root - INFO - lr: 2.7164e-04 gnorm: 0.77 [ 4:49:32<17:18:39] +[titan] 2025-07-09 18:24:34,212 - root - INFO - step: 21805 loss: 16.9819 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.3327 +[titan] 2025-07-09 18:24:34,212 - root - INFO - lr: 2.7163e-04 gnorm: 0.74 [ 4:49:36<17:18:35] +[titan] 2025-07-09 18:24:38,108 - root - INFO - step: 21810 loss: 16.7595 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1537 +[titan] 2025-07-09 18:24:38,108 - root - INFO - lr: 2.7162e-04 gnorm: 0.73 [ 4:49:40<17:18:30] +[titan] 2025-07-09 18:24:42,047 - root - INFO - step: 21815 loss: 17.1369 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.6680 global_avg_mtp_loss: 14.4689 +[titan] 2025-07-09 18:24:42,047 - root - INFO - lr: 2.7160e-04 gnorm: 0.76 [ 4:49:44<17:18:26] +[titan] 2025-07-09 18:24:45,999 - root - INFO - step: 21820 loss: 16.7095 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.5886 global_avg_mtp_loss: 14.1209 +[titan] 2025-07-09 18:24:45,999 - root - INFO - lr: 2.7159e-04 gnorm: 0.77 [ 4:49:48<17:18:22] +[titan] 2025-07-09 18:24:49,900 - root - INFO - step: 21825 loss: 17.1429 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.4659 +[titan] 2025-07-09 18:24:49,900 - root - INFO - lr: 2.7158e-04 gnorm: 0.85 [ 4:49:52<17:18:18] +[titan] 2025-07-09 18:24:53,822 - root - INFO - step: 21830 loss: 16.9799 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6433 global_avg_mtp_loss: 14.3367 +[titan] 2025-07-09 18:24:53,823 - root - INFO - lr: 2.7156e-04 gnorm: 0.76 [ 4:49:56<17:18:14] +[titan] 2025-07-09 18:24:57,727 - root - INFO - step: 21835 loss: 16.7554 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1580 +[titan] 2025-07-09 18:24:57,727 - root - INFO - lr: 2.7155e-04 gnorm: 0.73 [ 4:50:00<17:18:09] +[titan] 2025-07-09 18:25:01,620 - root - INFO - step: 21840 loss: 16.6771 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.0911 +[titan] 2025-07-09 18:25:01,621 - root - INFO - lr: 2.7154e-04 gnorm: 0.83 [ 4:50:04<17:18:05] +[titan] 2025-07-09 18:25:05,509 - root - INFO - step: 21845 loss: 16.9214 memory: 44.58GiB(31.99%) tps: 84,265 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.6291 global_avg_mtp_loss: 14.2923 +[titan] 2025-07-09 18:25:05,510 - root - INFO - lr: 2.7152e-04 gnorm: 0.77 [ 4:50:08<17:18:01] +[titan] 2025-07-09 18:25:08,613 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:25:09,420 - root - INFO - step: 21850 loss: 16.8857 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 18:25:09,420 - root - INFO - lr: 2.7151e-04 gnorm: 0.84 [ 4:50:12<17:17:57] +[titan] 2025-07-09 18:25:13,325 - root - INFO - step: 21855 loss: 17.0152 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6576 global_avg_mtp_loss: 14.3576 +[titan] 2025-07-09 18:25:13,325 - root - INFO - lr: 2.7150e-04 gnorm: 0.77 [ 4:50:15<17:17:52] +[titan] 2025-07-09 18:25:17,264 - root - INFO - step: 21860 loss: 16.8790 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.6198 global_avg_mtp_loss: 14.2592 +[titan] 2025-07-09 18:25:17,264 - root - INFO - lr: 2.7148e-04 gnorm: 0.79 [ 4:50:19<17:17:48] +[titan] 2025-07-09 18:25:21,166 - root - INFO - step: 21865 loss: 16.9750 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.6411 global_avg_mtp_loss: 14.3338 +[titan] 2025-07-09 18:25:21,166 - root - INFO - lr: 2.7147e-04 gnorm: 0.79 [ 4:50:23<17:17:44] +[titan] 2025-07-09 18:25:25,064 - root - INFO - step: 21870 loss: 16.9609 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.3187 +[titan] 2025-07-09 18:25:25,065 - root - INFO - lr: 2.7146e-04 gnorm: 0.80 [ 4:50:27<17:17:40] +[titan] 2025-07-09 18:25:28,996 - root - INFO - step: 21875 loss: 17.1894 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.6895 global_avg_mtp_loss: 14.4998 +[titan] 2025-07-09 18:25:28,997 - root - INFO - lr: 2.7145e-04 gnorm: 0.80 [ 4:50:31<17:17:35] +[titan] 2025-07-09 18:25:32,918 - root - INFO - step: 21880 loss: 16.8735 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6175 global_avg_mtp_loss: 14.2560 +[titan] 2025-07-09 18:25:32,918 - root - INFO - lr: 2.7143e-04 gnorm: 0.87 [ 4:50:35<17:17:31] +[titan] 2025-07-09 18:25:36,816 - root - INFO - step: 21885 loss: 16.9708 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6496 global_avg_mtp_loss: 14.3212 +[titan] 2025-07-09 18:25:36,817 - root - INFO - lr: 2.7142e-04 gnorm: 0.78 [ 4:50:39<17:17:27] +[titan] 2025-07-09 18:25:40,732 - root - INFO - step: 21890 loss: 16.9438 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6448 global_avg_mtp_loss: 14.2990 +[titan] 2025-07-09 18:25:40,733 - root - INFO - lr: 2.7141e-04 gnorm: 0.78 [ 4:50:43<17:17:23] +[titan] 2025-07-09 18:25:44,741 - root - INFO - step: 21895 loss: 16.9099 memory: 44.58GiB(31.99%) tps: 81,756 tflops: 282.15 mfu: 28.53% global_avg_ntp_loss: 2.6345 global_avg_mtp_loss: 14.2754 +[titan] 2025-07-09 18:25:44,741 - root - INFO - lr: 2.7139e-04 gnorm: 0.80 [ 4:50:47<17:17:19] +[titan] 2025-07-09 18:25:47,871 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:25:48,669 - root - INFO - step: 21900 loss: 16.9294 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.2978 +[titan] 2025-07-09 18:25:48,669 - root - INFO - lr: 2.7138e-04 gnorm: 0.78 [ 4:50:51<17:17:15] +[titan] 2025-07-09 18:25:52,575 - root - INFO - step: 21905 loss: 17.0142 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.3621 +[titan] 2025-07-09 18:25:52,576 - root - INFO - lr: 2.7137e-04 gnorm: 0.75 [ 4:50:55<17:17:10] +[titan] 2025-07-09 18:25:56,494 - root - INFO - step: 21910 loss: 17.2486 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6958 global_avg_mtp_loss: 14.5528 +[titan] 2025-07-09 18:25:56,494 - root - INFO - lr: 2.7135e-04 gnorm: 0.82 [ 4:50:59<17:17:06] +[titan] 2025-07-09 18:26:00,402 - root - INFO - step: 21915 loss: 17.0365 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6562 global_avg_mtp_loss: 14.3803 +[titan] 2025-07-09 18:26:00,403 - root - INFO - lr: 2.7134e-04 gnorm: 0.94 [ 4:51:03<17:17:02] +[titan] 2025-07-09 18:26:04,290 - root - INFO - step: 21920 loss: 17.0729 memory: 44.58GiB(31.99%) tps: 84,288 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.6684 global_avg_mtp_loss: 14.4045 +[titan] 2025-07-09 18:26:04,291 - root - INFO - lr: 2.7133e-04 gnorm: 0.78 [ 4:51:06<17:16:58] +[titan] 2025-07-09 18:26:08,202 - root - INFO - step: 21925 loss: 16.7630 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.1537 +[titan] 2025-07-09 18:26:08,203 - root - INFO - lr: 2.7131e-04 gnorm: 0.79 [ 4:51:10<17:16:53] +[titan] 2025-07-09 18:26:12,106 - root - INFO - step: 21930 loss: 16.7683 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1671 +[titan] 2025-07-09 18:26:12,107 - root - INFO - lr: 2.7130e-04 gnorm: 0.75 [ 4:51:14<17:16:49] +[titan] 2025-07-09 18:26:16,004 - root - INFO - step: 21935 loss: 16.9499 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6333 global_avg_mtp_loss: 14.3167 +[titan] 2025-07-09 18:26:16,004 - root - INFO - lr: 2.7129e-04 gnorm: 0.84 [ 4:51:18<17:16:45] +[titan] 2025-07-09 18:26:19,960 - root - INFO - step: 21940 loss: 16.9218 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.2933 +[titan] 2025-07-09 18:26:19,961 - root - INFO - lr: 2.7127e-04 gnorm: 0.79 [ 4:51:22<17:16:41] +[titan] 2025-07-09 18:26:23,872 - root - INFO - step: 21945 loss: 16.8567 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.2492 +[titan] 2025-07-09 18:26:23,872 - root - INFO - lr: 2.7126e-04 gnorm: 0.74 [ 4:51:26<17:16:36] +[titan] 2025-07-09 18:26:26,997 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:26:27,799 - root - INFO - step: 21950 loss: 16.8395 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6151 global_avg_mtp_loss: 14.2244 +[titan] 2025-07-09 18:26:27,800 - root - INFO - lr: 2.7125e-04 gnorm: 0.75 [ 4:51:30<17:16:32] +[titan] 2025-07-09 18:26:31,743 - root - INFO - step: 21955 loss: 16.9893 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6467 global_avg_mtp_loss: 14.3426 +[titan] 2025-07-09 18:26:31,743 - root - INFO - lr: 2.7123e-04 gnorm: 0.76 [ 4:51:34<17:16:28] +[titan] 2025-07-09 18:26:35,677 - root - INFO - step: 21960 loss: 16.7055 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-09 18:26:35,677 - root - INFO - lr: 2.7122e-04 gnorm: 0.80 [ 4:51:38<17:16:24] +[titan] 2025-07-09 18:26:39,624 - root - INFO - step: 21965 loss: 16.5668 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5749 global_avg_mtp_loss: 13.9919 +[titan] 2025-07-09 18:26:39,624 - root - INFO - lr: 2.7121e-04 gnorm: 0.72 [ 4:51:42<17:16:20] +[titan] 2025-07-09 18:26:43,554 - root - INFO - step: 21970 loss: 16.8157 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2029 +[titan] 2025-07-09 18:26:43,555 - root - INFO - lr: 2.7119e-04 gnorm: 0.74 [ 4:51:46<17:16:16] +[titan] 2025-07-09 18:26:47,462 - root - INFO - step: 21975 loss: 16.6522 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.0704 +[titan] 2025-07-09 18:26:47,462 - root - INFO - lr: 2.7118e-04 gnorm: 0.85 [ 4:51:50<17:16:11] +[titan] 2025-07-09 18:26:51,422 - root - INFO - step: 21980 loss: 16.9164 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2853 +[titan] 2025-07-09 18:26:51,422 - root - INFO - lr: 2.7117e-04 gnorm: 0.77 [ 4:51:54<17:16:07] +[titan] 2025-07-09 18:26:55,348 - root - INFO - step: 21985 loss: 16.7832 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6169 global_avg_mtp_loss: 14.1664 +[titan] 2025-07-09 18:26:55,349 - root - INFO - lr: 2.7115e-04 gnorm: 0.86 [ 4:51:57<17:16:03] +[titan] 2025-07-09 18:26:59,255 - root - INFO - step: 21990 loss: 16.8544 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6150 global_avg_mtp_loss: 14.2394 +[titan] 2025-07-09 18:26:59,255 - root - INFO - lr: 2.7114e-04 gnorm: 0.75 [ 4:52:01<17:15:59] +[titan] 2025-07-09 18:27:03,171 - root - INFO - step: 21995 loss: 16.9316 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6274 global_avg_mtp_loss: 14.3042 +[titan] 2025-07-09 18:27:03,172 - root - INFO - lr: 2.7113e-04 gnorm: 0.75 [ 4:52:05<17:15:55] +[titan] 2025-07-09 18:27:06,300 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:27:07,096 - root - INFO - step: 22000 loss: 16.8827 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.6182 global_avg_mtp_loss: 14.2644 +[titan] 2025-07-09 18:27:07,096 - root - INFO - lr: 2.7112e-04 gnorm: 0.79 [ 4:52:09<17:15:50] +[titan] 2025-07-09 18:27:11,008 - root - INFO - step: 22005 loss: 16.9452 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6407 global_avg_mtp_loss: 14.3045 +[titan] 2025-07-09 18:27:11,008 - root - INFO - lr: 2.7110e-04 gnorm: 0.75 [ 4:52:13<17:15:46] +[titan] 2025-07-09 18:27:14,952 - root - INFO - step: 22010 loss: 17.0796 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6626 global_avg_mtp_loss: 14.4170 +[titan] 2025-07-09 18:27:14,952 - root - INFO - lr: 2.7109e-04 gnorm: 0.79 [ 4:52:17<17:15:42] +[titan] 2025-07-09 18:27:18,934 - root - INFO - step: 22015 loss: 17.2427 memory: 44.58GiB(31.99%) tps: 82,290 tflops: 284.00 mfu: 28.72% global_avg_ntp_loss: 2.6945 global_avg_mtp_loss: 14.5482 +[titan] 2025-07-09 18:27:18,934 - root - INFO - lr: 2.7108e-04 gnorm: 0.79 [ 4:52:21<17:15:38] +[titan] 2025-07-09 18:27:19,878 - root - INFO - Dumping profiler traces at step 22016 +[titan] 2025-07-09 18:27:19,911 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:27:23,068 - root - INFO - step: 22020 loss: 17.0909 memory: 44.58GiB(31.99%) tps: 79,269 tflops: 273.57 mfu: 27.66% global_avg_ntp_loss: 2.6700 global_avg_mtp_loss: 14.4209 +[titan] 2025-07-09 18:27:23,068 - root - INFO - lr: 2.7106e-04 gnorm: 0.78 [ 4:52:25<17:15:35] +[titan] 2025-07-09 18:27:26,983 - root - INFO - step: 22025 loss: 16.9016 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.2603 +[titan] 2025-07-09 18:27:26,983 - root - INFO - lr: 2.7105e-04 gnorm: 0.73 [ 4:52:29<17:15:30] +[titan] 2025-07-09 18:27:30,883 - root - INFO - step: 22030 loss: 16.9450 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.3277 +[titan] 2025-07-09 18:27:30,883 - root - INFO - lr: 2.7104e-04 gnorm: 0.78 [ 4:52:33<17:15:26] +[titan] 2025-07-09 18:27:34,782 - root - INFO - step: 22035 loss: 16.7869 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.1835 +[titan] 2025-07-09 18:27:34,782 - root - INFO - lr: 2.7102e-04 gnorm: 0.78 [ 4:52:37<17:15:22] +[titan] 2025-07-09 18:27:38,680 - root - INFO - step: 22040 loss: 17.0418 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6636 global_avg_mtp_loss: 14.3782 +[titan] 2025-07-09 18:27:38,681 - root - INFO - lr: 2.7101e-04 gnorm: 0.86 [ 4:52:41<17:15:18] +[titan] 2025-07-09 18:27:42,625 - root - INFO - step: 22045 loss: 16.8949 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.6362 global_avg_mtp_loss: 14.2587 +[titan] 2025-07-09 18:27:42,625 - root - INFO - lr: 2.7100e-04 gnorm: 0.78 [ 4:52:45<17:15:13] +[titan] 2025-07-09 18:27:45,731 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:27:46,518 - root - INFO - step: 22050 loss: 16.9055 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2777 +[titan] 2025-07-09 18:27:46,518 - root - INFO - lr: 2.7098e-04 gnorm: 0.76 [ 4:52:49<17:15:09] +[titan] 2025-07-09 18:27:50,426 - root - INFO - step: 22055 loss: 17.0318 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6524 global_avg_mtp_loss: 14.3794 +[titan] 2025-07-09 18:27:50,426 - root - INFO - lr: 2.7097e-04 gnorm: 0.81 [ 4:52:53<17:15:05] +[titan] 2025-07-09 18:27:54,322 - root - INFO - step: 22060 loss: 16.6028 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.0229 +[titan] 2025-07-09 18:27:54,322 - root - INFO - lr: 2.7096e-04 gnorm: 0.82 [ 4:52:56<17:15:01] +[titan] 2025-07-09 18:27:58,264 - root - INFO - step: 22065 loss: 17.4043 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.7315 global_avg_mtp_loss: 14.6728 +[titan] 2025-07-09 18:27:58,264 - root - INFO - lr: 2.7094e-04 gnorm: 0.81 [ 4:53:00<17:14:56] +[titan] 2025-07-09 18:28:02,165 - root - INFO - step: 22070 loss: 16.9311 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.3026 +[titan] 2025-07-09 18:28:02,166 - root - INFO - lr: 2.7093e-04 gnorm: 0.84 [ 4:53:04<17:14:52] +[titan] 2025-07-09 18:28:06,083 - root - INFO - step: 22075 loss: 17.3184 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.7099 global_avg_mtp_loss: 14.6086 +[titan] 2025-07-09 18:28:06,083 - root - INFO - lr: 2.7092e-04 gnorm: 0.84 [ 4:53:08<17:14:48] +[titan] 2025-07-09 18:28:10,014 - root - INFO - step: 22080 loss: 16.8689 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.6117 global_avg_mtp_loss: 14.2572 +[titan] 2025-07-09 18:28:10,014 - root - INFO - lr: 2.7090e-04 gnorm: 0.78 [ 4:53:12<17:14:44] +[titan] 2025-07-09 18:28:13,923 - root - INFO - step: 22085 loss: 16.7384 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.1313 +[titan] 2025-07-09 18:28:13,923 - root - INFO - lr: 2.7089e-04 gnorm: 0.73 [ 4:53:16<17:14:39] +[titan] 2025-07-09 18:28:17,835 - root - INFO - step: 22090 loss: 17.0374 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6588 global_avg_mtp_loss: 14.3786 +[titan] 2025-07-09 18:28:17,835 - root - INFO - lr: 2.7088e-04 gnorm: 0.74 [ 4:53:20<17:14:35] +[titan] 2025-07-09 18:28:21,743 - root - INFO - step: 22095 loss: 17.0742 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6617 global_avg_mtp_loss: 14.4126 +[titan] 2025-07-09 18:28:21,743 - root - INFO - lr: 2.7086e-04 gnorm: 0.77 [ 4:53:24<17:14:31] +[titan] 2025-07-09 18:28:24,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:28:25,672 - root - INFO - step: 22100 loss: 16.7114 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5927 global_avg_mtp_loss: 14.1187 +[titan] 2025-07-09 18:28:25,672 - root - INFO - lr: 2.7085e-04 gnorm: 0.81 [ 4:53:28<17:14:27] +[titan] 2025-07-09 18:28:29,577 - root - INFO - step: 22105 loss: 16.9680 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.3292 +[titan] 2025-07-09 18:28:29,577 - root - INFO - lr: 2.7084e-04 gnorm: 0.77 [ 4:53:32<17:14:23] +[titan] 2025-07-09 18:28:33,486 - root - INFO - step: 22110 loss: 17.1614 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6722 global_avg_mtp_loss: 14.4891 +[titan] 2025-07-09 18:28:33,487 - root - INFO - lr: 2.7082e-04 gnorm: 0.74 [ 4:53:36<17:14:18] +[titan] 2025-07-09 18:28:37,421 - root - INFO - step: 22115 loss: 16.8422 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.6077 global_avg_mtp_loss: 14.2345 +[titan] 2025-07-09 18:28:37,421 - root - INFO - lr: 2.7081e-04 gnorm: 0.80 [ 4:53:40<17:14:14] +[titan] 2025-07-09 18:28:41,360 - root - INFO - step: 22120 loss: 16.8857 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.6195 global_avg_mtp_loss: 14.2662 +[titan] 2025-07-09 18:28:41,360 - root - INFO - lr: 2.7080e-04 gnorm: 0.81 [ 4:53:43<17:14:10] +[titan] 2025-07-09 18:28:45,274 - root - INFO - step: 22125 loss: 17.0737 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6585 global_avg_mtp_loss: 14.4152 +[titan] 2025-07-09 18:28:45,275 - root - INFO - lr: 2.7078e-04 gnorm: 0.81 [ 4:53:47<17:14:06] +[titan] 2025-07-09 18:28:49,224 - root - INFO - step: 22130 loss: 16.8447 memory: 44.58GiB(31.99%) tps: 82,966 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2178 +[titan] 2025-07-09 18:28:49,225 - root - INFO - lr: 2.7077e-04 gnorm: 0.81 [ 4:53:51<17:14:02] +[titan] 2025-07-09 18:28:53,125 - root - INFO - step: 22135 loss: 17.1098 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6744 global_avg_mtp_loss: 14.4354 +[titan] 2025-07-09 18:28:53,125 - root - INFO - lr: 2.7076e-04 gnorm: 0.79 [ 4:53:55<17:13:57] +[titan] 2025-07-09 18:28:57,080 - root - INFO - step: 22140 loss: 16.9745 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.3413 +[titan] 2025-07-09 18:28:57,081 - root - INFO - lr: 2.7074e-04 gnorm: 0.78 [ 4:53:59<17:13:53] +[titan] 2025-07-09 18:29:00,992 - root - INFO - step: 22145 loss: 16.6922 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1083 +[titan] 2025-07-09 18:29:00,992 - root - INFO - lr: 2.7073e-04 gnorm: 0.79 [ 4:54:03<17:13:49] +[titan] 2025-07-09 18:29:04,106 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:29:04,907 - root - INFO - step: 22150 loss: 16.9291 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6356 global_avg_mtp_loss: 14.2935 +[titan] 2025-07-09 18:29:04,908 - root - INFO - lr: 2.7072e-04 gnorm: 0.84 [ 4:54:07<17:13:45] +[titan] 2025-07-09 18:29:08,827 - root - INFO - step: 22155 loss: 17.0538 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6502 global_avg_mtp_loss: 14.4036 +[titan] 2025-07-09 18:29:08,828 - root - INFO - lr: 2.7070e-04 gnorm: 0.78 [ 4:54:11<17:13:41] +[titan] 2025-07-09 18:29:12,756 - root - INFO - step: 22160 loss: 16.8816 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6226 global_avg_mtp_loss: 14.2590 +[titan] 2025-07-09 18:29:12,757 - root - INFO - lr: 2.7069e-04 gnorm: 0.82 [ 4:54:15<17:13:36] +[titan] 2025-07-09 18:29:16,665 - root - INFO - step: 22165 loss: 16.7107 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.1154 +[titan] 2025-07-09 18:29:16,665 - root - INFO - lr: 2.7068e-04 gnorm: 0.73 [ 4:54:19<17:13:32] +[titan] 2025-07-09 18:29:20,643 - root - INFO - step: 22170 loss: 17.1617 memory: 44.58GiB(31.99%) tps: 82,370 tflops: 284.27 mfu: 28.74% global_avg_ntp_loss: 2.6743 global_avg_mtp_loss: 14.4874 +[titan] 2025-07-09 18:29:20,643 - root - INFO - lr: 2.7066e-04 gnorm: 0.75 [ 4:54:23<17:13:28] +[titan] 2025-07-09 18:29:24,572 - root - INFO - step: 22175 loss: 17.0793 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4037 +[titan] 2025-07-09 18:29:24,573 - root - INFO - lr: 2.7065e-04 gnorm: 0.79 [ 4:54:27<17:13:24] +[titan] 2025-07-09 18:29:28,504 - root - INFO - step: 22180 loss: 16.7351 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.1369 +[titan] 2025-07-09 18:29:28,504 - root - INFO - lr: 2.7064e-04 gnorm: 0.80 [ 4:54:31<17:13:20] +[titan] 2025-07-09 18:29:32,411 - root - INFO - step: 22185 loss: 16.8647 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2422 +[titan] 2025-07-09 18:29:32,412 - root - INFO - lr: 2.7062e-04 gnorm: 0.86 [ 4:54:35<17:13:16] +[titan] 2025-07-09 18:29:36,336 - root - INFO - step: 22190 loss: 17.0038 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6460 global_avg_mtp_loss: 14.3578 +[titan] 2025-07-09 18:29:36,337 - root - INFO - lr: 2.7061e-04 gnorm: 0.82 [ 4:54:38<17:13:11] +[titan] 2025-07-09 18:29:40,232 - root - INFO - step: 22195 loss: 16.9389 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.3192 +[titan] 2025-07-09 18:29:40,232 - root - INFO - lr: 2.7060e-04 gnorm: 0.89 [ 4:54:42<17:13:07] +[titan] 2025-07-09 18:29:43,374 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:29:44,159 - root - INFO - step: 22200 loss: 16.9621 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6430 global_avg_mtp_loss: 14.3191 +[titan] 2025-07-09 18:29:44,159 - root - INFO - lr: 2.7058e-04 gnorm: 0.86 [ 4:54:46<17:13:03] +[titan] 2025-07-09 18:29:48,075 - root - INFO - step: 22205 loss: 16.9185 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6396 global_avg_mtp_loss: 14.2789 +[titan] 2025-07-09 18:29:48,075 - root - INFO - lr: 2.7057e-04 gnorm: 0.82 [ 4:54:50<17:12:59] +[titan] 2025-07-09 18:29:51,977 - root - INFO - step: 22210 loss: 16.9205 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6267 global_avg_mtp_loss: 14.2938 +[titan] 2025-07-09 18:29:51,977 - root - INFO - lr: 2.7056e-04 gnorm: 0.77 [ 4:54:54<17:12:54] +[titan] 2025-07-09 18:29:55,886 - root - INFO - step: 22215 loss: 17.0396 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6548 global_avg_mtp_loss: 14.3849 +[titan] 2025-07-09 18:29:55,887 - root - INFO - lr: 2.7054e-04 gnorm: 0.82 [ 4:54:58<17:12:50] +[titan] 2025-07-09 18:29:59,812 - root - INFO - step: 22220 loss: 16.7625 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.1600 +[titan] 2025-07-09 18:29:59,812 - root - INFO - lr: 2.7053e-04 gnorm: 0.80 [ 4:55:02<17:12:46] +[titan] 2025-07-09 18:30:03,723 - root - INFO - step: 22225 loss: 16.9110 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2869 +[titan] 2025-07-09 18:30:03,724 - root - INFO - lr: 2.7052e-04 gnorm: 0.77 [ 4:55:06<17:12:42] +[titan] 2025-07-09 18:30:07,673 - root - INFO - step: 22230 loss: 17.0387 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.6549 global_avg_mtp_loss: 14.3838 +[titan] 2025-07-09 18:30:07,673 - root - INFO - lr: 2.7050e-04 gnorm: 0.79 [ 4:55:10<17:12:38] +[titan] 2025-07-09 18:30:11,592 - root - INFO - step: 22235 loss: 16.4544 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9083 +[titan] 2025-07-09 18:30:11,592 - root - INFO - lr: 2.7049e-04 gnorm: 0.78 [ 4:55:14<17:12:33] +[titan] 2025-07-09 18:30:15,506 - root - INFO - step: 22240 loss: 16.6980 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.1003 +[titan] 2025-07-09 18:30:15,507 - root - INFO - lr: 2.7048e-04 gnorm: 0.79 [ 4:55:18<17:12:29] +[titan] 2025-07-09 18:30:19,402 - root - INFO - step: 22245 loss: 17.0333 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.6504 global_avg_mtp_loss: 14.3829 +[titan] 2025-07-09 18:30:19,402 - root - INFO - lr: 2.7046e-04 gnorm: 0.74 [ 4:55:22<17:12:25] +[titan] 2025-07-09 18:30:22,514 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:30:23,311 - root - INFO - step: 22250 loss: 16.7213 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.1274 +[titan] 2025-07-09 18:30:23,311 - root - INFO - lr: 2.7045e-04 gnorm: 0.81 [ 4:55:25<17:12:21] +[titan] 2025-07-09 18:30:27,210 - root - INFO - step: 22255 loss: 16.6244 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5807 global_avg_mtp_loss: 14.0436 +[titan] 2025-07-09 18:30:27,210 - root - INFO - lr: 2.7044e-04 gnorm: 0.79 [ 4:55:29<17:12:16] +[titan] 2025-07-09 18:30:31,118 - root - INFO - step: 22260 loss: 16.8919 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6266 global_avg_mtp_loss: 14.2653 +[titan] 2025-07-09 18:30:31,119 - root - INFO - lr: 2.7042e-04 gnorm: 0.81 [ 4:55:33<17:12:12] +[titan] 2025-07-09 18:30:35,069 - root - INFO - step: 22265 loss: 16.7513 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6175 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 18:30:35,069 - root - INFO - lr: 2.7041e-04 gnorm: 0.76 [ 4:55:37<17:12:08] +[titan] 2025-07-09 18:30:38,983 - root - INFO - step: 22270 loss: 17.1146 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.4546 +[titan] 2025-07-09 18:30:38,983 - root - INFO - lr: 2.7040e-04 gnorm: 0.82 [ 4:55:41<17:12:04] +[titan] 2025-07-09 18:30:42,907 - root - INFO - step: 22275 loss: 16.8300 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.2247 +[titan] 2025-07-09 18:30:42,908 - root - INFO - lr: 2.7038e-04 gnorm: 0.81 [ 4:55:45<17:12:00] +[titan] 2025-07-09 18:30:46,864 - root - INFO - step: 22280 loss: 16.8571 memory: 44.58GiB(31.99%) tps: 82,821 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.6211 global_avg_mtp_loss: 14.2360 +[titan] 2025-07-09 18:30:46,865 - root - INFO - lr: 2.7037e-04 gnorm: 0.83 [ 4:55:49<17:11:56] +[titan] 2025-07-09 18:30:50,789 - root - INFO - step: 22285 loss: 16.7538 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.1509 +[titan] 2025-07-09 18:30:50,790 - root - INFO - lr: 2.7036e-04 gnorm: 0.79 [ 4:55:53<17:11:51] +[titan] 2025-07-09 18:30:54,701 - root - INFO - step: 22290 loss: 16.8109 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6119 global_avg_mtp_loss: 14.1990 +[titan] 2025-07-09 18:30:54,701 - root - INFO - lr: 2.7034e-04 gnorm: 0.81 [ 4:55:57<17:11:47] +[titan] 2025-07-09 18:30:58,671 - root - INFO - step: 22295 loss: 16.8917 memory: 44.58GiB(31.99%) tps: 82,551 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.6313 global_avg_mtp_loss: 14.2604 +[titan] 2025-07-09 18:30:58,671 - root - INFO - lr: 2.7033e-04 gnorm: 0.74 [ 4:56:01<17:11:43] +[titan] 2025-07-09 18:31:01,811 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:31:02,608 - root - INFO - step: 22300 loss: 16.7564 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1617 +[titan] 2025-07-09 18:31:02,609 - root - INFO - lr: 2.7032e-04 gnorm: 0.83 [ 4:56:05<17:11:39] +[titan] 2025-07-09 18:31:06,529 - root - INFO - step: 22305 loss: 16.8817 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.2532 +[titan] 2025-07-09 18:31:06,529 - root - INFO - lr: 2.7030e-04 gnorm: 0.82 [ 4:56:09<17:11:35] +[titan] 2025-07-09 18:31:10,444 - root - INFO - step: 22310 loss: 17.2319 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6862 global_avg_mtp_loss: 14.5456 +[titan] 2025-07-09 18:31:10,444 - root - INFO - lr: 2.7029e-04 gnorm: 0.88 [ 4:56:13<17:11:31] +[titan] 2025-07-09 18:31:14,359 - root - INFO - step: 22315 loss: 17.1837 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6838 global_avg_mtp_loss: 14.4999 +[titan] 2025-07-09 18:31:14,359 - root - INFO - lr: 2.7028e-04 gnorm: 0.85 [ 4:56:16<17:11:26] +[titan] 2025-07-09 18:31:18,302 - root - INFO - step: 22320 loss: 16.9708 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6551 global_avg_mtp_loss: 14.3157 +[titan] 2025-07-09 18:31:18,302 - root - INFO - lr: 2.7026e-04 gnorm: 0.78 [ 4:56:20<17:11:22] +[titan] 2025-07-09 18:31:22,207 - root - INFO - step: 22325 loss: 17.0873 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4213 +[titan] 2025-07-09 18:31:22,207 - root - INFO - lr: 2.7025e-04 gnorm: 0.80 [ 4:56:24<17:11:18] +[titan] 2025-07-09 18:31:26,125 - root - INFO - step: 22330 loss: 16.8235 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.6055 global_avg_mtp_loss: 14.2179 +[titan] 2025-07-09 18:31:26,125 - root - INFO - lr: 2.7024e-04 gnorm: 0.78 [ 4:56:28<17:11:14] +[titan] 2025-07-09 18:31:30,040 - root - INFO - step: 22335 loss: 16.6284 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0528 +[titan] 2025-07-09 18:31:30,040 - root - INFO - lr: 2.7022e-04 gnorm: 0.79 [ 4:56:32<17:11:10] +[titan] 2025-07-09 18:31:33,987 - root - INFO - step: 22340 loss: 17.1334 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6578 global_avg_mtp_loss: 14.4756 +[titan] 2025-07-09 18:31:33,987 - root - INFO - lr: 2.7021e-04 gnorm: 0.80 [ 4:56:36<17:11:05] +[titan] 2025-07-09 18:31:37,901 - root - INFO - step: 22345 loss: 17.0985 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.6619 global_avg_mtp_loss: 14.4365 +[titan] 2025-07-09 18:31:37,901 - root - INFO - lr: 2.7020e-04 gnorm: 0.79 [ 4:56:40<17:11:01] +[titan] 2025-07-09 18:31:41,026 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:31:41,826 - root - INFO - step: 22350 loss: 17.0204 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6540 global_avg_mtp_loss: 14.3664 +[titan] 2025-07-09 18:31:41,826 - root - INFO - lr: 2.7018e-04 gnorm: 0.77 [ 4:56:44<17:10:57] +[titan] 2025-07-09 18:31:45,772 - root - INFO - step: 22355 loss: 17.0862 memory: 44.58GiB(31.99%) tps: 83,043 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.6530 global_avg_mtp_loss: 14.4331 +[titan] 2025-07-09 18:31:45,772 - root - INFO - lr: 2.7017e-04 gnorm: 0.81 [ 4:56:48<17:10:53] +[titan] 2025-07-09 18:31:49,684 - root - INFO - step: 22360 loss: 16.9509 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3090 +[titan] 2025-07-09 18:31:49,684 - root - INFO - lr: 2.7015e-04 gnorm: 0.80 [ 4:56:52<17:10:49] +[titan] 2025-07-09 18:31:53,632 - root - INFO - step: 22365 loss: 16.9636 memory: 44.58GiB(31.99%) tps: 83,007 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.6384 global_avg_mtp_loss: 14.3252 +[titan] 2025-07-09 18:31:53,632 - root - INFO - lr: 2.7014e-04 gnorm: 0.80 [ 4:56:56<17:10:45] +[titan] 2025-07-09 18:31:57,530 - root - INFO - step: 22370 loss: 16.9151 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6228 global_avg_mtp_loss: 14.2923 +[titan] 2025-07-09 18:31:57,531 - root - INFO - lr: 2.7013e-04 gnorm: 0.80 [ 4:57:00<17:10:40] +[titan] 2025-07-09 18:32:01,438 - root - INFO - step: 22375 loss: 16.8899 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2641 +[titan] 2025-07-09 18:32:01,438 - root - INFO - lr: 2.7011e-04 gnorm: 0.82 [ 4:57:04<17:10:36] +[titan] 2025-07-09 18:32:05,336 - root - INFO - step: 22380 loss: 16.9962 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.3560 +[titan] 2025-07-09 18:32:05,337 - root - INFO - lr: 2.7010e-04 gnorm: 0.79 [ 4:57:07<17:10:32] +[titan] 2025-07-09 18:32:09,259 - root - INFO - step: 22385 loss: 16.9488 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.3077 +[titan] 2025-07-09 18:32:09,260 - root - INFO - lr: 2.7009e-04 gnorm: 0.74 [ 4:57:11<17:10:28] +[titan] 2025-07-09 18:32:13,161 - root - INFO - step: 22390 loss: 16.9283 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2963 +[titan] 2025-07-09 18:32:13,161 - root - INFO - lr: 2.7007e-04 gnorm: 0.81 [ 4:57:15<17:10:23] +[titan] 2025-07-09 18:32:17,152 - root - INFO - step: 22395 loss: 17.0128 memory: 44.58GiB(31.99%) tps: 82,113 tflops: 283.39 mfu: 28.65% global_avg_ntp_loss: 2.6504 global_avg_mtp_loss: 14.3624 +[titan] 2025-07-09 18:32:17,152 - root - INFO - lr: 2.7006e-04 gnorm: 0.73 [ 4:57:19<17:10:19] +[titan] 2025-07-09 18:32:20,254 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:32:21,044 - root - INFO - step: 22400 loss: 16.9236 memory: 44.58GiB(31.99%) tps: 84,207 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.2904 +[titan] 2025-07-09 18:32:21,044 - root - INFO - lr: 2.7005e-04 gnorm: 0.80 [ 4:57:23<17:10:15] +[titan] 2025-07-09 18:32:24,953 - root - INFO - step: 22405 loss: 16.9980 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.3488 +[titan] 2025-07-09 18:32:24,954 - root - INFO - lr: 2.7003e-04 gnorm: 0.81 [ 4:57:27<17:10:11] +[titan] 2025-07-09 18:32:28,914 - root - INFO - step: 22410 loss: 16.9047 memory: 44.58GiB(31.99%) tps: 82,749 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.6282 global_avg_mtp_loss: 14.2765 +[titan] 2025-07-09 18:32:28,914 - root - INFO - lr: 2.7002e-04 gnorm: 0.78 [ 4:57:31<17:10:07] +[titan] 2025-07-09 18:32:32,815 - root - INFO - step: 22415 loss: 17.1800 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6894 global_avg_mtp_loss: 14.4905 +[titan] 2025-07-09 18:32:32,815 - root - INFO - lr: 2.7001e-04 gnorm: 0.88 [ 4:57:35<17:10:02] +[titan] 2025-07-09 18:32:36,710 - root - INFO - step: 22420 loss: 16.5067 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9612 +[titan] 2025-07-09 18:32:36,711 - root - INFO - lr: 2.6999e-04 gnorm: 0.78 [ 4:57:39<17:09:58] +[titan] 2025-07-09 18:32:40,603 - root - INFO - step: 22425 loss: 16.8061 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.1990 +[titan] 2025-07-09 18:32:40,603 - root - INFO - lr: 2.6998e-04 gnorm: 0.80 [ 4:57:43<17:09:54] +[titan] 2025-07-09 18:32:44,515 - root - INFO - step: 22430 loss: 16.6922 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1075 +[titan] 2025-07-09 18:32:44,515 - root - INFO - lr: 2.6997e-04 gnorm: 0.80 [ 4:57:47<17:09:50] +[titan] 2025-07-09 18:32:48,431 - root - INFO - step: 22435 loss: 17.0693 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6723 global_avg_mtp_loss: 14.3969 +[titan] 2025-07-09 18:32:48,431 - root - INFO - lr: 2.6995e-04 gnorm: 0.77 [ 4:57:51<17:09:45] +[titan] 2025-07-09 18:32:52,333 - root - INFO - step: 22440 loss: 16.8259 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 18:32:52,333 - root - INFO - lr: 2.6994e-04 gnorm: 0.79 [ 4:57:54<17:09:41] +[titan] 2025-07-09 18:32:56,305 - root - INFO - step: 22445 loss: 17.0363 memory: 44.58GiB(31.99%) tps: 82,509 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.6647 global_avg_mtp_loss: 14.3716 +[titan] 2025-07-09 18:32:56,305 - root - INFO - lr: 2.6993e-04 gnorm: 0.79 [ 4:57:58<17:09:37] +[titan] 2025-07-09 18:32:59,428 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:33:00,229 - root - INFO - step: 22450 loss: 16.8330 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6127 global_avg_mtp_loss: 14.2203 +[titan] 2025-07-09 18:33:00,229 - root - INFO - lr: 2.6991e-04 gnorm: 0.84 [ 4:58:02<17:09:33] +[titan] 2025-07-09 18:33:04,128 - root - INFO - step: 22455 loss: 17.0967 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6621 global_avg_mtp_loss: 14.4346 +[titan] 2025-07-09 18:33:04,128 - root - INFO - lr: 2.6990e-04 gnorm: 0.84 [ 4:58:06<17:09:29] +[titan] 2025-07-09 18:33:08,065 - root - INFO - step: 22460 loss: 17.1655 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.6703 global_avg_mtp_loss: 14.4952 +[titan] 2025-07-09 18:33:08,065 - root - INFO - lr: 2.6989e-04 gnorm: 0.75 [ 4:58:10<17:09:25] +[titan] 2025-07-09 18:33:11,975 - root - INFO - step: 22465 loss: 16.8869 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6224 global_avg_mtp_loss: 14.2645 +[titan] 2025-07-09 18:33:11,975 - root - INFO - lr: 2.6987e-04 gnorm: 0.78 [ 4:58:14<17:09:20] +[titan] 2025-07-09 18:33:15,890 - root - INFO - step: 22470 loss: 16.9322 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.2971 +[titan] 2025-07-09 18:33:15,891 - root - INFO - lr: 2.6986e-04 gnorm: 1.10 [ 4:58:18<17:09:16] +[titan] 2025-07-09 18:33:19,797 - root - INFO - step: 22475 loss: 16.7782 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.1738 +[titan] 2025-07-09 18:33:19,797 - root - INFO - lr: 2.6985e-04 gnorm: 0.80 [ 4:58:22<17:09:12] +[titan] 2025-07-09 18:33:23,718 - root - INFO - step: 22480 loss: 17.0676 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.4038 +[titan] 2025-07-09 18:33:23,718 - root - INFO - lr: 2.6983e-04 gnorm: 0.78 [ 4:58:26<17:09:08] +[titan] 2025-07-09 18:33:27,634 - root - INFO - step: 22485 loss: 16.9022 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.2729 +[titan] 2025-07-09 18:33:27,634 - root - INFO - lr: 2.6982e-04 gnorm: 0.75 [ 4:58:30<17:09:03] +[titan] 2025-07-09 18:33:31,535 - root - INFO - step: 22490 loss: 17.0166 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.3565 +[titan] 2025-07-09 18:33:31,535 - root - INFO - lr: 2.6980e-04 gnorm: 0.79 [ 4:58:34<17:08:59] +[titan] 2025-07-09 18:33:35,439 - root - INFO - step: 22495 loss: 16.7418 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1561 +[titan] 2025-07-09 18:33:35,439 - root - INFO - lr: 2.6979e-04 gnorm: 0.81 [ 4:58:38<17:08:55] +[titan] 2025-07-09 18:33:38,551 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:33:39,339 - root - INFO - step: 22500 loss: 17.1514 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6736 global_avg_mtp_loss: 14.4778 +[titan] 2025-07-09 18:33:39,339 - root - INFO - lr: 2.6978e-04 gnorm: 0.79 [ 4:58:41<17:08:51] +[titan] 2025-07-09 18:33:43,289 - root - INFO - step: 22505 loss: 16.8605 memory: 44.58GiB(31.99%) tps: 82,968 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2455 +[titan] 2025-07-09 18:33:43,289 - root - INFO - lr: 2.6976e-04 gnorm: 0.74 [ 4:58:45<17:08:47] +[titan] 2025-07-09 18:33:47,198 - root - INFO - step: 22510 loss: 17.0121 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6580 global_avg_mtp_loss: 14.3542 +[titan] 2025-07-09 18:33:47,199 - root - INFO - lr: 2.6975e-04 gnorm: 0.78 [ 4:58:49<17:08:42] +[titan] 2025-07-09 18:33:51,106 - root - INFO - step: 22515 loss: 16.8725 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.2526 +[titan] 2025-07-09 18:33:51,107 - root - INFO - lr: 2.6974e-04 gnorm: 0.76 [ 4:58:53<17:08:38] +[titan] 2025-07-09 18:33:55,035 - root - INFO - step: 22520 loss: 16.9165 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6193 global_avg_mtp_loss: 14.2972 +[titan] 2025-07-09 18:33:55,035 - root - INFO - lr: 2.6972e-04 gnorm: 0.84 [ 4:58:57<17:08:34] +[titan] 2025-07-09 18:33:59,048 - root - INFO - step: 22525 loss: 16.8745 memory: 44.58GiB(31.99%) tps: 81,655 tflops: 281.81 mfu: 28.49% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2564 +[titan] 2025-07-09 18:33:59,049 - root - INFO - lr: 2.6971e-04 gnorm: 0.77 [ 4:59:01<17:08:30] +[titan] 2025-07-09 18:34:01,540 - root - INFO - Dumping profiler traces at step 22528 +[titan] 2025-07-09 18:34:01,578 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-09 18:34:03,143 - root - INFO - step: 22530 loss: 16.9525 memory: 44.58GiB(31.99%) tps: 80,039 tflops: 276.23 mfu: 27.93% global_avg_ntp_loss: 2.6309 global_avg_mtp_loss: 14.3216 +[titan] 2025-07-09 18:34:03,143 - root - INFO - lr: 2.6970e-04 gnorm: 0.81 [ 4:59:05<17:08:26] +[titan] 2025-07-09 18:34:07,062 - root - INFO - step: 22535 loss: 16.8953 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6293 global_avg_mtp_loss: 14.2661 +[titan] 2025-07-09 18:34:07,062 - root - INFO - lr: 2.6968e-04 gnorm: 0.83 [ 4:59:09<17:08:22] +[titan] 2025-07-09 18:34:10,959 - root - INFO - step: 22540 loss: 17.0424 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.4003 +[titan] 2025-07-09 18:34:10,960 - root - INFO - lr: 2.6967e-04 gnorm: 0.82 [ 4:59:13<17:08:18] +[titan] 2025-07-09 18:34:14,891 - root - INFO - step: 22545 loss: 16.8463 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.2248 +[titan] 2025-07-09 18:34:14,891 - root - INFO - lr: 2.6966e-04 gnorm: 0.79 [ 4:59:17<17:08:14] +[titan] 2025-07-09 18:34:18,040 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:34:18,831 - root - INFO - step: 22550 loss: 16.8270 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.2220 +[titan] 2025-07-09 18:34:18,831 - root - INFO - lr: 2.6964e-04 gnorm: 0.92 [ 4:59:21<17:08:10] +[titan] 2025-07-09 18:34:22,744 - root - INFO - step: 22555 loss: 17.0531 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.3904 +[titan] 2025-07-09 18:34:22,744 - root - INFO - lr: 2.6963e-04 gnorm: 0.82 [ 4:59:25<17:08:05] +[titan] 2025-07-09 18:34:26,719 - root - INFO - step: 22560 loss: 16.6439 memory: 44.58GiB(31.99%) tps: 82,450 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.0592 +[titan] 2025-07-09 18:34:26,719 - root - INFO - lr: 2.6962e-04 gnorm: 0.78 [ 4:59:29<17:08:01] +[titan] 2025-07-09 18:34:30,653 - root - INFO - step: 22565 loss: 17.0052 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3593 +[titan] 2025-07-09 18:34:30,653 - root - INFO - lr: 2.6960e-04 gnorm: 0.79 [ 4:59:33<17:07:57] +[titan] 2025-07-09 18:34:34,585 - root - INFO - step: 22570 loss: 16.8146 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6263 global_avg_mtp_loss: 14.1883 +[titan] 2025-07-09 18:34:34,586 - root - INFO - lr: 2.6959e-04 gnorm: 0.80 [ 4:59:37<17:07:53] +[titan] 2025-07-09 18:34:38,529 - root - INFO - step: 22575 loss: 16.9848 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.6464 global_avg_mtp_loss: 14.3384 +[titan] 2025-07-09 18:34:38,529 - root - INFO - lr: 2.6957e-04 gnorm: 0.74 [ 4:59:41<17:07:49] +[titan] 2025-07-09 18:34:42,459 - root - INFO - step: 22580 loss: 16.9418 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6487 global_avg_mtp_loss: 14.2930 +[titan] 2025-07-09 18:34:42,459 - root - INFO - lr: 2.6956e-04 gnorm: 0.87 [ 4:59:45<17:07:45] +[titan] 2025-07-09 18:34:46,384 - root - INFO - step: 22585 loss: 17.0624 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3986 +[titan] 2025-07-09 18:34:46,384 - root - INFO - lr: 2.6955e-04 gnorm: 0.85 [ 4:59:48<17:07:41] +[titan] 2025-07-09 18:34:50,314 - root - INFO - step: 22590 loss: 16.9631 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6302 global_avg_mtp_loss: 14.3329 +[titan] 2025-07-09 18:34:50,314 - root - INFO - lr: 2.6953e-04 gnorm: 0.76 [ 4:59:52<17:07:36] +[titan] 2025-07-09 18:34:54,214 - root - INFO - step: 22595 loss: 17.1244 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6739 global_avg_mtp_loss: 14.4505 +[titan] 2025-07-09 18:34:54,214 - root - INFO - lr: 2.6952e-04 gnorm: 0.78 [ 4:59:56<17:07:32] +[titan] 2025-07-09 18:34:57,329 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:34:58,113 - root - INFO - step: 22600 loss: 16.8468 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2386 +[titan] 2025-07-09 18:34:58,113 - root - INFO - lr: 2.6951e-04 gnorm: 0.74 [ 5:00:00<17:07:28] +[titan] 2025-07-09 18:35:02,023 - root - INFO - step: 22605 loss: 16.7752 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6157 global_avg_mtp_loss: 14.1595 +[titan] 2025-07-09 18:35:02,023 - root - INFO - lr: 2.6949e-04 gnorm: 0.71 [ 5:00:04<17:07:24] +[titan] 2025-07-09 18:35:05,937 - root - INFO - step: 22610 loss: 16.9019 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.2676 +[titan] 2025-07-09 18:35:05,938 - root - INFO - lr: 2.6948e-04 gnorm: 0.86 [ 5:00:08<17:07:19] +[titan] 2025-07-09 18:35:09,834 - root - INFO - step: 22615 loss: 17.3346 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.7357 global_avg_mtp_loss: 14.5989 +[titan] 2025-07-09 18:35:09,834 - root - INFO - lr: 2.6947e-04 gnorm: 0.83 [ 5:00:12<17:07:15] +[titan] 2025-07-09 18:35:13,748 - root - INFO - step: 22620 loss: 16.9367 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.3057 +[titan] 2025-07-09 18:35:13,748 - root - INFO - lr: 2.6945e-04 gnorm: 0.79 [ 5:00:16<17:07:11] +[titan] 2025-07-09 18:35:17,682 - root - INFO - step: 22625 loss: 16.9972 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6564 global_avg_mtp_loss: 14.3408 +[titan] 2025-07-09 18:35:17,682 - root - INFO - lr: 2.6944e-04 gnorm: 0.77 [ 5:00:20<17:07:07] +[titan] 2025-07-09 18:35:21,600 - root - INFO - step: 22630 loss: 16.8619 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6190 global_avg_mtp_loss: 14.2429 +[titan] 2025-07-09 18:35:21,601 - root - INFO - lr: 2.6943e-04 gnorm: 0.74 [ 5:00:24<17:07:03] +[titan] 2025-07-09 18:35:25,507 - root - INFO - step: 22635 loss: 16.8924 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2743 +[titan] 2025-07-09 18:35:25,507 - root - INFO - lr: 2.6941e-04 gnorm: 0.74 [ 5:00:28<17:06:58] +[titan] 2025-07-09 18:35:29,420 - root - INFO - step: 22640 loss: 16.8975 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.2669 +[titan] 2025-07-09 18:35:29,420 - root - INFO - lr: 2.6940e-04 gnorm: 0.78 [ 5:00:32<17:06:54] +[titan] 2025-07-09 18:35:33,332 - root - INFO - step: 22645 loss: 16.8932 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2663 +[titan] 2025-07-09 18:35:33,332 - root - INFO - lr: 2.6938e-04 gnorm: 0.79 [ 5:00:35<17:06:50] +[titan] 2025-07-09 18:35:36,451 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:35:37,241 - root - INFO - step: 22650 loss: 16.8466 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.2272 +[titan] 2025-07-09 18:35:37,241 - root - INFO - lr: 2.6937e-04 gnorm: 0.79 [ 5:00:39<17:06:46] +[titan] 2025-07-09 18:35:41,136 - root - INFO - step: 22655 loss: 16.9403 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.3052 +[titan] 2025-07-09 18:35:41,136 - root - INFO - lr: 2.6936e-04 gnorm: 0.73 [ 5:00:43<17:06:41] +[titan] 2025-07-09 18:35:45,036 - root - INFO - step: 22660 loss: 16.7924 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.33% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1809 +[titan] 2025-07-09 18:35:45,036 - root - INFO - lr: 2.6934e-04 gnorm: 0.75 [ 5:00:47<17:06:37] +[titan] 2025-07-09 18:35:48,964 - root - INFO - step: 22665 loss: 17.1293 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6811 global_avg_mtp_loss: 14.4482 +[titan] 2025-07-09 18:35:48,964 - root - INFO - lr: 2.6933e-04 gnorm: 0.77 [ 5:00:51<17:06:33] +[titan] 2025-07-09 18:35:52,873 - root - INFO - step: 22670 loss: 16.6455 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0711 +[titan] 2025-07-09 18:35:52,873 - root - INFO - lr: 2.6932e-04 gnorm: 0.75 [ 5:00:55<17:06:29] +[titan] 2025-07-09 18:35:56,776 - root - INFO - step: 22675 loss: 17.0292 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6568 global_avg_mtp_loss: 14.3724 +[titan] 2025-07-09 18:35:56,776 - root - INFO - lr: 2.6930e-04 gnorm: 0.80 [ 5:00:59<17:06:25] +[titan] 2025-07-09 18:36:00,689 - root - INFO - step: 22680 loss: 17.0140 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6500 global_avg_mtp_loss: 14.3640 +[titan] 2025-07-09 18:36:00,689 - root - INFO - lr: 2.6929e-04 gnorm: 0.83 [ 5:01:03<17:06:20] +[titan] 2025-07-09 18:36:04,620 - root - INFO - step: 22685 loss: 16.6505 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.0675 +[titan] 2025-07-09 18:36:04,621 - root - INFO - lr: 2.6928e-04 gnorm: 0.92 [ 5:01:07<17:06:16] +[titan] 2025-07-09 18:36:08,552 - root - INFO - step: 22690 loss: 16.6433 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5746 global_avg_mtp_loss: 14.0688 +[titan] 2025-07-09 18:36:08,553 - root - INFO - lr: 2.6926e-04 gnorm: 0.76 [ 5:01:11<17:06:12] +[titan] 2025-07-09 18:36:12,494 - root - INFO - step: 22695 loss: 16.5972 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.0251 +[titan] 2025-07-09 18:36:12,494 - root - INFO - lr: 2.6925e-04 gnorm: 0.83 [ 5:01:15<17:06:08] +[titan] 2025-07-09 18:36:15,610 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:36:16,417 - root - INFO - step: 22700 loss: 17.0140 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.6565 global_avg_mtp_loss: 14.3575 +[titan] 2025-07-09 18:36:16,417 - root - INFO - lr: 2.6924e-04 gnorm: 0.81 [ 5:01:19<17:06:04] +[titan] 2025-07-09 18:36:20,330 - root - INFO - step: 22705 loss: 16.9195 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.2815 +[titan] 2025-07-09 18:36:20,331 - root - INFO - lr: 2.6922e-04 gnorm: 0.84 [ 5:01:22<17:05:59] +[titan] 2025-07-09 18:36:24,238 - root - INFO - step: 22710 loss: 16.9367 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6433 global_avg_mtp_loss: 14.2934 +[titan] 2025-07-09 18:36:24,239 - root - INFO - lr: 2.6921e-04 gnorm: 0.80 [ 5:01:26<17:05:55] +[titan] 2025-07-09 18:36:28,148 - root - INFO - step: 22715 loss: 16.8614 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6365 global_avg_mtp_loss: 14.2249 +[titan] 2025-07-09 18:36:28,148 - root - INFO - lr: 2.6919e-04 gnorm: 0.81 [ 5:01:30<17:05:51] +[titan] 2025-07-09 18:36:32,058 - root - INFO - step: 22720 loss: 16.9466 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6318 global_avg_mtp_loss: 14.3149 +[titan] 2025-07-09 18:36:32,058 - root - INFO - lr: 2.6918e-04 gnorm: 0.80 [ 5:01:34<17:05:47] +[titan] 2025-07-09 18:36:35,977 - root - INFO - step: 22725 loss: 16.8153 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6143 global_avg_mtp_loss: 14.2010 +[titan] 2025-07-09 18:36:35,978 - root - INFO - lr: 2.6917e-04 gnorm: 0.81 [ 5:01:38<17:05:43] +[titan] 2025-07-09 18:36:39,893 - root - INFO - step: 22730 loss: 16.8868 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6236 global_avg_mtp_loss: 14.2632 +[titan] 2025-07-09 18:36:39,893 - root - INFO - lr: 2.6915e-04 gnorm: 0.78 [ 5:01:42<17:05:38] +[titan] 2025-07-09 18:36:43,803 - root - INFO - step: 22735 loss: 16.7451 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6076 global_avg_mtp_loss: 14.1375 +[titan] 2025-07-09 18:36:43,803 - root - INFO - lr: 2.6914e-04 gnorm: 0.78 [ 5:01:46<17:05:34] +[titan] 2025-07-09 18:36:47,750 - root - INFO - step: 22740 loss: 17.0285 memory: 44.58GiB(31.99%) tps: 83,033 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.6424 global_avg_mtp_loss: 14.3861 +[titan] 2025-07-09 18:36:47,750 - root - INFO - lr: 2.6913e-04 gnorm: 0.76 [ 5:01:50<17:05:30] +[titan] 2025-07-09 18:36:51,660 - root - INFO - step: 22745 loss: 16.7778 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.1686 +[titan] 2025-07-09 18:36:51,660 - root - INFO - lr: 2.6911e-04 gnorm: 0.81 [ 5:01:54<17:05:26] +[titan] 2025-07-09 18:36:54,788 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:36:55,573 - root - INFO - step: 22750 loss: 17.1890 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6831 global_avg_mtp_loss: 14.5060 +[titan] 2025-07-09 18:36:55,573 - root - INFO - lr: 2.6910e-04 gnorm: 0.79 [ 5:01:58<17:05:22] +[titan] 2025-07-09 18:36:59,534 - root - INFO - step: 22755 loss: 16.7501 memory: 44.58GiB(31.99%) tps: 82,743 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.6099 global_avg_mtp_loss: 14.1401 +[titan] 2025-07-09 18:36:59,534 - root - INFO - lr: 2.6909e-04 gnorm: 0.84 [ 5:02:02<17:05:18] +[titan] 2025-07-09 18:37:03,439 - root - INFO - step: 22760 loss: 16.6346 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.0569 +[titan] 2025-07-09 18:37:03,439 - root - INFO - lr: 2.6907e-04 gnorm: 0.77 [ 5:02:06<17:05:13] +[titan] 2025-07-09 18:37:07,389 - root - INFO - step: 22765 loss: 16.4523 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9149 +[titan] 2025-07-09 18:37:07,390 - root - INFO - lr: 2.6906e-04 gnorm: 0.81 [ 5:02:09<17:05:09] +[titan] 2025-07-09 18:37:11,327 - root - INFO - step: 22770 loss: 16.8090 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.1931 +[titan] 2025-07-09 18:37:11,327 - root - INFO - lr: 2.6904e-04 gnorm: 0.82 [ 5:02:13<17:05:05] +[titan] 2025-07-09 18:37:15,254 - root - INFO - step: 22775 loss: 16.8859 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.2671 +[titan] 2025-07-09 18:37:15,254 - root - INFO - lr: 2.6903e-04 gnorm: 0.74 [ 5:02:17<17:05:01] +[titan] 2025-07-09 18:37:19,175 - root - INFO - step: 22780 loss: 17.2394 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6841 global_avg_mtp_loss: 14.5553 +[titan] 2025-07-09 18:37:19,175 - root - INFO - lr: 2.6902e-04 gnorm: 0.81 [ 5:02:21<17:04:57] +[titan] 2025-07-09 18:37:23,110 - root - INFO - step: 22785 loss: 16.8606 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.2275 +[titan] 2025-07-09 18:37:23,111 - root - INFO - lr: 2.6900e-04 gnorm: 0.78 [ 5:02:25<17:04:53] +[titan] 2025-07-09 18:37:27,022 - root - INFO - step: 22790 loss: 17.1187 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6736 global_avg_mtp_loss: 14.4451 +[titan] 2025-07-09 18:37:27,022 - root - INFO - lr: 2.6899e-04 gnorm: 0.78 [ 5:02:29<17:04:48] +[titan] 2025-07-09 18:37:30,945 - root - INFO - step: 22795 loss: 16.9725 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.3363 +[titan] 2025-07-09 18:37:30,945 - root - INFO - lr: 2.6898e-04 gnorm: 0.83 [ 5:02:33<17:04:44] +[titan] 2025-07-09 18:37:34,091 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:37:34,879 - root - INFO - step: 22800 loss: 16.8169 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.6201 global_avg_mtp_loss: 14.1968 +[titan] 2025-07-09 18:37:34,879 - root - INFO - lr: 2.6896e-04 gnorm: 0.79 [ 5:02:37<17:04:40] +[titan] 2025-07-09 18:37:38,778 - root - INFO - step: 22805 loss: 17.0574 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6582 global_avg_mtp_loss: 14.3992 +[titan] 2025-07-09 18:37:38,779 - root - INFO - lr: 2.6895e-04 gnorm: 0.80 [ 5:02:41<17:04:36] +[titan] 2025-07-09 18:37:42,683 - root - INFO - step: 22810 loss: 17.0167 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6418 global_avg_mtp_loss: 14.3748 +[titan] 2025-07-09 18:37:42,683 - root - INFO - lr: 2.6894e-04 gnorm: 0.84 [ 5:02:45<17:04:31] +[titan] 2025-07-09 18:37:46,633 - root - INFO - step: 22815 loss: 16.7635 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.6049 global_avg_mtp_loss: 14.1586 +[titan] 2025-07-09 18:37:46,633 - root - INFO - lr: 2.6892e-04 gnorm: 0.79 [ 5:02:49<17:04:27] +[titan] 2025-07-09 18:37:50,533 - root - INFO - step: 22820 loss: 16.9528 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6283 global_avg_mtp_loss: 14.3245 +[titan] 2025-07-09 18:37:50,534 - root - INFO - lr: 2.6891e-04 gnorm: 0.82 [ 5:02:53<17:04:23] +[titan] 2025-07-09 18:37:54,433 - root - INFO - step: 22825 loss: 17.1657 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6782 global_avg_mtp_loss: 14.4875 +[titan] 2025-07-09 18:37:54,434 - root - INFO - lr: 2.6889e-04 gnorm: 0.73 [ 5:02:57<17:04:19] +[titan] 2025-07-09 18:37:58,360 - root - INFO - step: 22830 loss: 16.9814 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6431 global_avg_mtp_loss: 14.3383 +[titan] 2025-07-09 18:37:58,360 - root - INFO - lr: 2.6888e-04 gnorm: 0.79 [ 5:03:00<17:04:15] +[titan] 2025-07-09 18:38:02,269 - root - INFO - step: 22835 loss: 17.1416 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6930 global_avg_mtp_loss: 14.4486 +[titan] 2025-07-09 18:38:02,269 - root - INFO - lr: 2.6887e-04 gnorm: 0.81 [ 5:03:04<17:04:10] +[titan] 2025-07-09 18:38:06,179 - root - INFO - step: 22840 loss: 16.8681 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2412 +[titan] 2025-07-09 18:38:06,179 - root - INFO - lr: 2.6885e-04 gnorm: 0.78 [ 5:03:08<17:04:06] +[titan] 2025-07-09 18:38:10,120 - root - INFO - step: 22845 loss: 16.8035 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.1995 +[titan] 2025-07-09 18:38:10,120 - root - INFO - lr: 2.6884e-04 gnorm: 0.80 [ 5:03:12<17:04:02] +[titan] 2025-07-09 18:38:13,262 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:38:14,062 - root - INFO - step: 22850 loss: 17.2161 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.6908 global_avg_mtp_loss: 14.5253 +[titan] 2025-07-09 18:38:14,062 - root - INFO - lr: 2.6883e-04 gnorm: 0.83 [ 5:03:16<17:03:58] +[titan] 2025-07-09 18:38:17,970 - root - INFO - step: 22855 loss: 16.9186 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.2819 +[titan] 2025-07-09 18:38:17,970 - root - INFO - lr: 2.6881e-04 gnorm: 0.77 [ 5:03:20<17:03:54] +[titan] 2025-07-09 18:38:21,890 - root - INFO - step: 22860 loss: 16.8586 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.2338 +[titan] 2025-07-09 18:38:21,890 - root - INFO - lr: 2.6880e-04 gnorm: 0.77 [ 5:03:24<17:03:50] +[titan] 2025-07-09 18:38:25,790 - root - INFO - step: 22865 loss: 16.6899 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.1061 +[titan] 2025-07-09 18:38:25,790 - root - INFO - lr: 2.6878e-04 gnorm: 0.80 [ 5:03:28<17:03:45] +[titan] 2025-07-09 18:38:29,744 - root - INFO - step: 22870 loss: 16.8539 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.6267 global_avg_mtp_loss: 14.2271 +[titan] 2025-07-09 18:38:29,744 - root - INFO - lr: 2.6877e-04 gnorm: 0.76 [ 5:03:32<17:03:41] +[titan] 2025-07-09 18:38:33,636 - root - INFO - step: 22875 loss: 16.9005 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.6280 global_avg_mtp_loss: 14.2725 +[titan] 2025-07-09 18:38:33,636 - root - INFO - lr: 2.6876e-04 gnorm: 0.76 [ 5:03:36<17:03:37] +[titan] 2025-07-09 18:38:37,551 - root - INFO - step: 22880 loss: 17.0428 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6548 global_avg_mtp_loss: 14.3880 +[titan] 2025-07-09 18:38:37,551 - root - INFO - lr: 2.6874e-04 gnorm: 0.72 [ 5:03:40<17:03:33] +[titan] 2025-07-09 18:38:41,457 - root - INFO - step: 22885 loss: 16.9954 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.3616 +[titan] 2025-07-09 18:38:41,457 - root - INFO - lr: 2.6873e-04 gnorm: 0.78 [ 5:03:44<17:03:29] +[titan] 2025-07-09 18:38:45,416 - root - INFO - step: 22890 loss: 16.9767 memory: 44.58GiB(31.99%) tps: 82,761 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.6525 global_avg_mtp_loss: 14.3243 +[titan] 2025-07-09 18:38:45,417 - root - INFO - lr: 2.6872e-04 gnorm: 0.80 [ 5:03:47<17:03:24] +[titan] 2025-07-09 18:38:49,326 - root - INFO - step: 22895 loss: 16.8016 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.1856 +[titan] 2025-07-09 18:38:49,326 - root - INFO - lr: 2.6870e-04 gnorm: 0.80 [ 5:03:51<17:03:20] +[titan] 2025-07-09 18:38:52,451 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:38:53,236 - root - INFO - step: 22900 loss: 17.1246 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.4582 +[titan] 2025-07-09 18:38:53,237 - root - INFO - lr: 2.6869e-04 gnorm: 0.77 [ 5:03:55<17:03:16] +[titan] 2025-07-09 18:38:57,144 - root - INFO - step: 22905 loss: 16.9752 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3338 +[titan] 2025-07-09 18:38:57,144 - root - INFO - lr: 2.6867e-04 gnorm: 0.78 [ 5:03:59<17:03:12] +[titan] 2025-07-09 18:39:01,093 - root - INFO - step: 22910 loss: 16.8291 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.6150 global_avg_mtp_loss: 14.2141 +[titan] 2025-07-09 18:39:01,094 - root - INFO - lr: 2.6866e-04 gnorm: 0.79 [ 5:04:03<17:03:08] +[titan] 2025-07-09 18:39:05,047 - root - INFO - step: 22915 loss: 16.9018 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.6242 global_avg_mtp_loss: 14.2776 +[titan] 2025-07-09 18:39:05,048 - root - INFO - lr: 2.6865e-04 gnorm: 0.77 [ 5:04:07<17:03:04] +[titan] 2025-07-09 18:39:08,945 - root - INFO - step: 22920 loss: 17.0393 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.3971 +[titan] 2025-07-09 18:39:08,946 - root - INFO - lr: 2.6863e-04 gnorm: 0.77 [ 5:04:11<17:02:59] +[titan] 2025-07-09 18:39:12,873 - root - INFO - step: 22925 loss: 16.8180 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.2134 +[titan] 2025-07-09 18:39:12,873 - root - INFO - lr: 2.6862e-04 gnorm: 0.73 [ 5:04:15<17:02:55] +[titan] 2025-07-09 18:39:16,789 - root - INFO - step: 22930 loss: 16.9047 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2787 +[titan] 2025-07-09 18:39:16,789 - root - INFO - lr: 2.6861e-04 gnorm: 0.83 [ 5:04:19<17:02:51] +[titan] 2025-07-09 18:39:20,745 - root - INFO - step: 22935 loss: 16.6743 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.0829 +[titan] 2025-07-09 18:39:20,746 - root - INFO - lr: 2.6859e-04 gnorm: 0.82 [ 5:04:23<17:02:47] +[titan] 2025-07-09 18:39:24,665 - root - INFO - step: 22940 loss: 16.7586 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.1547 +[titan] 2025-07-09 18:39:24,665 - root - INFO - lr: 2.6858e-04 gnorm: 0.86 [ 5:04:27<17:02:43] +[titan] 2025-07-09 18:39:28,577 - root - INFO - step: 22945 loss: 17.0128 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6545 global_avg_mtp_loss: 14.3583 +[titan] 2025-07-09 18:39:28,577 - root - INFO - lr: 2.6857e-04 gnorm: 0.89 [ 5:04:31<17:02:39] +[titan] 2025-07-09 18:39:31,730 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:39:32,553 - root - INFO - step: 22950 loss: 16.9662 memory: 44.58GiB(31.99%) tps: 82,427 tflops: 284.47 mfu: 28.76% global_avg_ntp_loss: 2.6544 global_avg_mtp_loss: 14.3117 +[titan] 2025-07-09 18:39:32,553 - root - INFO - lr: 2.6855e-04 gnorm: 0.78 [ 5:04:35<17:02:35] +[titan] 2025-07-09 18:39:36,471 - root - INFO - step: 22955 loss: 16.9409 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.6262 global_avg_mtp_loss: 14.3147 +[titan] 2025-07-09 18:39:36,471 - root - INFO - lr: 2.6854e-04 gnorm: 0.72 [ 5:04:39<17:02:30] +[titan] 2025-07-09 18:39:40,369 - root - INFO - step: 22960 loss: 16.7178 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1273 +[titan] 2025-07-09 18:39:40,369 - root - INFO - lr: 2.6852e-04 gnorm: 0.77 [ 5:04:42<17:02:26] +[titan] 2025-07-09 18:39:44,315 - root - INFO - step: 22965 loss: 16.8566 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.6148 global_avg_mtp_loss: 14.2418 +[titan] 2025-07-09 18:39:44,315 - root - INFO - lr: 2.6851e-04 gnorm: 0.80 [ 5:04:46<17:02:22] +[titan] 2025-07-09 18:39:48,213 - root - INFO - step: 22970 loss: 16.9444 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6400 global_avg_mtp_loss: 14.3044 +[titan] 2025-07-09 18:39:48,213 - root - INFO - lr: 2.6850e-04 gnorm: 0.87 [ 5:04:50<17:02:18] +[titan] 2025-07-09 18:39:52,131 - root - INFO - step: 22975 loss: 16.9811 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.3352 +[titan] 2025-07-09 18:39:52,132 - root - INFO - lr: 2.6848e-04 gnorm: 0.82 [ 5:04:54<17:02:13] +[titan] 2025-07-09 18:39:56,036 - root - INFO - step: 22980 loss: 16.8545 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6248 global_avg_mtp_loss: 14.2297 +[titan] 2025-07-09 18:39:56,036 - root - INFO - lr: 2.6847e-04 gnorm: 0.81 [ 5:04:58<17:02:09] +[titan] 2025-07-09 18:39:59,939 - root - INFO - step: 22985 loss: 17.1424 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6975 global_avg_mtp_loss: 14.4448 +[titan] 2025-07-09 18:39:59,939 - root - INFO - lr: 2.6846e-04 gnorm: 0.78 [ 5:05:02<17:02:05] +[titan] 2025-07-09 18:40:03,829 - root - INFO - step: 22990 loss: 17.0963 memory: 44.58GiB(31.99%) tps: 84,250 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.6712 global_avg_mtp_loss: 14.4252 +[titan] 2025-07-09 18:40:03,829 - root - INFO - lr: 2.6844e-04 gnorm: 0.78 [ 5:05:06<17:02:01] +[titan] 2025-07-09 18:40:07,736 - root - INFO - step: 22995 loss: 16.8702 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.2442 +[titan] 2025-07-09 18:40:07,737 - root - INFO - lr: 2.6843e-04 gnorm: 0.86 [ 5:05:10<17:01:56] +[titan] 2025-07-09 18:40:10,875 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:40:11,660 - root - INFO - step: 23000 loss: 17.0471 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6464 global_avg_mtp_loss: 14.4007 +[titan] 2025-07-09 18:40:11,660 - root - INFO - lr: 2.6841e-04 gnorm: 0.75 [ 5:05:14<17:01:52] +[titan] 2025-07-09 18:40:15,576 - root - INFO - step: 23005 loss: 16.7237 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6000 global_avg_mtp_loss: 14.1237 +[titan] 2025-07-09 18:40:15,577 - root - INFO - lr: 2.6840e-04 gnorm: 0.77 [ 5:05:18<17:01:48] +[titan] 2025-07-09 18:40:19,541 - root - INFO - step: 23010 loss: 17.0127 memory: 44.58GiB(31.99%) tps: 82,653 tflops: 285.25 mfu: 28.84% global_avg_ntp_loss: 2.6433 global_avg_mtp_loss: 14.3693 +[titan] 2025-07-09 18:40:19,541 - root - INFO - lr: 2.6839e-04 gnorm: 0.71 [ 5:05:22<17:01:44] +[titan] 2025-07-09 18:40:23,475 - root - INFO - step: 23015 loss: 16.9964 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6547 global_avg_mtp_loss: 14.3416 +[titan] 2025-07-09 18:40:23,475 - root - INFO - lr: 2.6837e-04 gnorm: 0.74 [ 5:05:26<17:01:40] +[titan] 2025-07-09 18:40:27,415 - root - INFO - step: 23020 loss: 17.0261 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6474 global_avg_mtp_loss: 14.3787 +[titan] 2025-07-09 18:40:27,415 - root - INFO - lr: 2.6836e-04 gnorm: 0.79 [ 5:05:29<17:01:36] +[titan] 2025-07-09 18:40:31,356 - root - INFO - step: 23025 loss: 16.6188 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.0419 +[titan] 2025-07-09 18:40:31,356 - root - INFO - lr: 2.6834e-04 gnorm: 0.83 [ 5:05:33<17:01:32] +[titan] 2025-07-09 18:40:35,269 - root - INFO - step: 23030 loss: 17.1170 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6751 global_avg_mtp_loss: 14.4419 +[titan] 2025-07-09 18:40:35,269 - root - INFO - lr: 2.6833e-04 gnorm: 0.76 [ 5:05:37<17:01:27] +[titan] 2025-07-09 18:40:39,197 - root - INFO - step: 23035 loss: 16.8761 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.2608 +[titan] 2025-07-09 18:40:39,197 - root - INFO - lr: 2.6832e-04 gnorm: 0.70 [ 5:05:41<17:01:23] +[titan] 2025-07-09 18:40:43,242 - root - INFO - step: 23040 loss: 16.9668 memory: 44.58GiB(31.99%) tps: 81,017 tflops: 279.60 mfu: 28.27% global_avg_ntp_loss: 2.6377 global_avg_mtp_loss: 14.3292 +[titan] 2025-07-09 18:40:43,242 - root - INFO - lr: 2.6830e-04 gnorm: 0.81 [ 5:05:45<17:01:20] +[titan] 2025-07-09 18:40:43,395 - root - INFO - Dumping profiler traces at step 23040 +[titan] 2025-07-09 18:40:43,426 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:40:47,322 - root - INFO - step: 23045 loss: 16.9609 memory: 44.58GiB(31.99%) tps: 80,322 tflops: 277.21 mfu: 28.03% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.3221 +[titan] 2025-07-09 18:40:47,322 - root - INFO - lr: 2.6829e-04 gnorm: 0.77 [ 5:05:49<17:01:16] +[titan] 2025-07-09 18:40:50,461 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:40:51,259 - root - INFO - step: 23050 loss: 16.5847 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.0051 +[titan] 2025-07-09 18:40:51,259 - root - INFO - lr: 2.6828e-04 gnorm: 0.78 [ 5:05:53<17:01:12] +[titan] 2025-07-09 18:40:55,176 - root - INFO - step: 23055 loss: 16.5578 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 13.9842 +[titan] 2025-07-09 18:40:55,176 - root - INFO - lr: 2.6826e-04 gnorm: 0.79 [ 5:05:57<17:01:08] +[titan] 2025-07-09 18:40:59,090 - root - INFO - step: 23060 loss: 16.6953 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.1037 +[titan] 2025-07-09 18:40:59,090 - root - INFO - lr: 2.6825e-04 gnorm: 0.74 [ 5:06:01<17:01:03] +[titan] 2025-07-09 18:41:02,981 - root - INFO - step: 23065 loss: 16.7235 memory: 44.58GiB(31.99%) tps: 84,219 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1269 +[titan] 2025-07-09 18:41:02,982 - root - INFO - lr: 2.6823e-04 gnorm: 0.79 [ 5:06:05<17:00:59] +[titan] 2025-07-09 18:41:06,887 - root - INFO - step: 23070 loss: 17.0755 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6634 global_avg_mtp_loss: 14.4121 +[titan] 2025-07-09 18:41:06,887 - root - INFO - lr: 2.6822e-04 gnorm: 0.74 [ 5:06:09<17:00:55] +[titan] 2025-07-09 18:41:10,787 - root - INFO - step: 23075 loss: 16.9992 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6600 global_avg_mtp_loss: 14.3392 +[titan] 2025-07-09 18:41:10,787 - root - INFO - lr: 2.6821e-04 gnorm: 0.83 [ 5:06:13<17:00:51] +[titan] 2025-07-09 18:41:14,709 - root - INFO - step: 23080 loss: 16.8795 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.2380 +[titan] 2025-07-09 18:41:14,709 - root - INFO - lr: 2.6819e-04 gnorm: 0.80 [ 5:06:17<17:00:46] +[titan] 2025-07-09 18:41:18,646 - root - INFO - step: 23085 loss: 16.8306 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.6132 global_avg_mtp_loss: 14.2173 +[titan] 2025-07-09 18:41:18,646 - root - INFO - lr: 2.6818e-04 gnorm: 0.80 [ 5:06:21<17:00:42] +[titan] 2025-07-09 18:41:22,546 - root - INFO - step: 23090 loss: 16.9636 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3242 +[titan] 2025-07-09 18:41:22,546 - root - INFO - lr: 2.6817e-04 gnorm: 0.80 [ 5:06:25<17:00:38] +[titan] 2025-07-09 18:41:26,497 - root - INFO - step: 23095 loss: 16.9809 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.6504 global_avg_mtp_loss: 14.3305 +[titan] 2025-07-09 18:41:26,497 - root - INFO - lr: 2.6815e-04 gnorm: 0.77 [ 5:06:29<17:00:34] +[titan] 2025-07-09 18:41:29,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:41:30,389 - root - INFO - step: 23100 loss: 17.1095 memory: 44.58GiB(31.99%) tps: 84,193 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.6732 global_avg_mtp_loss: 14.4363 +[titan] 2025-07-09 18:41:30,390 - root - INFO - lr: 2.6814e-04 gnorm: 0.71 [ 5:06:32<17:00:30] +[titan] 2025-07-09 18:41:34,319 - root - INFO - step: 23105 loss: 16.8775 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.2490 +[titan] 2025-07-09 18:41:34,319 - root - INFO - lr: 2.6812e-04 gnorm: 0.72 [ 5:06:36<17:00:26] +[titan] 2025-07-09 18:41:38,230 - root - INFO - step: 23110 loss: 16.6905 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.0985 +[titan] 2025-07-09 18:41:38,230 - root - INFO - lr: 2.6811e-04 gnorm: 0.77 [ 5:06:40<17:00:21] +[titan] 2025-07-09 18:41:42,139 - root - INFO - step: 23115 loss: 17.1018 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6644 global_avg_mtp_loss: 14.4374 +[titan] 2025-07-09 18:41:42,140 - root - INFO - lr: 2.6810e-04 gnorm: 0.78 [ 5:06:44<17:00:17] +[titan] 2025-07-09 18:41:46,060 - root - INFO - step: 23120 loss: 17.0825 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6567 global_avg_mtp_loss: 14.4258 +[titan] 2025-07-09 18:41:46,060 - root - INFO - lr: 2.6808e-04 gnorm: 0.75 [ 5:06:48<17:00:13] +[titan] 2025-07-09 18:41:49,954 - root - INFO - step: 23125 loss: 16.6926 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1012 +[titan] 2025-07-09 18:41:49,954 - root - INFO - lr: 2.6807e-04 gnorm: 0.84 [ 5:06:52<17:00:09] +[titan] 2025-07-09 18:41:53,841 - root - INFO - step: 23130 loss: 16.9912 memory: 44.58GiB(31.99%) tps: 84,317 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.3499 +[titan] 2025-07-09 18:41:53,841 - root - INFO - lr: 2.6805e-04 gnorm: 0.75 [ 5:06:56<17:00:04] +[titan] 2025-07-09 18:41:57,771 - root - INFO - step: 23135 loss: 16.9125 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6452 global_avg_mtp_loss: 14.2673 +[titan] 2025-07-09 18:41:57,771 - root - INFO - lr: 2.6804e-04 gnorm: 0.86 [ 5:07:00<17:00:00] +[titan] 2025-07-09 18:42:01,675 - root - INFO - step: 23140 loss: 16.7986 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6085 global_avg_mtp_loss: 14.1901 +[titan] 2025-07-09 18:42:01,675 - root - INFO - lr: 2.6803e-04 gnorm: 0.85 [ 5:07:04<16:59:56] +[titan] 2025-07-09 18:42:05,566 - root - INFO - step: 23145 loss: 16.8968 memory: 44.58GiB(31.99%) tps: 84,219 tflops: 290.65 mfu: 29.39% global_avg_ntp_loss: 2.6198 global_avg_mtp_loss: 14.2770 +[titan] 2025-07-09 18:42:05,567 - root - INFO - lr: 2.6801e-04 gnorm: 0.78 [ 5:07:08<16:59:52] +[titan] 2025-07-09 18:42:08,700 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:42:09,486 - root - INFO - step: 23150 loss: 16.9118 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.2699 +[titan] 2025-07-09 18:42:09,486 - root - INFO - lr: 2.6800e-04 gnorm: 0.77 [ 5:07:12<16:59:47] +[titan] 2025-07-09 18:42:13,402 - root - INFO - step: 23155 loss: 17.2260 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.7214 global_avg_mtp_loss: 14.5045 +[titan] 2025-07-09 18:42:13,402 - root - INFO - lr: 2.6799e-04 gnorm: 0.80 [ 5:07:15<16:59:43] +[titan] 2025-07-09 18:42:17,311 - root - INFO - step: 23160 loss: 16.9129 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.2819 +[titan] 2025-07-09 18:42:17,312 - root - INFO - lr: 2.6797e-04 gnorm: 0.77 [ 5:07:19<16:59:39] +[titan] 2025-07-09 18:42:21,226 - root - INFO - step: 23165 loss: 16.9688 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6356 global_avg_mtp_loss: 14.3331 +[titan] 2025-07-09 18:42:21,226 - root - INFO - lr: 2.6796e-04 gnorm: 0.79 [ 5:07:23<16:59:35] +[titan] 2025-07-09 18:42:25,121 - root - INFO - step: 23170 loss: 16.5632 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5678 global_avg_mtp_loss: 13.9953 +[titan] 2025-07-09 18:42:25,121 - root - INFO - lr: 2.6794e-04 gnorm: 0.78 [ 5:07:27<16:59:31] +[titan] 2025-07-09 18:42:29,051 - root - INFO - step: 23175 loss: 16.6507 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.0618 +[titan] 2025-07-09 18:42:29,051 - root - INFO - lr: 2.6793e-04 gnorm: 0.77 [ 5:07:31<16:59:26] +[titan] 2025-07-09 18:42:32,970 - root - INFO - step: 23180 loss: 16.9143 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6382 global_avg_mtp_loss: 14.2762 +[titan] 2025-07-09 18:42:32,970 - root - INFO - lr: 2.6792e-04 gnorm: 0.82 [ 5:07:35<16:59:22] +[titan] 2025-07-09 18:42:36,871 - root - INFO - step: 23185 loss: 16.7994 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.1944 +[titan] 2025-07-09 18:42:36,871 - root - INFO - lr: 2.6790e-04 gnorm: 0.76 [ 5:07:39<16:59:18] +[titan] 2025-07-09 18:42:40,783 - root - INFO - step: 23190 loss: 17.1113 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6640 global_avg_mtp_loss: 14.4473 +[titan] 2025-07-09 18:42:40,783 - root - INFO - lr: 2.6789e-04 gnorm: 0.75 [ 5:07:43<16:59:14] +[titan] 2025-07-09 18:42:44,716 - root - INFO - step: 23195 loss: 17.0498 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6497 global_avg_mtp_loss: 14.4001 +[titan] 2025-07-09 18:42:44,716 - root - INFO - lr: 2.6787e-04 gnorm: 0.81 [ 5:07:47<16:59:10] +[titan] 2025-07-09 18:42:47,839 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:42:48,630 - root - INFO - step: 23200 loss: 16.7840 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.1647 +[titan] 2025-07-09 18:42:48,630 - root - INFO - lr: 2.6786e-04 gnorm: 0.78 [ 5:07:51<16:59:05] +[titan] 2025-07-09 18:42:52,553 - root - INFO - step: 23205 loss: 17.0403 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3765 +[titan] 2025-07-09 18:42:52,553 - root - INFO - lr: 2.6785e-04 gnorm: 0.79 [ 5:07:55<16:59:01] +[titan] 2025-07-09 18:42:56,448 - root - INFO - step: 23210 loss: 16.8716 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2493 +[titan] 2025-07-09 18:42:56,449 - root - INFO - lr: 2.6783e-04 gnorm: 0.77 [ 5:07:58<16:58:57] +[titan] 2025-07-09 18:43:00,333 - root - INFO - step: 23215 loss: 16.7774 memory: 44.58GiB(31.99%) tps: 84,356 tflops: 291.13 mfu: 29.44% global_avg_ntp_loss: 2.6097 global_avg_mtp_loss: 14.1677 +[titan] 2025-07-09 18:43:00,334 - root - INFO - lr: 2.6782e-04 gnorm: 0.81 [ 5:08:02<16:58:53] +[titan] 2025-07-09 18:43:04,238 - root - INFO - step: 23220 loss: 16.7840 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.1780 +[titan] 2025-07-09 18:43:04,238 - root - INFO - lr: 2.6781e-04 gnorm: 0.76 [ 5:08:06<16:58:48] +[titan] 2025-07-09 18:43:08,137 - root - INFO - step: 23225 loss: 17.1780 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6856 global_avg_mtp_loss: 14.4925 +[titan] 2025-07-09 18:43:08,137 - root - INFO - lr: 2.6779e-04 gnorm: 0.77 [ 5:08:10<16:58:44] +[titan] 2025-07-09 18:43:12,091 - root - INFO - step: 23230 loss: 17.0127 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.6571 global_avg_mtp_loss: 14.3555 +[titan] 2025-07-09 18:43:12,092 - root - INFO - lr: 2.6778e-04 gnorm: 0.78 [ 5:08:14<16:58:40] +[titan] 2025-07-09 18:43:16,006 - root - INFO - step: 23235 loss: 17.0334 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.3843 +[titan] 2025-07-09 18:43:16,007 - root - INFO - lr: 2.6776e-04 gnorm: 0.73 [ 5:08:18<16:58:36] +[titan] 2025-07-09 18:43:19,917 - root - INFO - step: 23240 loss: 17.0087 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6524 global_avg_mtp_loss: 14.3563 +[titan] 2025-07-09 18:43:19,917 - root - INFO - lr: 2.6775e-04 gnorm: 0.80 [ 5:08:22<16:58:32] +[titan] 2025-07-09 18:43:23,809 - root - INFO - step: 23245 loss: 16.9066 memory: 44.58GiB(31.99%) tps: 84,195 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.2814 +[titan] 2025-07-09 18:43:23,809 - root - INFO - lr: 2.6774e-04 gnorm: 0.76 [ 5:08:26<16:58:27] +[titan] 2025-07-09 18:43:26,918 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:43:27,721 - root - INFO - step: 23250 loss: 16.7748 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.1546 +[titan] 2025-07-09 18:43:27,721 - root - INFO - lr: 2.6772e-04 gnorm: 0.86 [ 5:08:30<16:58:23] +[titan] 2025-07-09 18:43:31,635 - root - INFO - step: 23255 loss: 16.9744 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6370 global_avg_mtp_loss: 14.3374 +[titan] 2025-07-09 18:43:31,635 - root - INFO - lr: 2.6771e-04 gnorm: 0.87 [ 5:08:34<16:58:19] +[titan] 2025-07-09 18:43:35,541 - root - INFO - step: 23260 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2504 +[titan] 2025-07-09 18:43:35,541 - root - INFO - lr: 2.6769e-04 gnorm: 0.93 [ 5:08:38<16:58:15] +[titan] 2025-07-09 18:43:39,447 - root - INFO - step: 23265 loss: 16.5641 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 13.9928 +[titan] 2025-07-09 18:43:39,447 - root - INFO - lr: 2.6768e-04 gnorm: 1.98 [ 5:08:41<16:58:11] +[titan] 2025-07-09 18:43:43,360 - root - INFO - step: 23270 loss: 16.7252 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1193 +[titan] 2025-07-09 18:43:43,360 - root - INFO - lr: 2.6767e-04 gnorm: 0.84 [ 5:08:45<16:58:06] +[titan] 2025-07-09 18:43:47,292 - root - INFO - step: 23275 loss: 16.7914 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6100 global_avg_mtp_loss: 14.1814 +[titan] 2025-07-09 18:43:47,292 - root - INFO - lr: 2.6765e-04 gnorm: 0.82 [ 5:08:49<16:58:02] +[titan] 2025-07-09 18:43:51,184 - root - INFO - step: 23280 loss: 16.9514 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.3000 +[titan] 2025-07-09 18:43:51,184 - root - INFO - lr: 2.6764e-04 gnorm: 0.79 [ 5:08:53<16:57:58] +[titan] 2025-07-09 18:43:55,074 - root - INFO - step: 23285 loss: 16.8541 memory: 44.58GiB(31.99%) tps: 84,245 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.2474 +[titan] 2025-07-09 18:43:55,074 - root - INFO - lr: 2.6762e-04 gnorm: 0.70 [ 5:08:57<16:57:54] +[titan] 2025-07-09 18:43:59,026 - root - INFO - step: 23290 loss: 16.9628 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3233 +[titan] 2025-07-09 18:43:59,027 - root - INFO - lr: 2.6761e-04 gnorm: 0.76 [ 5:09:01<16:57:50] +[titan] 2025-07-09 18:44:02,945 - root - INFO - step: 23295 loss: 17.0268 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.6618 global_avg_mtp_loss: 14.3650 +[titan] 2025-07-09 18:44:02,945 - root - INFO - lr: 2.6760e-04 gnorm: 0.75 [ 5:09:05<16:57:45] +[titan] 2025-07-09 18:44:06,061 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:44:06,848 - root - INFO - step: 23300 loss: 17.0142 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.3662 +[titan] 2025-07-09 18:44:06,849 - root - INFO - lr: 2.6758e-04 gnorm: 0.82 [ 5:09:09<16:57:41] +[titan] 2025-07-09 18:44:10,741 - root - INFO - step: 23305 loss: 16.8573 memory: 44.58GiB(31.99%) tps: 84,183 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.6233 global_avg_mtp_loss: 14.2339 +[titan] 2025-07-09 18:44:10,742 - root - INFO - lr: 2.6757e-04 gnorm: 0.75 [ 5:09:13<16:57:37] +[titan] 2025-07-09 18:44:14,645 - root - INFO - step: 23310 loss: 16.9193 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.2984 +[titan] 2025-07-09 18:44:14,646 - root - INFO - lr: 2.6756e-04 gnorm: 0.76 [ 5:09:17<16:57:33] +[titan] 2025-07-09 18:44:18,539 - root - INFO - step: 23315 loss: 16.8164 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.2124 +[titan] 2025-07-09 18:44:18,539 - root - INFO - lr: 2.6754e-04 gnorm: 0.81 [ 5:09:21<16:57:28] +[titan] 2025-07-09 18:44:22,491 - root - INFO - step: 23320 loss: 16.8307 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.2311 +[titan] 2025-07-09 18:44:22,492 - root - INFO - lr: 2.6753e-04 gnorm: 0.80 [ 5:09:25<16:57:24] +[titan] 2025-07-09 18:44:26,416 - root - INFO - step: 23325 loss: 16.8470 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.2260 +[titan] 2025-07-09 18:44:26,416 - root - INFO - lr: 2.6751e-04 gnorm: 0.75 [ 5:09:28<16:57:20] +[titan] 2025-07-09 18:44:30,342 - root - INFO - step: 23330 loss: 16.9096 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6299 global_avg_mtp_loss: 14.2797 +[titan] 2025-07-09 18:44:30,343 - root - INFO - lr: 2.6750e-04 gnorm: 0.79 [ 5:09:32<16:57:16] +[titan] 2025-07-09 18:44:34,260 - root - INFO - step: 23335 loss: 17.1176 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6715 global_avg_mtp_loss: 14.4461 +[titan] 2025-07-09 18:44:34,260 - root - INFO - lr: 2.6749e-04 gnorm: 0.83 [ 5:09:36<16:57:12] +[titan] 2025-07-09 18:44:38,206 - root - INFO - step: 23340 loss: 16.8302 memory: 44.58GiB(31.99%) tps: 83,043 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.2195 +[titan] 2025-07-09 18:44:38,206 - root - INFO - lr: 2.6747e-04 gnorm: 0.74 [ 5:09:40<16:57:08] +[titan] 2025-07-09 18:44:42,147 - root - INFO - step: 23345 loss: 16.9233 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6476 global_avg_mtp_loss: 14.2757 +[titan] 2025-07-09 18:44:42,147 - root - INFO - lr: 2.6746e-04 gnorm: 0.77 [ 5:09:44<16:57:04] +[titan] 2025-07-09 18:44:45,272 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:44:46,075 - root - INFO - step: 23350 loss: 16.7794 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6010 global_avg_mtp_loss: 14.1784 +[titan] 2025-07-09 18:44:46,076 - root - INFO - lr: 2.6744e-04 gnorm: 0.76 [ 5:09:48<16:56:59] +[titan] 2025-07-09 18:44:49,964 - root - INFO - step: 23355 loss: 16.6315 memory: 44.58GiB(31.99%) tps: 84,274 tflops: 290.84 mfu: 29.41% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.0462 +[titan] 2025-07-09 18:44:49,964 - root - INFO - lr: 2.6743e-04 gnorm: 0.77 [ 5:09:52<16:56:55] +[titan] 2025-07-09 18:44:53,874 - root - INFO - step: 23360 loss: 16.7532 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1472 +[titan] 2025-07-09 18:44:53,875 - root - INFO - lr: 2.6742e-04 gnorm: 0.79 [ 5:09:56<16:56:51] +[titan] 2025-07-09 18:44:57,789 - root - INFO - step: 23365 loss: 16.7358 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1434 +[titan] 2025-07-09 18:44:57,789 - root - INFO - lr: 2.6740e-04 gnorm: 0.81 [ 5:10:00<16:56:47] +[titan] 2025-07-09 18:45:01,701 - root - INFO - step: 23370 loss: 16.9704 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6455 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 18:45:01,701 - root - INFO - lr: 2.6739e-04 gnorm: 0.74 [ 5:10:04<16:56:43] +[titan] 2025-07-09 18:45:05,606 - root - INFO - step: 23375 loss: 16.9891 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6478 global_avg_mtp_loss: 14.3413 +[titan] 2025-07-09 18:45:05,607 - root - INFO - lr: 2.6737e-04 gnorm: 0.79 [ 5:10:08<16:56:38] +[titan] 2025-07-09 18:45:09,522 - root - INFO - step: 23380 loss: 16.9722 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6386 global_avg_mtp_loss: 14.3336 +[titan] 2025-07-09 18:45:09,522 - root - INFO - lr: 2.6736e-04 gnorm: 0.80 [ 5:10:12<16:56:34] +[titan] 2025-07-09 18:45:13,451 - root - INFO - step: 23385 loss: 16.7721 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.6037 global_avg_mtp_loss: 14.1684 +[titan] 2025-07-09 18:45:13,451 - root - INFO - lr: 2.6735e-04 gnorm: 0.72 [ 5:10:15<16:56:30] +[titan] 2025-07-09 18:45:17,377 - root - INFO - step: 23390 loss: 16.8111 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.2087 +[titan] 2025-07-09 18:45:17,378 - root - INFO - lr: 2.6733e-04 gnorm: 0.73 [ 5:10:19<16:56:26] +[titan] 2025-07-09 18:45:21,285 - root - INFO - step: 23395 loss: 17.1029 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6768 global_avg_mtp_loss: 14.4261 +[titan] 2025-07-09 18:45:21,285 - root - INFO - lr: 2.6732e-04 gnorm: 0.74 [ 5:10:23<16:56:22] +[titan] 2025-07-09 18:45:24,388 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:45:25,177 - root - INFO - step: 23400 loss: 16.9495 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.3243 +[titan] 2025-07-09 18:45:25,177 - root - INFO - lr: 2.6730e-04 gnorm: 0.75 [ 5:10:27<16:56:17] +[titan] 2025-07-09 18:45:29,083 - root - INFO - step: 23405 loss: 17.0406 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6510 global_avg_mtp_loss: 14.3896 +[titan] 2025-07-09 18:45:29,083 - root - INFO - lr: 2.6729e-04 gnorm: 0.79 [ 5:10:31<16:56:13] +[titan] 2025-07-09 18:45:32,992 - root - INFO - step: 23410 loss: 17.0131 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6463 global_avg_mtp_loss: 14.3667 +[titan] 2025-07-09 18:45:32,992 - root - INFO - lr: 2.6728e-04 gnorm: 0.79 [ 5:10:35<16:56:09] +[titan] 2025-07-09 18:45:36,920 - root - INFO - step: 23415 loss: 16.8293 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6098 global_avg_mtp_loss: 14.2194 +[titan] 2025-07-09 18:45:36,921 - root - INFO - lr: 2.6726e-04 gnorm: 0.80 [ 5:10:39<16:56:05] +[titan] 2025-07-09 18:45:40,840 - root - INFO - step: 23420 loss: 17.0201 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.3642 +[titan] 2025-07-09 18:45:40,840 - root - INFO - lr: 2.6725e-04 gnorm: 0.80 [ 5:10:43<16:56:01] +[titan] 2025-07-09 18:45:44,794 - root - INFO - step: 23425 loss: 16.9804 memory: 44.58GiB(31.99%) tps: 82,870 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.6411 global_avg_mtp_loss: 14.3393 +[titan] 2025-07-09 18:45:44,794 - root - INFO - lr: 2.6723e-04 gnorm: 0.77 [ 5:10:47<16:55:57] +[titan] 2025-07-09 18:45:48,710 - root - INFO - step: 23430 loss: 16.7992 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6011 global_avg_mtp_loss: 14.1981 +[titan] 2025-07-09 18:45:48,711 - root - INFO - lr: 2.6722e-04 gnorm: 0.75 [ 5:10:51<16:55:52] +[titan] 2025-07-09 18:45:52,599 - root - INFO - step: 23435 loss: 16.7909 memory: 44.58GiB(31.99%) tps: 84,269 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.6096 global_avg_mtp_loss: 14.1813 +[titan] 2025-07-09 18:45:52,600 - root - INFO - lr: 2.6721e-04 gnorm: 0.76 [ 5:10:55<16:55:48] +[titan] 2025-07-09 18:45:56,511 - root - INFO - step: 23440 loss: 16.9346 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6413 global_avg_mtp_loss: 14.2933 +[titan] 2025-07-09 18:45:56,511 - root - INFO - lr: 2.6719e-04 gnorm: 0.79 [ 5:10:59<16:55:44] +[titan] 2025-07-09 18:46:00,425 - root - INFO - step: 23445 loss: 16.8763 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.6263 global_avg_mtp_loss: 14.2500 +[titan] 2025-07-09 18:46:00,425 - root - INFO - lr: 2.6718e-04 gnorm: 0.76 [ 5:11:02<16:55:40] +[titan] 2025-07-09 18:46:03,547 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:46:04,339 - root - INFO - step: 23450 loss: 16.6146 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.0232 +[titan] 2025-07-09 18:46:04,339 - root - INFO - lr: 2.6716e-04 gnorm: 0.77 [ 5:11:06<16:55:35] +[titan] 2025-07-09 18:46:08,257 - root - INFO - step: 23455 loss: 16.8261 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6233 global_avg_mtp_loss: 14.2028 +[titan] 2025-07-09 18:46:08,257 - root - INFO - lr: 2.6715e-04 gnorm: 0.80 [ 5:11:10<16:55:31] +[titan] 2025-07-09 18:46:12,156 - root - INFO - step: 23460 loss: 17.0109 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6516 global_avg_mtp_loss: 14.3593 +[titan] 2025-07-09 18:46:12,156 - root - INFO - lr: 2.6714e-04 gnorm: 0.79 [ 5:11:14<16:55:27] +[titan] 2025-07-09 18:46:16,101 - root - INFO - step: 23465 loss: 16.6428 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5850 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-09 18:46:16,101 - root - INFO - lr: 2.6712e-04 gnorm: 0.77 [ 5:11:18<16:55:23] +[titan] 2025-07-09 18:46:19,994 - root - INFO - step: 23470 loss: 16.7160 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5952 global_avg_mtp_loss: 14.1208 +[titan] 2025-07-09 18:46:19,995 - root - INFO - lr: 2.6711e-04 gnorm: 0.81 [ 5:11:22<16:55:19] +[titan] 2025-07-09 18:46:23,915 - root - INFO - step: 23475 loss: 16.8073 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.2086 +[titan] 2025-07-09 18:46:23,915 - root - INFO - lr: 2.6709e-04 gnorm: 0.78 [ 5:11:26<16:55:14] +[titan] 2025-07-09 18:46:27,812 - root - INFO - step: 23480 loss: 16.9739 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6564 global_avg_mtp_loss: 14.3174 +[titan] 2025-07-09 18:46:27,813 - root - INFO - lr: 2.6708e-04 gnorm: 0.78 [ 5:11:30<16:55:10] +[titan] 2025-07-09 18:46:31,711 - root - INFO - step: 23485 loss: 16.9687 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.3344 +[titan] 2025-07-09 18:46:31,711 - root - INFO - lr: 2.6707e-04 gnorm: 0.82 [ 5:11:34<16:55:06] +[titan] 2025-07-09 18:46:35,644 - root - INFO - step: 23490 loss: 16.8128 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.2105 +[titan] 2025-07-09 18:46:35,644 - root - INFO - lr: 2.6705e-04 gnorm: 0.72 [ 5:11:38<16:55:02] +[titan] 2025-07-09 18:46:39,577 - root - INFO - step: 23495 loss: 16.5819 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0168 +[titan] 2025-07-09 18:46:39,578 - root - INFO - lr: 2.6704e-04 gnorm: 0.77 [ 5:11:42<16:54:58] +[titan] 2025-07-09 18:46:42,725 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:46:43,519 - root - INFO - step: 23500 loss: 16.8643 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.6309 global_avg_mtp_loss: 14.2334 +[titan] 2025-07-09 18:46:43,519 - root - INFO - lr: 2.6702e-04 gnorm: 0.72 [ 5:11:46<16:54:54] +[titan] 2025-07-09 18:46:47,418 - root - INFO - step: 23505 loss: 16.6389 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.0624 +[titan] 2025-07-09 18:46:47,418 - root - INFO - lr: 2.6701e-04 gnorm: 0.80 [ 5:11:49<16:54:49] +[titan] 2025-07-09 18:46:51,312 - root - INFO - step: 23510 loss: 16.9504 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6469 global_avg_mtp_loss: 14.3035 +[titan] 2025-07-09 18:46:51,312 - root - INFO - lr: 2.6700e-04 gnorm: 0.78 [ 5:11:53<16:54:45] +[titan] 2025-07-09 18:46:55,246 - root - INFO - step: 23515 loss: 16.6508 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5827 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-09 18:46:55,247 - root - INFO - lr: 2.6698e-04 gnorm: 0.75 [ 5:11:57<16:54:41] +[titan] 2025-07-09 18:46:59,150 - root - INFO - step: 23520 loss: 16.8601 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6258 global_avg_mtp_loss: 14.2343 +[titan] 2025-07-09 18:46:59,150 - root - INFO - lr: 2.6697e-04 gnorm: 0.78 [ 5:12:01<16:54:37] +[titan] 2025-07-09 18:47:03,063 - root - INFO - step: 23525 loss: 16.9324 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.3065 +[titan] 2025-07-09 18:47:03,063 - root - INFO - lr: 2.6695e-04 gnorm: 0.78 [ 5:12:05<16:54:33] +[titan] 2025-07-09 18:47:06,981 - root - INFO - step: 23530 loss: 17.0585 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3948 +[titan] 2025-07-09 18:47:06,982 - root - INFO - lr: 2.6694e-04 gnorm: 0.70 [ 5:12:09<16:54:28] +[titan] 2025-07-09 18:47:10,922 - root - INFO - step: 23535 loss: 16.8569 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.2283 +[titan] 2025-07-09 18:47:10,922 - root - INFO - lr: 2.6693e-04 gnorm: 0.77 [ 5:12:13<16:54:24] +[titan] 2025-07-09 18:47:14,875 - root - INFO - step: 23540 loss: 16.9340 memory: 44.58GiB(31.99%) tps: 82,897 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.6418 global_avg_mtp_loss: 14.2922 +[titan] 2025-07-09 18:47:14,875 - root - INFO - lr: 2.6691e-04 gnorm: 0.77 [ 5:12:17<16:54:20] +[titan] 2025-07-09 18:47:18,790 - root - INFO - step: 23545 loss: 16.9160 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6357 global_avg_mtp_loss: 14.2804 +[titan] 2025-07-09 18:47:18,790 - root - INFO - lr: 2.6690e-04 gnorm: 0.80 [ 5:12:21<16:54:16] +[titan] 2025-07-09 18:47:21,994 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:47:22,788 - root - INFO - step: 23550 loss: 16.6494 memory: 44.58GiB(31.99%) tps: 81,958 tflops: 282.85 mfu: 28.60% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.0619 +[titan] 2025-07-09 18:47:22,789 - root - INFO - lr: 2.6688e-04 gnorm: 0.78 [ 5:12:25<16:54:12] +[titan] 2025-07-09 18:47:24,506 - root - INFO - Dumping profiler traces at step 23552 +[titan] 2025-07-09 18:47:24,537 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:47:26,895 - root - INFO - step: 23555 loss: 16.9634 memory: 44.58GiB(31.99%) tps: 79,795 tflops: 275.39 mfu: 27.84% global_avg_ntp_loss: 2.6324 global_avg_mtp_loss: 14.3310 +[titan] 2025-07-09 18:47:26,896 - root - INFO - lr: 2.6687e-04 gnorm: 0.82 [ 5:12:29<16:54:08] +[titan] 2025-07-09 18:47:30,803 - root - INFO - step: 23560 loss: 17.0099 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6455 global_avg_mtp_loss: 14.3644 +[titan] 2025-07-09 18:47:30,803 - root - INFO - lr: 2.6686e-04 gnorm: 0.78 [ 5:12:33<16:54:04] +[titan] 2025-07-09 18:47:34,711 - root - INFO - step: 23565 loss: 17.1375 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6816 global_avg_mtp_loss: 14.4559 +[titan] 2025-07-09 18:47:34,711 - root - INFO - lr: 2.6684e-04 gnorm: 0.81 [ 5:12:37<16:54:00] +[titan] 2025-07-09 18:47:38,598 - root - INFO - step: 23570 loss: 16.7456 memory: 44.58GiB(31.99%) tps: 84,310 tflops: 290.97 mfu: 29.42% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1536 +[titan] 2025-07-09 18:47:38,598 - root - INFO - lr: 2.6683e-04 gnorm: 0.85 [ 5:12:41<16:53:56] +[titan] 2025-07-09 18:47:42,543 - root - INFO - step: 23575 loss: 17.0375 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.6495 global_avg_mtp_loss: 14.3879 +[titan] 2025-07-09 18:47:42,543 - root - INFO - lr: 2.6681e-04 gnorm: 0.81 [ 5:12:45<16:53:52] +[titan] 2025-07-09 18:47:46,456 - root - INFO - step: 23580 loss: 16.7326 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.1285 +[titan] 2025-07-09 18:47:46,456 - root - INFO - lr: 2.6680e-04 gnorm: 0.77 [ 5:12:48<16:53:47] +[titan] 2025-07-09 18:47:50,405 - root - INFO - step: 23585 loss: 16.7271 memory: 44.58GiB(31.99%) tps: 82,986 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5880 global_avg_mtp_loss: 14.1391 +[titan] 2025-07-09 18:47:50,405 - root - INFO - lr: 2.6679e-04 gnorm: 0.73 [ 5:12:52<16:53:43] +[titan] 2025-07-09 18:47:54,321 - root - INFO - step: 23590 loss: 16.7614 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6057 global_avg_mtp_loss: 14.1557 +[titan] 2025-07-09 18:47:54,321 - root - INFO - lr: 2.6677e-04 gnorm: 0.80 [ 5:12:56<16:53:39] +[titan] 2025-07-09 18:47:58,246 - root - INFO - step: 23595 loss: 17.0802 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4142 +[titan] 2025-07-09 18:47:58,246 - root - INFO - lr: 2.6676e-04 gnorm: 0.79 [ 5:13:00<16:53:35] +[titan] 2025-07-09 18:48:01,371 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:48:02,170 - root - INFO - step: 23600 loss: 16.9396 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6257 global_avg_mtp_loss: 14.3139 +[titan] 2025-07-09 18:48:02,170 - root - INFO - lr: 2.6674e-04 gnorm: 0.71 [ 5:13:04<16:53:31] +[titan] 2025-07-09 18:48:06,102 - root - INFO - step: 23605 loss: 16.8205 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.6180 global_avg_mtp_loss: 14.2025 +[titan] 2025-07-09 18:48:06,103 - root - INFO - lr: 2.6673e-04 gnorm: 0.81 [ 5:13:08<16:53:27] +[titan] 2025-07-09 18:48:10,010 - root - INFO - step: 23610 loss: 16.9495 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6409 global_avg_mtp_loss: 14.3086 +[titan] 2025-07-09 18:48:10,010 - root - INFO - lr: 2.6671e-04 gnorm: 0.77 [ 5:13:12<16:53:23] +[titan] 2025-07-09 18:48:13,990 - root - INFO - step: 23615 loss: 16.7623 memory: 44.58GiB(31.99%) tps: 82,342 tflops: 284.18 mfu: 28.73% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1719 +[titan] 2025-07-09 18:48:13,990 - root - INFO - lr: 2.6670e-04 gnorm: 0.78 [ 5:13:16<16:53:19] +[titan] 2025-07-09 18:48:17,938 - root - INFO - step: 23620 loss: 16.8858 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.2615 +[titan] 2025-07-09 18:48:17,938 - root - INFO - lr: 2.6669e-04 gnorm: 0.82 [ 5:13:20<16:53:14] +[titan] 2025-07-09 18:48:21,875 - root - INFO - step: 23625 loss: 16.7320 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.1267 +[titan] 2025-07-09 18:48:21,876 - root - INFO - lr: 2.6667e-04 gnorm: 0.78 [ 5:13:24<16:53:10] +[titan] 2025-07-09 18:48:25,774 - root - INFO - step: 23630 loss: 16.9570 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6462 global_avg_mtp_loss: 14.3108 +[titan] 2025-07-09 18:48:25,774 - root - INFO - lr: 2.6666e-04 gnorm: 0.74 [ 5:13:28<16:53:06] +[titan] 2025-07-09 18:48:29,723 - root - INFO - step: 23635 loss: 16.8354 memory: 44.58GiB(31.99%) tps: 82,989 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2226 +[titan] 2025-07-09 18:48:29,723 - root - INFO - lr: 2.6664e-04 gnorm: 0.77 [ 5:13:32<16:53:02] +[titan] 2025-07-09 18:48:33,651 - root - INFO - step: 23640 loss: 17.1272 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6696 global_avg_mtp_loss: 14.4575 +[titan] 2025-07-09 18:48:33,652 - root - INFO - lr: 2.6663e-04 gnorm: 0.98 [ 5:13:36<16:52:58] +[titan] 2025-07-09 18:48:37,565 - root - INFO - step: 23645 loss: 17.0448 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6602 global_avg_mtp_loss: 14.3846 +[titan] 2025-07-09 18:48:37,565 - root - INFO - lr: 2.6662e-04 gnorm: 0.75 [ 5:13:40<16:52:54] +[titan] 2025-07-09 18:48:40,676 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:48:41,468 - root - INFO - step: 23650 loss: 17.0202 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.3780 +[titan] 2025-07-09 18:48:41,468 - root - INFO - lr: 2.6660e-04 gnorm: 0.83 [ 5:13:43<16:52:49] +[titan] 2025-07-09 18:48:45,360 - root - INFO - step: 23655 loss: 17.0000 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.3587 +[titan] 2025-07-09 18:48:45,360 - root - INFO - lr: 2.6659e-04 gnorm: 0.74 [ 5:13:47<16:52:45] +[titan] 2025-07-09 18:48:49,264 - root - INFO - step: 23660 loss: 17.1283 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6711 global_avg_mtp_loss: 14.4572 +[titan] 2025-07-09 18:48:49,264 - root - INFO - lr: 2.6657e-04 gnorm: 0.85 [ 5:13:51<16:52:41] +[titan] 2025-07-09 18:48:53,184 - root - INFO - step: 23665 loss: 16.8556 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6253 global_avg_mtp_loss: 14.2303 +[titan] 2025-07-09 18:48:53,184 - root - INFO - lr: 2.6656e-04 gnorm: 0.76 [ 5:13:55<16:52:37] +[titan] 2025-07-09 18:48:57,108 - root - INFO - step: 23670 loss: 16.8721 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.2567 +[titan] 2025-07-09 18:48:57,108 - root - INFO - lr: 2.6655e-04 gnorm: 0.80 [ 5:13:59<16:52:33] +[titan] 2025-07-09 18:49:01,019 - root - INFO - step: 23675 loss: 16.8564 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6171 global_avg_mtp_loss: 14.2393 +[titan] 2025-07-09 18:49:01,019 - root - INFO - lr: 2.6653e-04 gnorm: 0.80 [ 5:14:03<16:52:28] +[titan] 2025-07-09 18:49:04,926 - root - INFO - step: 23680 loss: 17.0285 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6616 global_avg_mtp_loss: 14.3668 +[titan] 2025-07-09 18:49:04,926 - root - INFO - lr: 2.6652e-04 gnorm: 0.81 [ 5:14:07<16:52:24] +[titan] 2025-07-09 18:49:08,880 - root - INFO - step: 23685 loss: 16.7797 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.1744 +[titan] 2025-07-09 18:49:08,880 - root - INFO - lr: 2.6650e-04 gnorm: 0.79 [ 5:14:11<16:52:20] +[titan] 2025-07-09 18:49:12,868 - root - INFO - step: 23690 loss: 16.7333 memory: 44.58GiB(31.99%) tps: 82,177 tflops: 283.61 mfu: 28.68% global_avg_ntp_loss: 2.5983 global_avg_mtp_loss: 14.1350 +[titan] 2025-07-09 18:49:12,868 - root - INFO - lr: 2.6649e-04 gnorm: 0.79 [ 5:14:15<16:52:16] +[titan] 2025-07-09 18:49:16,778 - root - INFO - step: 23695 loss: 16.5526 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 13.9870 +[titan] 2025-07-09 18:49:16,778 - root - INFO - lr: 2.6648e-04 gnorm: 0.78 [ 5:14:19<16:52:12] +[titan] 2025-07-09 18:49:19,962 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:49:20,746 - root - INFO - step: 23700 loss: 17.0178 memory: 44.58GiB(31.99%) tps: 82,579 tflops: 284.99 mfu: 28.82% global_avg_ntp_loss: 2.6505 global_avg_mtp_loss: 14.3673 +[titan] 2025-07-09 18:49:20,746 - root - INFO - lr: 2.6646e-04 gnorm: 0.75 [ 5:14:23<16:52:08] +[titan] 2025-07-09 18:49:24,690 - root - INFO - step: 23705 loss: 16.7553 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6019 global_avg_mtp_loss: 14.1534 +[titan] 2025-07-09 18:49:24,691 - root - INFO - lr: 2.6645e-04 gnorm: 0.79 [ 5:14:27<16:52:04] +[titan] 2025-07-09 18:49:28,647 - root - INFO - step: 23710 loss: 16.7098 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1180 +[titan] 2025-07-09 18:49:28,647 - root - INFO - lr: 2.6643e-04 gnorm: 0.77 [ 5:14:31<16:52:00] +[titan] 2025-07-09 18:49:32,542 - root - INFO - step: 23715 loss: 16.9135 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2936 +[titan] 2025-07-09 18:49:32,542 - root - INFO - lr: 2.6642e-04 gnorm: 0.81 [ 5:14:35<16:51:56] +[titan] 2025-07-09 18:49:36,446 - root - INFO - step: 23720 loss: 16.9018 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6329 global_avg_mtp_loss: 14.2689 +[titan] 2025-07-09 18:49:36,447 - root - INFO - lr: 2.6640e-04 gnorm: 0.77 [ 5:14:38<16:51:51] +[titan] 2025-07-09 18:49:40,369 - root - INFO - step: 23725 loss: 17.0156 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6448 global_avg_mtp_loss: 14.3707 +[titan] 2025-07-09 18:49:40,369 - root - INFO - lr: 2.6639e-04 gnorm: 0.74 [ 5:14:42<16:51:47] +[titan] 2025-07-09 18:49:44,311 - root - INFO - step: 23730 loss: 16.9180 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.2873 +[titan] 2025-07-09 18:49:44,311 - root - INFO - lr: 2.6638e-04 gnorm: 0.72 [ 5:14:46<16:51:43] +[titan] 2025-07-09 18:49:48,217 - root - INFO - step: 23735 loss: 16.9455 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.3123 +[titan] 2025-07-09 18:49:48,217 - root - INFO - lr: 2.6636e-04 gnorm: 0.79 [ 5:14:50<16:51:39] +[titan] 2025-07-09 18:49:52,180 - root - INFO - step: 23740 loss: 16.9016 memory: 44.58GiB(31.99%) tps: 82,697 tflops: 285.40 mfu: 28.86% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.2685 +[titan] 2025-07-09 18:49:52,180 - root - INFO - lr: 2.6635e-04 gnorm: 0.77 [ 5:14:54<16:51:35] +[titan] 2025-07-09 18:49:56,139 - root - INFO - step: 23745 loss: 16.9227 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2958 +[titan] 2025-07-09 18:49:56,139 - root - INFO - lr: 2.6633e-04 gnorm: 0.73 [ 5:14:58<16:51:31] +[titan] 2025-07-09 18:49:59,257 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:50:00,047 - root - INFO - step: 23750 loss: 17.0470 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6445 global_avg_mtp_loss: 14.4025 +[titan] 2025-07-09 18:50:00,047 - root - INFO - lr: 2.6632e-04 gnorm: 0.77 [ 5:15:02<16:51:27] +[titan] 2025-07-09 18:50:03,957 - root - INFO - step: 23755 loss: 16.7371 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.1513 +[titan] 2025-07-09 18:50:03,957 - root - INFO - lr: 2.6631e-04 gnorm: 0.83 [ 5:15:06<16:51:22] +[titan] 2025-07-09 18:50:07,855 - root - INFO - step: 23760 loss: 16.6618 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.0738 +[titan] 2025-07-09 18:50:07,856 - root - INFO - lr: 2.6629e-04 gnorm: 0.80 [ 5:15:10<16:51:18] +[titan] 2025-07-09 18:50:11,754 - root - INFO - step: 23765 loss: 16.8585 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2386 +[titan] 2025-07-09 18:50:11,754 - root - INFO - lr: 2.6628e-04 gnorm: 0.81 [ 5:15:14<16:51:14] +[titan] 2025-07-09 18:50:15,670 - root - INFO - step: 23770 loss: 16.9873 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6499 global_avg_mtp_loss: 14.3373 +[titan] 2025-07-09 18:50:15,670 - root - INFO - lr: 2.6626e-04 gnorm: 0.75 [ 5:15:18<16:51:10] +[titan] 2025-07-09 18:50:19,561 - root - INFO - step: 23775 loss: 16.9254 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.6405 global_avg_mtp_loss: 14.2849 +[titan] 2025-07-09 18:50:19,562 - root - INFO - lr: 2.6625e-04 gnorm: 0.81 [ 5:15:22<16:51:05] +[titan] 2025-07-09 18:50:23,458 - root - INFO - step: 23780 loss: 16.9668 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6353 global_avg_mtp_loss: 14.3314 +[titan] 2025-07-09 18:50:23,458 - root - INFO - lr: 2.6623e-04 gnorm: 0.81 [ 5:15:25<16:51:01] +[titan] 2025-07-09 18:50:27,371 - root - INFO - step: 23785 loss: 17.1182 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6598 global_avg_mtp_loss: 14.4584 +[titan] 2025-07-09 18:50:27,371 - root - INFO - lr: 2.6622e-04 gnorm: 0.81 [ 5:15:29<16:50:57] +[titan] 2025-07-09 18:50:31,285 - root - INFO - step: 23790 loss: 16.9360 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.6379 global_avg_mtp_loss: 14.2982 +[titan] 2025-07-09 18:50:31,285 - root - INFO - lr: 2.6621e-04 gnorm: 0.78 [ 5:15:33<16:50:53] +[titan] 2025-07-09 18:50:35,234 - root - INFO - step: 23795 loss: 16.8617 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.2357 +[titan] 2025-07-09 18:50:35,234 - root - INFO - lr: 2.6619e-04 gnorm: 0.76 [ 5:15:37<16:50:49] +[titan] 2025-07-09 18:50:38,373 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:50:39,160 - root - INFO - step: 23800 loss: 16.7785 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.05 mfu: 29.12% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.1737 +[titan] 2025-07-09 18:50:39,160 - root - INFO - lr: 2.6618e-04 gnorm: 0.75 [ 5:15:41<16:50:45] +[titan] 2025-07-09 18:50:43,051 - root - INFO - step: 23805 loss: 16.7755 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.1726 +[titan] 2025-07-09 18:50:43,051 - root - INFO - lr: 2.6616e-04 gnorm: 0.82 [ 5:15:45<16:50:40] +[titan] 2025-07-09 18:50:46,938 - root - INFO - step: 23810 loss: 16.8999 memory: 44.58GiB(31.99%) tps: 84,304 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.6306 global_avg_mtp_loss: 14.2693 +[titan] 2025-07-09 18:50:46,938 - root - INFO - lr: 2.6615e-04 gnorm: 0.81 [ 5:15:49<16:50:36] +[titan] 2025-07-09 18:50:50,835 - root - INFO - step: 23815 loss: 16.8757 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.6220 global_avg_mtp_loss: 14.2537 +[titan] 2025-07-09 18:50:50,835 - root - INFO - lr: 2.6614e-04 gnorm: 0.80 [ 5:15:53<16:50:32] +[titan] 2025-07-09 18:50:54,779 - root - INFO - step: 23820 loss: 17.1718 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.6816 global_avg_mtp_loss: 14.4902 +[titan] 2025-07-09 18:50:54,780 - root - INFO - lr: 2.6612e-04 gnorm: 0.80 [ 5:15:57<16:50:28] +[titan] 2025-07-09 18:50:58,691 - root - INFO - step: 23825 loss: 16.8123 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6042 global_avg_mtp_loss: 14.2081 +[titan] 2025-07-09 18:50:58,692 - root - INFO - lr: 2.6611e-04 gnorm: 0.75 [ 5:16:01<16:50:24] +[titan] 2025-07-09 18:51:02,621 - root - INFO - step: 23830 loss: 16.6215 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-09 18:51:02,621 - root - INFO - lr: 2.6609e-04 gnorm: 0.79 [ 5:16:05<16:50:19] +[titan] 2025-07-09 18:51:06,521 - root - INFO - step: 23835 loss: 16.7729 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5950 global_avg_mtp_loss: 14.1779 +[titan] 2025-07-09 18:51:06,521 - root - INFO - lr: 2.6608e-04 gnorm: 0.73 [ 5:16:09<16:50:15] +[titan] 2025-07-09 18:51:10,436 - root - INFO - step: 23840 loss: 16.8534 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6179 global_avg_mtp_loss: 14.2355 +[titan] 2025-07-09 18:51:10,436 - root - INFO - lr: 2.6606e-04 gnorm: 0.76 [ 5:16:12<16:50:11] +[titan] 2025-07-09 18:51:14,396 - root - INFO - step: 23845 loss: 16.9234 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.2808 +[titan] 2025-07-09 18:51:14,396 - root - INFO - lr: 2.6605e-04 gnorm: 0.75 [ 5:16:16<16:50:07] +[titan] 2025-07-09 18:51:17,513 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:51:18,335 - root - INFO - step: 23850 loss: 16.8656 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.6201 global_avg_mtp_loss: 14.2455 +[titan] 2025-07-09 18:51:18,335 - root - INFO - lr: 2.6604e-04 gnorm: 0.80 [ 5:16:20<16:50:03] +[titan] 2025-07-09 18:51:22,225 - root - INFO - step: 23855 loss: 16.8376 memory: 44.58GiB(31.99%) tps: 84,230 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2191 +[titan] 2025-07-09 18:51:22,226 - root - INFO - lr: 2.6602e-04 gnorm: 0.75 [ 5:16:24<16:49:59] +[titan] 2025-07-09 18:51:26,128 - root - INFO - step: 23860 loss: 16.8464 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6242 global_avg_mtp_loss: 14.2222 +[titan] 2025-07-09 18:51:26,128 - root - INFO - lr: 2.6601e-04 gnorm: 0.81 [ 5:16:28<16:49:54] +[titan] 2025-07-09 18:51:30,105 - root - INFO - step: 23865 loss: 16.7096 memory: 44.58GiB(31.99%) tps: 82,392 tflops: 284.35 mfu: 28.75% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1252 +[titan] 2025-07-09 18:51:30,105 - root - INFO - lr: 2.6599e-04 gnorm: 0.76 [ 5:16:32<16:49:50] +[titan] 2025-07-09 18:51:34,023 - root - INFO - step: 23870 loss: 16.7944 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6036 global_avg_mtp_loss: 14.1907 +[titan] 2025-07-09 18:51:34,023 - root - INFO - lr: 2.6598e-04 gnorm: 0.78 [ 5:16:36<16:49:46] +[titan] 2025-07-09 18:51:37,949 - root - INFO - step: 23875 loss: 16.9956 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.3510 +[titan] 2025-07-09 18:51:37,949 - root - INFO - lr: 2.6596e-04 gnorm: 0.79 [ 5:16:40<16:49:42] +[titan] 2025-07-09 18:51:41,887 - root - INFO - step: 23880 loss: 17.0326 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.6502 global_avg_mtp_loss: 14.3824 +[titan] 2025-07-09 18:51:41,887 - root - INFO - lr: 2.6595e-04 gnorm: 0.77 [ 5:16:44<16:49:38] +[titan] 2025-07-09 18:51:45,812 - root - INFO - step: 23885 loss: 16.6604 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5917 global_avg_mtp_loss: 14.0687 +[titan] 2025-07-09 18:51:45,812 - root - INFO - lr: 2.6594e-04 gnorm: 0.87 [ 5:16:48<16:49:34] +[titan] 2025-07-09 18:51:49,706 - root - INFO - step: 23890 loss: 17.0661 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.6489 global_avg_mtp_loss: 14.4171 +[titan] 2025-07-09 18:51:49,706 - root - INFO - lr: 2.6592e-04 gnorm: 0.80 [ 5:16:52<16:49:29] +[titan] 2025-07-09 18:51:53,616 - root - INFO - step: 23895 loss: 16.9720 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6508 global_avg_mtp_loss: 14.3212 +[titan] 2025-07-09 18:51:53,617 - root - INFO - lr: 2.6591e-04 gnorm: 0.74 [ 5:16:56<16:49:25] +[titan] 2025-07-09 18:51:56,756 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:51:57,588 - root - INFO - step: 23900 loss: 16.8168 memory: 44.58GiB(31.99%) tps: 82,521 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.6186 global_avg_mtp_loss: 14.1983 +[titan] 2025-07-09 18:51:57,588 - root - INFO - lr: 2.6589e-04 gnorm: 0.83 [ 5:17:00<16:49:21] +[titan] 2025-07-09 18:52:01,543 - root - INFO - step: 23905 loss: 16.9557 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.6399 global_avg_mtp_loss: 14.3158 +[titan] 2025-07-09 18:52:01,543 - root - INFO - lr: 2.6588e-04 gnorm: 0.71 [ 5:17:04<16:49:17] +[titan] 2025-07-09 18:52:05,448 - root - INFO - step: 23910 loss: 16.6770 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.0912 +[titan] 2025-07-09 18:52:05,448 - root - INFO - lr: 2.6587e-04 gnorm: 0.76 [ 5:17:07<16:49:13] +[titan] 2025-07-09 18:52:09,352 - root - INFO - step: 23915 loss: 17.1329 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6697 global_avg_mtp_loss: 14.4631 +[titan] 2025-07-09 18:52:09,353 - root - INFO - lr: 2.6585e-04 gnorm: 0.80 [ 5:17:11<16:49:09] +[titan] 2025-07-09 18:52:13,305 - root - INFO - step: 23920 loss: 16.9321 memory: 44.58GiB(31.99%) tps: 82,907 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2986 +[titan] 2025-07-09 18:52:13,305 - root - INFO - lr: 2.6584e-04 gnorm: 0.75 [ 5:17:15<16:49:05] +[titan] 2025-07-09 18:52:17,259 - root - INFO - step: 23925 loss: 16.6508 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5783 global_avg_mtp_loss: 14.0725 +[titan] 2025-07-09 18:52:17,259 - root - INFO - lr: 2.6582e-04 gnorm: 0.78 [ 5:17:19<16:49:01] +[titan] 2025-07-09 18:52:21,168 - root - INFO - step: 23930 loss: 17.0357 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6510 global_avg_mtp_loss: 14.3847 +[titan] 2025-07-09 18:52:21,168 - root - INFO - lr: 2.6581e-04 gnorm: 0.82 [ 5:17:23<16:48:56] +[titan] 2025-07-09 18:52:25,131 - root - INFO - step: 23935 loss: 17.3892 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.7349 global_avg_mtp_loss: 14.6543 +[titan] 2025-07-09 18:52:25,132 - root - INFO - lr: 2.6579e-04 gnorm: 0.74 [ 5:17:27<16:48:52] +[titan] 2025-07-09 18:52:29,093 - root - INFO - step: 23940 loss: 16.8597 memory: 44.58GiB(31.99%) tps: 82,732 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2362 +[titan] 2025-07-09 18:52:29,093 - root - INFO - lr: 2.6578e-04 gnorm: 0.86 [ 5:17:31<16:48:48] +[titan] 2025-07-09 18:52:33,050 - root - INFO - step: 23945 loss: 17.0880 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.4252 +[titan] 2025-07-09 18:52:33,050 - root - INFO - lr: 2.6577e-04 gnorm: 0.75 [ 5:17:35<16:48:44] +[titan] 2025-07-09 18:52:36,190 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:52:36,978 - root - INFO - step: 23950 loss: 16.7992 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6118 global_avg_mtp_loss: 14.1875 +[titan] 2025-07-09 18:52:36,978 - root - INFO - lr: 2.6575e-04 gnorm: 0.86 [ 5:17:39<16:48:40] +[titan] 2025-07-09 18:52:40,874 - root - INFO - step: 23955 loss: 17.0334 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6421 global_avg_mtp_loss: 14.3914 +[titan] 2025-07-09 18:52:40,874 - root - INFO - lr: 2.6574e-04 gnorm: 0.89 [ 5:17:43<16:48:36] +[titan] 2025-07-09 18:52:44,786 - root - INFO - step: 23960 loss: 17.1382 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6693 global_avg_mtp_loss: 14.4690 +[titan] 2025-07-09 18:52:44,786 - root - INFO - lr: 2.6572e-04 gnorm: 0.81 [ 5:17:47<16:48:32] +[titan] 2025-07-09 18:52:48,717 - root - INFO - step: 23965 loss: 16.7231 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1358 +[titan] 2025-07-09 18:52:48,717 - root - INFO - lr: 2.6571e-04 gnorm: 0.81 [ 5:17:51<16:48:28] +[titan] 2025-07-09 18:52:52,615 - root - INFO - step: 23970 loss: 16.4625 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 13.9097 +[titan] 2025-07-09 18:52:52,615 - root - INFO - lr: 2.6569e-04 gnorm: 0.81 [ 5:17:55<16:48:23] +[titan] 2025-07-09 18:52:56,519 - root - INFO - step: 23975 loss: 16.9665 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6274 global_avg_mtp_loss: 14.3391 +[titan] 2025-07-09 18:52:56,519 - root - INFO - lr: 2.6568e-04 gnorm: 0.82 [ 5:17:58<16:48:19] +[titan] 2025-07-09 18:53:00,427 - root - INFO - step: 23980 loss: 16.8244 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6166 global_avg_mtp_loss: 14.2079 +[titan] 2025-07-09 18:53:00,427 - root - INFO - lr: 2.6567e-04 gnorm: 0.76 [ 5:18:02<16:48:15] +[titan] 2025-07-09 18:53:04,329 - root - INFO - step: 23985 loss: 16.6477 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.0710 +[titan] 2025-07-09 18:53:04,329 - root - INFO - lr: 2.6565e-04 gnorm: 0.77 [ 5:18:06<16:48:11] +[titan] 2025-07-09 18:53:08,228 - root - INFO - step: 23990 loss: 16.8793 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.2532 +[titan] 2025-07-09 18:53:08,228 - root - INFO - lr: 2.6564e-04 gnorm: 0.80 [ 5:18:10<16:48:07] +[titan] 2025-07-09 18:53:12,123 - root - INFO - step: 23995 loss: 16.8936 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2676 +[titan] 2025-07-09 18:53:12,123 - root - INFO - lr: 2.6562e-04 gnorm: 0.79 [ 5:18:14<16:48:02] +[titan] 2025-07-09 18:53:15,230 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:53:16,018 - root - INFO - step: 24000 loss: 16.9623 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.3297 +[titan] 2025-07-09 18:53:16,018 - root - INFO - lr: 2.6561e-04 gnorm: 0.77 [ 5:18:18<16:47:58] +[titan] 2025-07-09 18:53:19,911 - root - INFO - step: 24005 loss: 16.9529 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.3010 +[titan] 2025-07-09 18:53:19,911 - root - INFO - lr: 2.6559e-04 gnorm: 0.84 [ 5:18:22<16:47:54] +[titan] 2025-07-09 18:53:23,810 - root - INFO - step: 24010 loss: 16.9025 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6273 global_avg_mtp_loss: 14.2752 +[titan] 2025-07-09 18:53:23,811 - root - INFO - lr: 2.6558e-04 gnorm: 0.74 [ 5:18:26<16:47:50] +[titan] 2025-07-09 18:53:27,746 - root - INFO - step: 24015 loss: 16.9799 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6441 global_avg_mtp_loss: 14.3358 +[titan] 2025-07-09 18:53:27,746 - root - INFO - lr: 2.6557e-04 gnorm: 0.71 [ 5:18:30<16:47:45] +[titan] 2025-07-09 18:53:31,661 - root - INFO - step: 24020 loss: 16.6092 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0354 +[titan] 2025-07-09 18:53:31,661 - root - INFO - lr: 2.6555e-04 gnorm: 0.81 [ 5:18:34<16:47:41] +[titan] 2025-07-09 18:53:35,561 - root - INFO - step: 24025 loss: 16.7877 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.1796 +[titan] 2025-07-09 18:53:35,562 - root - INFO - lr: 2.6554e-04 gnorm: 0.78 [ 5:18:38<16:47:37] +[titan] 2025-07-09 18:53:39,470 - root - INFO - step: 24030 loss: 16.8214 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6102 global_avg_mtp_loss: 14.2112 +[titan] 2025-07-09 18:53:39,470 - root - INFO - lr: 2.6552e-04 gnorm: 0.82 [ 5:18:41<16:47:33] +[titan] 2025-07-09 18:53:43,364 - root - INFO - step: 24035 loss: 16.8521 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6190 global_avg_mtp_loss: 14.2331 +[titan] 2025-07-09 18:53:43,364 - root - INFO - lr: 2.6551e-04 gnorm: 0.78 [ 5:18:45<16:47:29] +[titan] 2025-07-09 18:53:47,248 - root - INFO - step: 24040 loss: 16.9405 memory: 44.58GiB(31.99%) tps: 84,362 tflops: 291.15 mfu: 29.44% global_avg_ntp_loss: 2.6389 global_avg_mtp_loss: 14.3016 +[titan] 2025-07-09 18:53:47,249 - root - INFO - lr: 2.6549e-04 gnorm: 0.79 [ 5:18:49<16:47:24] +[titan] 2025-07-09 18:53:51,153 - root - INFO - step: 24045 loss: 16.9030 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6267 global_avg_mtp_loss: 14.2763 +[titan] 2025-07-09 18:53:51,153 - root - INFO - lr: 2.6548e-04 gnorm: 0.79 [ 5:18:53<16:47:20] +[titan] 2025-07-09 18:53:54,269 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:53:55,056 - root - INFO - step: 24050 loss: 16.9772 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6348 global_avg_mtp_loss: 14.3424 +[titan] 2025-07-09 18:53:55,056 - root - INFO - lr: 2.6547e-04 gnorm: 0.77 [ 5:18:57<16:47:16] +[titan] 2025-07-09 18:53:58,998 - root - INFO - step: 24055 loss: 16.9476 memory: 44.58GiB(31.99%) tps: 83,127 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.6478 global_avg_mtp_loss: 14.2999 +[titan] 2025-07-09 18:53:58,998 - root - INFO - lr: 2.6545e-04 gnorm: 0.77 [ 5:19:01<16:47:12] +[titan] 2025-07-09 18:54:02,918 - root - INFO - step: 24060 loss: 16.7757 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1815 +[titan] 2025-07-09 18:54:02,918 - root - INFO - lr: 2.6544e-04 gnorm: 0.82 [ 5:19:05<16:47:08] +[titan] 2025-07-09 18:54:06,263 - root - INFO - Dumping profiler traces at step 24064 +[titan] 2025-07-09 18:54:06,296 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 18:54:07,083 - root - INFO - step: 24065 loss: 16.7364 memory: 44.58GiB(31.99%) tps: 78,676 tflops: 271.52 mfu: 27.45% global_avg_ntp_loss: 2.5941 global_avg_mtp_loss: 14.1423 +[titan] 2025-07-09 18:54:07,083 - root - INFO - lr: 2.6542e-04 gnorm: 0.80 [ 5:19:09<16:47:04] +[titan] 2025-07-09 18:54:10,989 - root - INFO - step: 24070 loss: 16.5940 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 14.0311 +[titan] 2025-07-09 18:54:10,989 - root - INFO - lr: 2.6541e-04 gnorm: 0.83 [ 5:19:13<16:47:00] +[titan] 2025-07-09 18:54:14,912 - root - INFO - step: 24075 loss: 16.8951 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.2749 +[titan] 2025-07-09 18:54:14,912 - root - INFO - lr: 2.6539e-04 gnorm: 0.90 [ 5:19:17<16:46:56] +[titan] 2025-07-09 18:54:18,823 - root - INFO - step: 24080 loss: 16.9624 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.3159 +[titan] 2025-07-09 18:54:18,824 - root - INFO - lr: 2.6538e-04 gnorm: 0.88 [ 5:19:21<16:46:52] +[titan] 2025-07-09 18:54:22,755 - root - INFO - step: 24085 loss: 16.8155 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.2114 +[titan] 2025-07-09 18:54:22,755 - root - INFO - lr: 2.6537e-04 gnorm: 0.75 [ 5:19:25<16:46:48] +[titan] 2025-07-09 18:54:26,684 - root - INFO - step: 24090 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1253 +[titan] 2025-07-09 18:54:26,684 - root - INFO - lr: 2.6535e-04 gnorm: 0.79 [ 5:19:29<16:46:43] +[titan] 2025-07-09 18:54:30,602 - root - INFO - step: 24095 loss: 17.0097 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.3730 +[titan] 2025-07-09 18:54:30,603 - root - INFO - lr: 2.6534e-04 gnorm: 0.75 [ 5:19:33<16:46:39] +[titan] 2025-07-09 18:54:33,745 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:54:34,529 - root - INFO - step: 24100 loss: 16.8375 memory: 44.58GiB(31.99%) tps: 83,451 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2237 +[titan] 2025-07-09 18:54:34,530 - root - INFO - lr: 2.6532e-04 gnorm: 0.84 [ 5:19:36<16:46:35] +[titan] 2025-07-09 18:54:38,430 - root - INFO - step: 24105 loss: 17.2097 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.6924 global_avg_mtp_loss: 14.5173 +[titan] 2025-07-09 18:54:38,430 - root - INFO - lr: 2.6531e-04 gnorm: 0.76 [ 5:19:40<16:46:31] +[titan] 2025-07-09 18:54:42,378 - root - INFO - step: 24110 loss: 16.9713 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6552 global_avg_mtp_loss: 14.3160 +[titan] 2025-07-09 18:54:42,379 - root - INFO - lr: 2.6529e-04 gnorm: 0.78 [ 5:19:44<16:46:27] +[titan] 2025-07-09 18:54:46,306 - root - INFO - step: 24115 loss: 17.0554 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.6757 global_avg_mtp_loss: 14.3798 +[titan] 2025-07-09 18:54:46,306 - root - INFO - lr: 2.6528e-04 gnorm: 0.80 [ 5:19:48<16:46:23] +[titan] 2025-07-09 18:54:50,206 - root - INFO - step: 24120 loss: 16.9224 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6434 global_avg_mtp_loss: 14.2790 +[titan] 2025-07-09 18:54:50,206 - root - INFO - lr: 2.6527e-04 gnorm: 0.74 [ 5:19:52<16:46:18] +[titan] 2025-07-09 18:54:54,127 - root - INFO - step: 24125 loss: 16.9276 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.2926 +[titan] 2025-07-09 18:54:54,127 - root - INFO - lr: 2.6525e-04 gnorm: 0.87 [ 5:19:56<16:46:14] +[titan] 2025-07-09 18:54:58,028 - root - INFO - step: 24130 loss: 16.9640 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6638 global_avg_mtp_loss: 14.3002 +[titan] 2025-07-09 18:54:58,029 - root - INFO - lr: 2.6524e-04 gnorm: 0.86 [ 5:20:00<16:46:10] +[titan] 2025-07-09 18:55:01,960 - root - INFO - step: 24135 loss: 17.0156 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6584 global_avg_mtp_loss: 14.3571 +[titan] 2025-07-09 18:55:01,960 - root - INFO - lr: 2.6522e-04 gnorm: 0.79 [ 5:20:04<16:46:06] +[titan] 2025-07-09 18:55:05,869 - root - INFO - step: 24140 loss: 16.8373 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.2209 +[titan] 2025-07-09 18:55:05,869 - root - INFO - lr: 2.6521e-04 gnorm: 0.73 [ 5:20:08<16:46:02] +[titan] 2025-07-09 18:55:09,782 - root - INFO - step: 24145 loss: 16.5642 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 13.9955 +[titan] 2025-07-09 18:55:09,782 - root - INFO - lr: 2.6519e-04 gnorm: 0.84 [ 5:20:12<16:45:57] +[titan] 2025-07-09 18:55:12,900 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:55:13,697 - root - INFO - step: 24150 loss: 16.9716 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.3251 +[titan] 2025-07-09 18:55:13,697 - root - INFO - lr: 2.6518e-04 gnorm: 0.75 [ 5:20:16<16:45:53] +[titan] 2025-07-09 18:55:17,611 - root - INFO - step: 24155 loss: 16.8502 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6203 global_avg_mtp_loss: 14.2299 +[titan] 2025-07-09 18:55:17,611 - root - INFO - lr: 2.6516e-04 gnorm: 0.73 [ 5:20:20<16:45:49] +[titan] 2025-07-09 18:55:21,524 - root - INFO - step: 24160 loss: 16.8226 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6139 global_avg_mtp_loss: 14.2088 +[titan] 2025-07-09 18:55:21,525 - root - INFO - lr: 2.6515e-04 gnorm: 0.76 [ 5:20:23<16:45:45] +[titan] 2025-07-09 18:55:25,420 - root - INFO - step: 24165 loss: 16.9876 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6531 global_avg_mtp_loss: 14.3345 +[titan] 2025-07-09 18:55:25,420 - root - INFO - lr: 2.6514e-04 gnorm: 0.79 [ 5:20:27<16:45:41] +[titan] 2025-07-09 18:55:29,350 - root - INFO - step: 24170 loss: 16.8123 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6015 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 18:55:29,350 - root - INFO - lr: 2.6512e-04 gnorm: 0.76 [ 5:20:31<16:45:37] +[titan] 2025-07-09 18:55:33,258 - root - INFO - step: 24175 loss: 16.8823 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6354 global_avg_mtp_loss: 14.2469 +[titan] 2025-07-09 18:55:33,258 - root - INFO - lr: 2.6511e-04 gnorm: 0.76 [ 5:20:35<16:45:32] +[titan] 2025-07-09 18:55:37,169 - root - INFO - step: 24180 loss: 17.2230 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.7004 global_avg_mtp_loss: 14.5227 +[titan] 2025-07-09 18:55:37,169 - root - INFO - lr: 2.6509e-04 gnorm: 0.77 [ 5:20:39<16:45:28] +[titan] 2025-07-09 18:55:41,079 - root - INFO - step: 24185 loss: 16.6748 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5768 global_avg_mtp_loss: 14.0980 +[titan] 2025-07-09 18:55:41,079 - root - INFO - lr: 2.6508e-04 gnorm: 0.77 [ 5:20:43<16:45:24] +[titan] 2025-07-09 18:55:45,003 - root - INFO - step: 24190 loss: 16.7251 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1400 +[titan] 2025-07-09 18:55:45,003 - root - INFO - lr: 2.6506e-04 gnorm: 0.73 [ 5:20:47<16:45:20] +[titan] 2025-07-09 18:55:48,902 - root - INFO - step: 24195 loss: 16.7618 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.1601 +[titan] 2025-07-09 18:55:48,902 - root - INFO - lr: 2.6505e-04 gnorm: 0.77 [ 5:20:51<16:45:16] +[titan] 2025-07-09 18:55:52,020 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:55:52,808 - root - INFO - step: 24200 loss: 16.6468 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.0619 +[titan] 2025-07-09 18:55:52,809 - root - INFO - lr: 2.6504e-04 gnorm: 0.79 [ 5:20:55<16:45:11] +[titan] 2025-07-09 18:55:56,720 - root - INFO - step: 24205 loss: 16.8224 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6116 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 18:55:56,720 - root - INFO - lr: 2.6502e-04 gnorm: 0.82 [ 5:20:59<16:45:07] +[titan] 2025-07-09 18:56:00,619 - root - INFO - step: 24210 loss: 16.8638 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2413 +[titan] 2025-07-09 18:56:00,620 - root - INFO - lr: 2.6501e-04 gnorm: 0.79 [ 5:21:03<16:45:03] +[titan] 2025-07-09 18:56:04,527 - root - INFO - step: 24215 loss: 16.6412 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5685 global_avg_mtp_loss: 14.0727 +[titan] 2025-07-09 18:56:04,527 - root - INFO - lr: 2.6499e-04 gnorm: 0.81 [ 5:21:06<16:44:59] +[titan] 2025-07-09 18:56:08,433 - root - INFO - step: 24220 loss: 17.0770 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6685 global_avg_mtp_loss: 14.4085 +[titan] 2025-07-09 18:56:08,433 - root - INFO - lr: 2.6498e-04 gnorm: 0.84 [ 5:21:10<16:44:55] +[titan] 2025-07-09 18:56:12,343 - root - INFO - step: 24225 loss: 16.7169 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1390 +[titan] 2025-07-09 18:56:12,343 - root - INFO - lr: 2.6496e-04 gnorm: 0.77 [ 5:21:14<16:44:50] +[titan] 2025-07-09 18:56:16,251 - root - INFO - step: 24230 loss: 16.9226 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.2984 +[titan] 2025-07-09 18:56:16,252 - root - INFO - lr: 2.6495e-04 gnorm: 0.77 [ 5:21:18<16:44:46] +[titan] 2025-07-09 18:56:20,199 - root - INFO - step: 24235 loss: 16.6671 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.0823 +[titan] 2025-07-09 18:56:20,199 - root - INFO - lr: 2.6493e-04 gnorm: 0.85 [ 5:21:22<16:44:42] +[titan] 2025-07-09 18:56:24,100 - root - INFO - step: 24240 loss: 16.9010 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.2709 +[titan] 2025-07-09 18:56:24,100 - root - INFO - lr: 2.6492e-04 gnorm: 0.77 [ 5:21:26<16:44:38] +[titan] 2025-07-09 18:56:28,007 - root - INFO - step: 24245 loss: 16.8428 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2300 +[titan] 2025-07-09 18:56:28,007 - root - INFO - lr: 2.6491e-04 gnorm: 0.76 [ 5:21:30<16:44:34] +[titan] 2025-07-09 18:56:31,152 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:56:31,956 - root - INFO - step: 24250 loss: 16.7699 memory: 44.58GiB(31.99%) tps: 82,989 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1762 +[titan] 2025-07-09 18:56:31,956 - root - INFO - lr: 2.6489e-04 gnorm: 0.77 [ 5:21:34<16:44:30] +[titan] 2025-07-09 18:56:35,903 - root - INFO - step: 24255 loss: 16.9397 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.3113 +[titan] 2025-07-09 18:56:35,903 - root - INFO - lr: 2.6488e-04 gnorm: 0.81 [ 5:21:38<16:44:26] +[titan] 2025-07-09 18:56:39,811 - root - INFO - step: 24260 loss: 16.8233 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6191 global_avg_mtp_loss: 14.2042 +[titan] 2025-07-09 18:56:39,811 - root - INFO - lr: 2.6486e-04 gnorm: 0.76 [ 5:21:42<16:44:21] +[titan] 2025-07-09 18:56:43,716 - root - INFO - step: 24265 loss: 16.5221 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 13.9600 +[titan] 2025-07-09 18:56:43,716 - root - INFO - lr: 2.6485e-04 gnorm: 0.77 [ 5:21:46<16:44:17] +[titan] 2025-07-09 18:56:47,645 - root - INFO - step: 24270 loss: 16.7852 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.1870 +[titan] 2025-07-09 18:56:47,645 - root - INFO - lr: 2.6483e-04 gnorm: 0.80 [ 5:21:50<16:44:13] +[titan] 2025-07-09 18:56:51,554 - root - INFO - step: 24275 loss: 16.8038 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.1977 +[titan] 2025-07-09 18:56:51,554 - root - INFO - lr: 2.6482e-04 gnorm: 0.75 [ 5:21:53<16:44:09] +[titan] 2025-07-09 18:56:55,469 - root - INFO - step: 24280 loss: 16.9949 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6471 global_avg_mtp_loss: 14.3478 +[titan] 2025-07-09 18:56:55,469 - root - INFO - lr: 2.6480e-04 gnorm: 0.81 [ 5:21:57<16:44:05] +[titan] 2025-07-09 18:56:59,355 - root - INFO - step: 24285 loss: 16.9772 memory: 44.58GiB(31.99%) tps: 84,339 tflops: 291.07 mfu: 29.43% global_avg_ntp_loss: 2.6498 global_avg_mtp_loss: 14.3275 +[titan] 2025-07-09 18:56:59,355 - root - INFO - lr: 2.6479e-04 gnorm: 0.84 [ 5:22:01<16:44:00] +[titan] 2025-07-09 18:57:03,269 - root - INFO - step: 24290 loss: 17.0154 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6431 global_avg_mtp_loss: 14.3723 +[titan] 2025-07-09 18:57:03,269 - root - INFO - lr: 2.6478e-04 gnorm: 0.77 [ 5:22:05<16:43:56] +[titan] 2025-07-09 18:57:07,201 - root - INFO - step: 24295 loss: 16.5792 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5786 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-09 18:57:07,201 - root - INFO - lr: 2.6476e-04 gnorm: 0.81 [ 5:22:09<16:43:52] +[titan] 2025-07-09 18:57:10,336 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:57:11,166 - root - INFO - step: 24300 loss: 16.7206 memory: 44.58GiB(31.99%) tps: 82,656 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1299 +[titan] 2025-07-09 18:57:11,166 - root - INFO - lr: 2.6475e-04 gnorm: 0.80 [ 5:22:13<16:43:48] +[titan] 2025-07-09 18:57:15,086 - root - INFO - step: 24305 loss: 16.8999 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.6203 global_avg_mtp_loss: 14.2795 +[titan] 2025-07-09 18:57:15,086 - root - INFO - lr: 2.6473e-04 gnorm: 0.80 [ 5:22:17<16:43:44] +[titan] 2025-07-09 18:57:19,060 - root - INFO - step: 24310 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 82,461 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.6140 global_avg_mtp_loss: 14.2537 +[titan] 2025-07-09 18:57:19,061 - root - INFO - lr: 2.6472e-04 gnorm: 0.81 [ 5:22:21<16:43:40] +[titan] 2025-07-09 18:57:22,993 - root - INFO - step: 24315 loss: 16.9983 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.3519 +[titan] 2025-07-09 18:57:22,994 - root - INFO - lr: 2.6470e-04 gnorm: 0.78 [ 5:22:25<16:43:36] +[titan] 2025-07-09 18:57:26,908 - root - INFO - step: 24320 loss: 16.6821 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.0932 +[titan] 2025-07-09 18:57:26,908 - root - INFO - lr: 2.6469e-04 gnorm: 0.83 [ 5:22:29<16:43:32] +[titan] 2025-07-09 18:57:30,823 - root - INFO - step: 24325 loss: 16.8372 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2185 +[titan] 2025-07-09 18:57:30,824 - root - INFO - lr: 2.6467e-04 gnorm: 0.76 [ 5:22:33<16:43:27] +[titan] 2025-07-09 18:57:34,717 - root - INFO - step: 24330 loss: 16.6130 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-09 18:57:34,717 - root - INFO - lr: 2.6466e-04 gnorm: 0.72 [ 5:22:37<16:43:23] +[titan] 2025-07-09 18:57:38,639 - root - INFO - step: 24335 loss: 16.8623 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.2407 +[titan] 2025-07-09 18:57:38,639 - root - INFO - lr: 2.6465e-04 gnorm: 0.74 [ 5:22:41<16:43:19] +[titan] 2025-07-09 18:57:42,571 - root - INFO - step: 24340 loss: 16.7833 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.1845 +[titan] 2025-07-09 18:57:42,572 - root - INFO - lr: 2.6463e-04 gnorm: 0.75 [ 5:22:45<16:43:15] +[titan] 2025-07-09 18:57:46,483 - root - INFO - step: 24345 loss: 16.6941 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1016 +[titan] 2025-07-09 18:57:46,483 - root - INFO - lr: 2.6462e-04 gnorm: 0.96 [ 5:22:48<16:43:11] +[titan] 2025-07-09 18:57:49,596 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:57:50,411 - root - INFO - step: 24350 loss: 17.0610 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.6669 global_avg_mtp_loss: 14.3942 +[titan] 2025-07-09 18:57:50,411 - root - INFO - lr: 2.6460e-04 gnorm: 0.80 [ 5:22:52<16:43:07] +[titan] 2025-07-09 18:57:54,315 - root - INFO - step: 24355 loss: 16.8073 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.1939 +[titan] 2025-07-09 18:57:54,316 - root - INFO - lr: 2.6459e-04 gnorm: 0.83 [ 5:22:56<16:43:02] +[titan] 2025-07-09 18:57:58,216 - root - INFO - step: 24360 loss: 17.0803 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.4175 +[titan] 2025-07-09 18:57:58,216 - root - INFO - lr: 2.6457e-04 gnorm: 0.88 [ 5:23:00<16:42:58] +[titan] 2025-07-09 18:58:02,187 - root - INFO - step: 24365 loss: 16.7920 memory: 44.58GiB(31.99%) tps: 82,533 tflops: 284.83 mfu: 28.80% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.1806 +[titan] 2025-07-09 18:58:02,187 - root - INFO - lr: 2.6456e-04 gnorm: 0.79 [ 5:23:04<16:42:54] +[titan] 2025-07-09 18:58:06,128 - root - INFO - step: 24370 loss: 17.1763 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.4993 +[titan] 2025-07-09 18:58:06,128 - root - INFO - lr: 2.6454e-04 gnorm: 0.79 [ 5:23:08<16:42:50] +[titan] 2025-07-09 18:58:10,037 - root - INFO - step: 24375 loss: 16.9919 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.6577 global_avg_mtp_loss: 14.3341 +[titan] 2025-07-09 18:58:10,037 - root - INFO - lr: 2.6453e-04 gnorm: 0.74 [ 5:23:12<16:42:46] +[titan] 2025-07-09 18:58:13,972 - root - INFO - step: 24380 loss: 16.6961 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5917 global_avg_mtp_loss: 14.1045 +[titan] 2025-07-09 18:58:13,972 - root - INFO - lr: 2.6452e-04 gnorm: 0.78 [ 5:23:16<16:42:42] +[titan] 2025-07-09 18:58:17,923 - root - INFO - step: 24385 loss: 16.7586 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1601 +[titan] 2025-07-09 18:58:17,923 - root - INFO - lr: 2.6450e-04 gnorm: 0.73 [ 5:23:20<16:42:38] +[titan] 2025-07-09 18:58:21,824 - root - INFO - step: 24390 loss: 17.0889 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6759 global_avg_mtp_loss: 14.4129 +[titan] 2025-07-09 18:58:21,825 - root - INFO - lr: 2.6449e-04 gnorm: 0.80 [ 5:23:24<16:42:33] +[titan] 2025-07-09 18:58:25,746 - root - INFO - step: 24395 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6250 global_avg_mtp_loss: 14.2428 +[titan] 2025-07-09 18:58:25,746 - root - INFO - lr: 2.6447e-04 gnorm: 0.73 [ 5:23:28<16:42:29] +[titan] 2025-07-09 18:58:28,854 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:58:29,645 - root - INFO - step: 24400 loss: 16.5334 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 13.9637 +[titan] 2025-07-09 18:58:29,645 - root - INFO - lr: 2.6446e-04 gnorm: 0.76 [ 5:23:32<16:42:25] +[titan] 2025-07-09 18:58:33,555 - root - INFO - step: 24405 loss: 17.0190 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6425 global_avg_mtp_loss: 14.3765 +[titan] 2025-07-09 18:58:33,555 - root - INFO - lr: 2.6444e-04 gnorm: 0.84 [ 5:23:35<16:42:21] +[titan] 2025-07-09 18:58:37,469 - root - INFO - step: 24410 loss: 17.1983 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6704 global_avg_mtp_loss: 14.5280 +[titan] 2025-07-09 18:58:37,470 - root - INFO - lr: 2.6443e-04 gnorm: 0.77 [ 5:23:39<16:42:17] +[titan] 2025-07-09 18:58:41,406 - root - INFO - step: 24415 loss: 16.7908 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.6019 global_avg_mtp_loss: 14.1889 +[titan] 2025-07-09 18:58:41,406 - root - INFO - lr: 2.6441e-04 gnorm: 0.81 [ 5:23:43<16:42:13] +[titan] 2025-07-09 18:58:45,307 - root - INFO - step: 24420 loss: 16.9765 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.3368 +[titan] 2025-07-09 18:58:45,307 - root - INFO - lr: 2.6440e-04 gnorm: 0.87 [ 5:23:47<16:42:08] +[titan] 2025-07-09 18:58:49,237 - root - INFO - step: 24425 loss: 16.9420 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6496 global_avg_mtp_loss: 14.2924 +[titan] 2025-07-09 18:58:49,238 - root - INFO - lr: 2.6439e-04 gnorm: 0.90 [ 5:23:51<16:42:04] +[titan] 2025-07-09 18:58:53,133 - root - INFO - step: 24430 loss: 17.0541 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6609 global_avg_mtp_loss: 14.3932 +[titan] 2025-07-09 18:58:53,133 - root - INFO - lr: 2.6437e-04 gnorm: 0.85 [ 5:23:55<16:42:00] +[titan] 2025-07-09 18:58:57,021 - root - INFO - step: 24435 loss: 16.8946 memory: 44.58GiB(31.99%) tps: 84,290 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.6234 global_avg_mtp_loss: 14.2712 +[titan] 2025-07-09 18:58:57,021 - root - INFO - lr: 2.6436e-04 gnorm: 0.84 [ 5:23:59<16:41:56] +[titan] 2025-07-09 18:59:00,959 - root - INFO - step: 24440 loss: 16.7127 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.1019 +[titan] 2025-07-09 18:59:00,960 - root - INFO - lr: 2.6434e-04 gnorm: 0.80 [ 5:24:03<16:41:52] +[titan] 2025-07-09 18:59:04,853 - root - INFO - step: 24445 loss: 16.9680 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6370 global_avg_mtp_loss: 14.3310 +[titan] 2025-07-09 18:59:04,853 - root - INFO - lr: 2.6433e-04 gnorm: 0.74 [ 5:24:07<16:41:47] +[titan] 2025-07-09 18:59:07,974 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:59:08,767 - root - INFO - step: 24450 loss: 16.7469 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5962 global_avg_mtp_loss: 14.1507 +[titan] 2025-07-09 18:59:08,767 - root - INFO - lr: 2.6431e-04 gnorm: 0.79 [ 5:24:11<16:41:43] +[titan] 2025-07-09 18:59:12,715 - root - INFO - step: 24455 loss: 17.0236 memory: 44.58GiB(31.99%) tps: 82,996 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3799 +[titan] 2025-07-09 18:59:12,715 - root - INFO - lr: 2.6430e-04 gnorm: 0.77 [ 5:24:15<16:41:39] +[titan] 2025-07-09 18:59:16,628 - root - INFO - step: 24460 loss: 16.9115 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2917 +[titan] 2025-07-09 18:59:16,628 - root - INFO - lr: 2.6428e-04 gnorm: 0.74 [ 5:24:19<16:41:35] +[titan] 2025-07-09 18:59:20,535 - root - INFO - step: 24465 loss: 16.6905 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1012 +[titan] 2025-07-09 18:59:20,535 - root - INFO - lr: 2.6427e-04 gnorm: 0.77 [ 5:24:22<16:41:31] +[titan] 2025-07-09 18:59:24,442 - root - INFO - step: 24470 loss: 17.0214 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6464 global_avg_mtp_loss: 14.3750 +[titan] 2025-07-09 18:59:24,442 - root - INFO - lr: 2.6425e-04 gnorm: 0.75 [ 5:24:26<16:41:27] +[titan] 2025-07-09 18:59:28,386 - root - INFO - step: 24475 loss: 17.1685 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6660 global_avg_mtp_loss: 14.5025 +[titan] 2025-07-09 18:59:28,386 - root - INFO - lr: 2.6424e-04 gnorm: 0.77 [ 5:24:30<16:41:23] +[titan] 2025-07-09 18:59:32,273 - root - INFO - step: 24480 loss: 16.8171 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6167 global_avg_mtp_loss: 14.2004 +[titan] 2025-07-09 18:59:32,273 - root - INFO - lr: 2.6423e-04 gnorm: 0.78 [ 5:24:34<16:41:18] +[titan] 2025-07-09 18:59:36,199 - root - INFO - step: 24485 loss: 16.7510 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5980 global_avg_mtp_loss: 14.1530 +[titan] 2025-07-09 18:59:36,199 - root - INFO - lr: 2.6421e-04 gnorm: 0.78 [ 5:24:38<16:41:14] +[titan] 2025-07-09 18:59:40,104 - root - INFO - step: 24490 loss: 17.0328 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6433 global_avg_mtp_loss: 14.3895 +[titan] 2025-07-09 18:59:40,104 - root - INFO - lr: 2.6420e-04 gnorm: 0.80 [ 5:24:42<16:41:10] +[titan] 2025-07-09 18:59:43,998 - root - INFO - step: 24495 loss: 16.7952 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.1913 +[titan] 2025-07-09 18:59:43,998 - root - INFO - lr: 2.6418e-04 gnorm: 0.79 [ 5:24:46<16:41:06] +[titan] 2025-07-09 18:59:47,118 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 18:59:47,911 - root - INFO - step: 24500 loss: 16.7713 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6132 global_avg_mtp_loss: 14.1581 +[titan] 2025-07-09 18:59:47,911 - root - INFO - lr: 2.6417e-04 gnorm: 0.84 [ 5:24:50<16:41:02] +[titan] 2025-07-09 18:59:51,823 - root - INFO - step: 24505 loss: 16.8993 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6193 global_avg_mtp_loss: 14.2800 +[titan] 2025-07-09 18:59:51,823 - root - INFO - lr: 2.6415e-04 gnorm: 0.80 [ 5:24:54<16:40:57] +[titan] 2025-07-09 18:59:55,722 - root - INFO - step: 24510 loss: 16.7703 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1824 +[titan] 2025-07-09 18:59:55,723 - root - INFO - lr: 2.6414e-04 gnorm: 0.77 [ 5:24:58<16:40:53] +[titan] 2025-07-09 18:59:59,624 - root - INFO - step: 24515 loss: 17.0240 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6555 global_avg_mtp_loss: 14.3685 +[titan] 2025-07-09 18:59:59,624 - root - INFO - lr: 2.6412e-04 gnorm: 0.95 [ 5:25:02<16:40:49] +[titan] 2025-07-09 19:00:03,571 - root - INFO - step: 24520 loss: 16.7549 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1620 +[titan] 2025-07-09 19:00:03,571 - root - INFO - lr: 2.6411e-04 gnorm: 0.75 [ 5:25:05<16:40:45] +[titan] 2025-07-09 19:00:07,476 - root - INFO - step: 24525 loss: 16.7330 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.1386 +[titan] 2025-07-09 19:00:07,476 - root - INFO - lr: 2.6409e-04 gnorm: 0.86 [ 5:25:09<16:40:41] +[titan] 2025-07-09 19:00:11,369 - root - INFO - step: 24530 loss: 16.8506 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.6249 global_avg_mtp_loss: 14.2257 +[titan] 2025-07-09 19:00:11,369 - root - INFO - lr: 2.6408e-04 gnorm: 0.80 [ 5:25:13<16:40:36] +[titan] 2025-07-09 19:00:15,277 - root - INFO - step: 24535 loss: 16.9004 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6246 global_avg_mtp_loss: 14.2759 +[titan] 2025-07-09 19:00:15,277 - root - INFO - lr: 2.6407e-04 gnorm: 0.84 [ 5:25:17<16:40:32] +[titan] 2025-07-09 19:00:19,165 - root - INFO - step: 24540 loss: 16.7900 memory: 44.58GiB(31.99%) tps: 84,296 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.6042 global_avg_mtp_loss: 14.1858 +[titan] 2025-07-09 19:00:19,165 - root - INFO - lr: 2.6405e-04 gnorm: 0.77 [ 5:25:21<16:40:28] +[titan] 2025-07-09 19:00:23,078 - root - INFO - step: 24545 loss: 16.9053 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6268 global_avg_mtp_loss: 14.2785 +[titan] 2025-07-09 19:00:23,078 - root - INFO - lr: 2.6404e-04 gnorm: 0.74 [ 5:25:25<16:40:24] +[titan] 2025-07-09 19:00:26,237 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:00:27,026 - root - INFO - step: 24550 loss: 17.0174 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3655 +[titan] 2025-07-09 19:00:27,027 - root - INFO - lr: 2.6402e-04 gnorm: 0.89 [ 5:25:29<16:40:20] +[titan] 2025-07-09 19:00:30,986 - root - INFO - step: 24555 loss: 16.7353 memory: 44.58GiB(31.99%) tps: 82,763 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.6015 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 19:00:30,986 - root - INFO - lr: 2.6401e-04 gnorm: 0.73 [ 5:25:33<16:40:16] +[titan] 2025-07-09 19:00:34,909 - root - INFO - step: 24560 loss: 16.6407 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0705 +[titan] 2025-07-09 19:00:34,909 - root - INFO - lr: 2.6399e-04 gnorm: 0.74 [ 5:25:37<16:40:12] +[titan] 2025-07-09 19:00:38,832 - root - INFO - step: 24565 loss: 16.7194 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1287 +[titan] 2025-07-09 19:00:38,833 - root - INFO - lr: 2.6398e-04 gnorm: 0.76 [ 5:25:41<16:40:07] +[titan] 2025-07-09 19:00:42,735 - root - INFO - step: 24570 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.0575 +[titan] 2025-07-09 19:00:42,735 - root - INFO - lr: 2.6396e-04 gnorm: 0.83 [ 5:25:45<16:40:03] +[titan] 2025-07-09 19:00:46,764 - root - INFO - step: 24575 loss: 17.0690 memory: 44.58GiB(31.99%) tps: 81,343 tflops: 280.73 mfu: 28.39% global_avg_ntp_loss: 2.6459 global_avg_mtp_loss: 14.4231 +[titan] 2025-07-09 19:00:46,764 - root - INFO - lr: 2.6395e-04 gnorm: 0.80 [ 5:25:49<16:39:59] +[titan] 2025-07-09 19:00:47,704 - root - INFO - Dumping profiler traces at step 24576 +[titan] 2025-07-09 19:00:47,736 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:00:50,859 - root - INFO - step: 24580 loss: 16.6435 memory: 44.58GiB(31.99%) tps: 80,026 tflops: 276.18 mfu: 27.93% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.0604 +[titan] 2025-07-09 19:00:50,859 - root - INFO - lr: 2.6393e-04 gnorm: 0.80 [ 5:25:53<16:39:56] +[titan] 2025-07-09 19:00:54,772 - root - INFO - step: 24585 loss: 17.1521 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6752 global_avg_mtp_loss: 14.4769 +[titan] 2025-07-09 19:00:54,772 - root - INFO - lr: 2.6392e-04 gnorm: 0.76 [ 5:25:57<16:39:52] +[titan] 2025-07-09 19:00:58,677 - root - INFO - step: 24590 loss: 16.8513 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.2400 +[titan] 2025-07-09 19:00:58,677 - root - INFO - lr: 2.6391e-04 gnorm: 0.77 [ 5:26:01<16:39:47] +[titan] 2025-07-09 19:01:02,584 - root - INFO - step: 24595 loss: 16.8345 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.2347 +[titan] 2025-07-09 19:01:02,584 - root - INFO - lr: 2.6389e-04 gnorm: 0.77 [ 5:26:04<16:39:43] +[titan] 2025-07-09 19:01:05,705 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:01:06,510 - root - INFO - step: 24600 loss: 16.8498 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6193 global_avg_mtp_loss: 14.2305 +[titan] 2025-07-09 19:01:06,511 - root - INFO - lr: 2.6388e-04 gnorm: 0.76 [ 5:26:08<16:39:39] +[titan] 2025-07-09 19:01:10,448 - root - INFO - step: 24605 loss: 16.4356 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 13.8884 +[titan] 2025-07-09 19:01:10,448 - root - INFO - lr: 2.6386e-04 gnorm: 0.80 [ 5:26:12<16:39:35] +[titan] 2025-07-09 19:01:14,362 - root - INFO - step: 24610 loss: 16.7409 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5949 global_avg_mtp_loss: 14.1460 +[titan] 2025-07-09 19:01:14,362 - root - INFO - lr: 2.6385e-04 gnorm: 0.73 [ 5:26:16<16:39:31] +[titan] 2025-07-09 19:01:18,289 - root - INFO - step: 24615 loss: 16.9610 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6399 global_avg_mtp_loss: 14.3210 +[titan] 2025-07-09 19:01:18,289 - root - INFO - lr: 2.6383e-04 gnorm: 0.74 [ 5:26:20<16:39:27] +[titan] 2025-07-09 19:01:22,283 - root - INFO - step: 24620 loss: 16.8395 memory: 44.58GiB(31.99%) tps: 82,058 tflops: 283.20 mfu: 28.63% global_avg_ntp_loss: 2.6118 global_avg_mtp_loss: 14.2278 +[titan] 2025-07-09 19:01:22,283 - root - INFO - lr: 2.6382e-04 gnorm: 0.79 [ 5:26:24<16:39:23] +[titan] 2025-07-09 19:01:26,192 - root - INFO - step: 24625 loss: 16.8312 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.2209 +[titan] 2025-07-09 19:01:26,192 - root - INFO - lr: 2.6380e-04 gnorm: 0.79 [ 5:26:28<16:39:19] +[titan] 2025-07-09 19:01:30,111 - root - INFO - step: 24630 loss: 16.7744 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6076 global_avg_mtp_loss: 14.1668 +[titan] 2025-07-09 19:01:30,112 - root - INFO - lr: 2.6379e-04 gnorm: 0.84 [ 5:26:32<16:39:14] +[titan] 2025-07-09 19:01:34,040 - root - INFO - step: 24635 loss: 17.1554 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6716 global_avg_mtp_loss: 14.4838 +[titan] 2025-07-09 19:01:34,040 - root - INFO - lr: 2.6377e-04 gnorm: 0.79 [ 5:26:36<16:39:10] +[titan] 2025-07-09 19:01:37,979 - root - INFO - step: 24640 loss: 16.9584 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.3283 +[titan] 2025-07-09 19:01:37,980 - root - INFO - lr: 2.6376e-04 gnorm: 0.74 [ 5:26:40<16:39:06] +[titan] 2025-07-09 19:01:41,909 - root - INFO - step: 24645 loss: 17.0736 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6583 global_avg_mtp_loss: 14.4154 +[titan] 2025-07-09 19:01:41,909 - root - INFO - lr: 2.6375e-04 gnorm: 0.79 [ 5:26:44<16:39:02] +[titan] 2025-07-09 19:01:45,018 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:01:45,815 - root - INFO - step: 24650 loss: 16.8621 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.2449 +[titan] 2025-07-09 19:01:45,816 - root - INFO - lr: 2.6373e-04 gnorm: 0.73 [ 5:26:48<16:38:58] +[titan] 2025-07-09 19:01:49,713 - root - INFO - step: 24655 loss: 17.0673 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6665 global_avg_mtp_loss: 14.4008 +[titan] 2025-07-09 19:01:49,713 - root - INFO - lr: 2.6372e-04 gnorm: 0.78 [ 5:26:52<16:38:54] +[titan] 2025-07-09 19:01:53,637 - root - INFO - step: 24660 loss: 16.8971 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2712 +[titan] 2025-07-09 19:01:53,637 - root - INFO - lr: 2.6370e-04 gnorm: 0.77 [ 5:26:56<16:38:49] +[titan] 2025-07-09 19:01:57,558 - root - INFO - step: 24665 loss: 17.1123 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6764 global_avg_mtp_loss: 14.4358 +[titan] 2025-07-09 19:01:57,558 - root - INFO - lr: 2.6369e-04 gnorm: 0.75 [ 5:26:59<16:38:45] +[titan] 2025-07-09 19:02:01,464 - root - INFO - step: 24670 loss: 17.1488 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6779 global_avg_mtp_loss: 14.4709 +[titan] 2025-07-09 19:02:01,464 - root - INFO - lr: 2.6367e-04 gnorm: 0.82 [ 5:27:03<16:38:41] +[titan] 2025-07-09 19:02:05,368 - root - INFO - step: 24675 loss: 16.7869 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1856 +[titan] 2025-07-09 19:02:05,369 - root - INFO - lr: 2.6366e-04 gnorm: 0.79 [ 5:27:07<16:38:37] +[titan] 2025-07-09 19:02:09,301 - root - INFO - step: 24680 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5931 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-09 19:02:09,301 - root - INFO - lr: 2.6364e-04 gnorm: 0.86 [ 5:27:11<16:38:33] +[titan] 2025-07-09 19:02:13,216 - root - INFO - step: 24685 loss: 16.8816 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6273 global_avg_mtp_loss: 14.2543 +[titan] 2025-07-09 19:02:13,216 - root - INFO - lr: 2.6363e-04 gnorm: 0.82 [ 5:27:15<16:38:29] +[titan] 2025-07-09 19:02:17,123 - root - INFO - step: 24690 loss: 16.9382 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.3122 +[titan] 2025-07-09 19:02:17,124 - root - INFO - lr: 2.6361e-04 gnorm: 0.76 [ 5:27:19<16:38:24] +[titan] 2025-07-09 19:02:21,038 - root - INFO - step: 24695 loss: 17.1560 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6789 global_avg_mtp_loss: 14.4771 +[titan] 2025-07-09 19:02:21,039 - root - INFO - lr: 2.6360e-04 gnorm: 0.80 [ 5:27:23<16:38:20] +[titan] 2025-07-09 19:02:24,188 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:02:24,975 - root - INFO - step: 24700 loss: 16.8331 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.2159 +[titan] 2025-07-09 19:02:24,976 - root - INFO - lr: 2.6358e-04 gnorm: 0.73 [ 5:27:27<16:38:16] +[titan] 2025-07-09 19:02:28,965 - root - INFO - step: 24705 loss: 16.7203 memory: 44.58GiB(31.99%) tps: 82,145 tflops: 283.50 mfu: 28.66% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1280 +[titan] 2025-07-09 19:02:28,965 - root - INFO - lr: 2.6357e-04 gnorm: 0.85 [ 5:27:31<16:38:12] +[titan] 2025-07-09 19:02:32,895 - root - INFO - step: 24710 loss: 16.6793 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5886 global_avg_mtp_loss: 14.0907 +[titan] 2025-07-09 19:02:32,895 - root - INFO - lr: 2.6356e-04 gnorm: 0.82 [ 5:27:35<16:38:08] +[titan] 2025-07-09 19:02:36,854 - root - INFO - step: 24715 loss: 17.1238 memory: 44.58GiB(31.99%) tps: 82,757 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.6679 global_avg_mtp_loss: 14.4559 +[titan] 2025-07-09 19:02:36,855 - root - INFO - lr: 2.6354e-04 gnorm: 0.85 [ 5:27:39<16:38:04] +[titan] 2025-07-09 19:02:40,776 - root - INFO - step: 24720 loss: 16.9774 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6321 global_avg_mtp_loss: 14.3453 +[titan] 2025-07-09 19:02:40,777 - root - INFO - lr: 2.6353e-04 gnorm: 0.80 [ 5:27:43<16:38:00] +[titan] 2025-07-09 19:02:44,701 - root - INFO - step: 24725 loss: 17.0761 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.4165 +[titan] 2025-07-09 19:02:44,701 - root - INFO - lr: 2.6351e-04 gnorm: 0.77 [ 5:27:47<16:37:56] +[titan] 2025-07-09 19:02:48,649 - root - INFO - step: 24730 loss: 16.7292 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.1215 +[titan] 2025-07-09 19:02:48,650 - root - INFO - lr: 2.6350e-04 gnorm: 0.76 [ 5:27:51<16:37:52] +[titan] 2025-07-09 19:02:52,557 - root - INFO - step: 24735 loss: 16.7351 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6000 global_avg_mtp_loss: 14.1351 +[titan] 2025-07-09 19:02:52,557 - root - INFO - lr: 2.6348e-04 gnorm: 0.74 [ 5:27:54<16:37:48] +[titan] 2025-07-09 19:02:56,463 - root - INFO - step: 24740 loss: 16.9978 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6212 global_avg_mtp_loss: 14.3766 +[titan] 2025-07-09 19:02:56,463 - root - INFO - lr: 2.6347e-04 gnorm: 0.76 [ 5:27:58<16:37:43] +[titan] 2025-07-09 19:03:00,397 - root - INFO - step: 24745 loss: 16.9301 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.6244 global_avg_mtp_loss: 14.3057 +[titan] 2025-07-09 19:03:00,397 - root - INFO - lr: 2.6345e-04 gnorm: 0.75 [ 5:28:02<16:37:39] +[titan] 2025-07-09 19:03:03,508 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:03:04,304 - root - INFO - step: 24750 loss: 16.6087 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.0319 +[titan] 2025-07-09 19:03:04,305 - root - INFO - lr: 2.6344e-04 gnorm: 0.77 [ 5:28:06<16:37:35] +[titan] 2025-07-09 19:03:08,230 - root - INFO - step: 24755 loss: 17.1127 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6671 global_avg_mtp_loss: 14.4455 +[titan] 2025-07-09 19:03:08,231 - root - INFO - lr: 2.6342e-04 gnorm: 0.75 [ 5:28:10<16:37:31] +[titan] 2025-07-09 19:03:12,164 - root - INFO - step: 24760 loss: 16.9056 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6232 global_avg_mtp_loss: 14.2823 +[titan] 2025-07-09 19:03:12,165 - root - INFO - lr: 2.6341e-04 gnorm: 0.76 [ 5:28:14<16:37:27] +[titan] 2025-07-09 19:03:16,058 - root - INFO - step: 24765 loss: 16.9034 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.2708 +[titan] 2025-07-09 19:03:16,059 - root - INFO - lr: 2.6339e-04 gnorm: 0.75 [ 5:28:18<16:37:23] +[titan] 2025-07-09 19:03:19,959 - root - INFO - step: 24770 loss: 16.9688 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6275 global_avg_mtp_loss: 14.3413 +[titan] 2025-07-09 19:03:19,960 - root - INFO - lr: 2.6338e-04 gnorm: 0.75 [ 5:28:22<16:37:18] +[titan] 2025-07-09 19:03:23,857 - root - INFO - step: 24775 loss: 16.9510 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 19:03:23,857 - root - INFO - lr: 2.6336e-04 gnorm: 0.88 [ 5:28:26<16:37:14] +[titan] 2025-07-09 19:03:27,797 - root - INFO - step: 24780 loss: 16.7282 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.1424 +[titan] 2025-07-09 19:03:27,797 - root - INFO - lr: 2.6335e-04 gnorm: 0.76 [ 5:28:30<16:37:10] +[titan] 2025-07-09 19:03:31,744 - root - INFO - step: 24785 loss: 16.8081 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.6196 global_avg_mtp_loss: 14.1884 +[titan] 2025-07-09 19:03:31,744 - root - INFO - lr: 2.6334e-04 gnorm: 0.81 [ 5:28:34<16:37:06] +[titan] 2025-07-09 19:03:35,723 - root - INFO - step: 24790 loss: 16.4741 memory: 44.58GiB(31.99%) tps: 82,361 tflops: 284.24 mfu: 28.74% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9230 +[titan] 2025-07-09 19:03:35,723 - root - INFO - lr: 2.6332e-04 gnorm: 0.74 [ 5:28:38<16:37:02] +[titan] 2025-07-09 19:03:39,653 - root - INFO - step: 24795 loss: 16.9875 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6316 global_avg_mtp_loss: 14.3559 +[titan] 2025-07-09 19:03:39,653 - root - INFO - lr: 2.6331e-04 gnorm: 0.73 [ 5:28:42<16:36:58] +[titan] 2025-07-09 19:03:42,788 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:03:43,586 - root - INFO - step: 24800 loss: 16.9035 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.2715 +[titan] 2025-07-09 19:03:43,586 - root - INFO - lr: 2.6329e-04 gnorm: 0.75 [ 5:28:45<16:36:54] +[titan] 2025-07-09 19:03:47,488 - root - INFO - step: 24805 loss: 16.8172 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6156 global_avg_mtp_loss: 14.2016 +[titan] 2025-07-09 19:03:47,488 - root - INFO - lr: 2.6328e-04 gnorm: 0.80 [ 5:28:49<16:36:50] +[titan] 2025-07-09 19:03:51,434 - root - INFO - step: 24810 loss: 17.0169 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.6468 global_avg_mtp_loss: 14.3701 +[titan] 2025-07-09 19:03:51,435 - root - INFO - lr: 2.6326e-04 gnorm: 0.75 [ 5:28:53<16:36:45] +[titan] 2025-07-09 19:03:55,350 - root - INFO - step: 24815 loss: 16.9777 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6410 global_avg_mtp_loss: 14.3368 +[titan] 2025-07-09 19:03:55,350 - root - INFO - lr: 2.6325e-04 gnorm: 0.75 [ 5:28:57<16:36:41] +[titan] 2025-07-09 19:03:59,253 - root - INFO - step: 24820 loss: 16.5885 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-09 19:03:59,254 - root - INFO - lr: 2.6323e-04 gnorm: 0.87 [ 5:29:01<16:36:37] +[titan] 2025-07-09 19:04:03,202 - root - INFO - step: 24825 loss: 16.8347 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.2226 +[titan] 2025-07-09 19:04:03,202 - root - INFO - lr: 2.6322e-04 gnorm: 0.77 [ 5:29:05<16:36:33] +[titan] 2025-07-09 19:04:07,105 - root - INFO - step: 24830 loss: 16.8914 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.2597 +[titan] 2025-07-09 19:04:07,105 - root - INFO - lr: 2.6320e-04 gnorm: 0.78 [ 5:29:09<16:36:29] +[titan] 2025-07-09 19:04:11,042 - root - INFO - step: 24835 loss: 16.8288 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.6090 global_avg_mtp_loss: 14.2198 +[titan] 2025-07-09 19:04:11,043 - root - INFO - lr: 2.6319e-04 gnorm: 0.77 [ 5:29:13<16:36:25] +[titan] 2025-07-09 19:04:14,975 - root - INFO - step: 24840 loss: 16.8763 memory: 44.58GiB(31.99%) tps: 83,321 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.6354 global_avg_mtp_loss: 14.2409 +[titan] 2025-07-09 19:04:14,976 - root - INFO - lr: 2.6317e-04 gnorm: 0.73 [ 5:29:17<16:36:21] +[titan] 2025-07-09 19:04:18,909 - root - INFO - step: 24845 loss: 16.9601 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.6458 global_avg_mtp_loss: 14.3143 +[titan] 2025-07-09 19:04:18,909 - root - INFO - lr: 2.6316e-04 gnorm: 0.77 [ 5:29:21<16:36:17] +[titan] 2025-07-09 19:04:22,022 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:04:22,821 - root - INFO - step: 24850 loss: 16.7744 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6043 global_avg_mtp_loss: 14.1701 +[titan] 2025-07-09 19:04:22,821 - root - INFO - lr: 2.6314e-04 gnorm: 0.73 [ 5:29:25<16:36:12] +[titan] 2025-07-09 19:04:26,746 - root - INFO - step: 24855 loss: 16.7040 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5911 global_avg_mtp_loss: 14.1129 +[titan] 2025-07-09 19:04:26,746 - root - INFO - lr: 2.6313e-04 gnorm: 0.84 [ 5:29:29<16:36:08] +[titan] 2025-07-09 19:04:30,715 - root - INFO - step: 24860 loss: 16.8969 memory: 44.58GiB(31.99%) tps: 82,570 tflops: 284.96 mfu: 28.81% global_avg_ntp_loss: 2.6337 global_avg_mtp_loss: 14.2632 +[titan] 2025-07-09 19:04:30,715 - root - INFO - lr: 2.6311e-04 gnorm: 0.90 [ 5:29:33<16:36:04] +[titan] 2025-07-09 19:04:34,622 - root - INFO - step: 24865 loss: 17.1079 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6632 global_avg_mtp_loss: 14.4447 +[titan] 2025-07-09 19:04:34,623 - root - INFO - lr: 2.6310e-04 gnorm: 0.78 [ 5:29:37<16:36:00] +[titan] 2025-07-09 19:04:38,553 - root - INFO - step: 24870 loss: 16.6411 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0593 +[titan] 2025-07-09 19:04:38,553 - root - INFO - lr: 2.6309e-04 gnorm: 0.82 [ 5:29:40<16:35:56] +[titan] 2025-07-09 19:04:42,488 - root - INFO - step: 24875 loss: 16.5505 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5529 global_avg_mtp_loss: 13.9976 +[titan] 2025-07-09 19:04:42,488 - root - INFO - lr: 2.6307e-04 gnorm: 0.76 [ 5:29:44<16:35:52] +[titan] 2025-07-09 19:04:46,413 - root - INFO - step: 24880 loss: 16.7794 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1825 +[titan] 2025-07-09 19:04:46,414 - root - INFO - lr: 2.6306e-04 gnorm: 0.77 [ 5:29:48<16:35:48] +[titan] 2025-07-09 19:04:50,328 - root - INFO - step: 24885 loss: 16.9391 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6458 global_avg_mtp_loss: 14.2933 +[titan] 2025-07-09 19:04:50,328 - root - INFO - lr: 2.6304e-04 gnorm: 0.77 [ 5:29:52<16:35:43] +[titan] 2025-07-09 19:04:54,240 - root - INFO - step: 24890 loss: 16.9915 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.3356 +[titan] 2025-07-09 19:04:54,240 - root - INFO - lr: 2.6303e-04 gnorm: 0.81 [ 5:29:56<16:35:39] +[titan] 2025-07-09 19:04:58,180 - root - INFO - step: 24895 loss: 16.5486 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5615 global_avg_mtp_loss: 13.9871 +[titan] 2025-07-09 19:04:58,180 - root - INFO - lr: 2.6301e-04 gnorm: 0.73 [ 5:30:00<16:35:35] +[titan] 2025-07-09 19:05:01,334 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:05:02,125 - root - INFO - step: 24900 loss: 16.7214 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1248 +[titan] 2025-07-09 19:05:02,125 - root - INFO - lr: 2.6300e-04 gnorm: 0.74 [ 5:30:04<16:35:31] +[titan] 2025-07-09 19:05:06,060 - root - INFO - step: 24905 loss: 16.8318 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.6256 global_avg_mtp_loss: 14.2062 +[titan] 2025-07-09 19:05:06,061 - root - INFO - lr: 2.6298e-04 gnorm: 0.77 [ 5:30:08<16:35:27] +[titan] 2025-07-09 19:05:09,981 - root - INFO - step: 24910 loss: 17.1310 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.4714 +[titan] 2025-07-09 19:05:09,981 - root - INFO - lr: 2.6297e-04 gnorm: 0.78 [ 5:30:12<16:35:23] +[titan] 2025-07-09 19:05:13,924 - root - INFO - step: 24915 loss: 16.8713 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.2428 +[titan] 2025-07-09 19:05:13,924 - root - INFO - lr: 2.6295e-04 gnorm: 0.76 [ 5:30:16<16:35:19] +[titan] 2025-07-09 19:05:17,852 - root - INFO - step: 24920 loss: 16.7520 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.1542 +[titan] 2025-07-09 19:05:17,853 - root - INFO - lr: 2.6294e-04 gnorm: 0.82 [ 5:30:20<16:35:15] +[titan] 2025-07-09 19:05:21,776 - root - INFO - step: 24925 loss: 16.9608 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.3298 +[titan] 2025-07-09 19:05:21,777 - root - INFO - lr: 2.6292e-04 gnorm: 0.75 [ 5:30:24<16:35:11] +[titan] 2025-07-09 19:05:25,699 - root - INFO - step: 24930 loss: 17.1055 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6616 global_avg_mtp_loss: 14.4439 +[titan] 2025-07-09 19:05:25,700 - root - INFO - lr: 2.6291e-04 gnorm: 0.78 [ 5:30:28<16:35:06] +[titan] 2025-07-09 19:05:29,604 - root - INFO - step: 24935 loss: 16.8628 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.2387 +[titan] 2025-07-09 19:05:29,605 - root - INFO - lr: 2.6289e-04 gnorm: 0.75 [ 5:30:31<16:35:02] +[titan] 2025-07-09 19:05:33,518 - root - INFO - step: 24940 loss: 17.0362 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6570 global_avg_mtp_loss: 14.3791 +[titan] 2025-07-09 19:05:33,519 - root - INFO - lr: 2.6288e-04 gnorm: 0.77 [ 5:30:35<16:34:58] +[titan] 2025-07-09 19:05:37,425 - root - INFO - step: 24945 loss: 16.8099 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.2007 +[titan] 2025-07-09 19:05:37,426 - root - INFO - lr: 2.6286e-04 gnorm: 0.77 [ 5:30:39<16:34:54] +[titan] 2025-07-09 19:05:40,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:05:41,350 - root - INFO - step: 24950 loss: 16.9043 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6245 global_avg_mtp_loss: 14.2798 +[titan] 2025-07-09 19:05:41,350 - root - INFO - lr: 2.6285e-04 gnorm: 0.76 [ 5:30:43<16:34:50] +[titan] 2025-07-09 19:05:45,240 - root - INFO - step: 24955 loss: 16.7129 memory: 44.58GiB(31.99%) tps: 84,238 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1294 +[titan] 2025-07-09 19:05:45,240 - root - INFO - lr: 2.6283e-04 gnorm: 0.81 [ 5:30:47<16:34:46] +[titan] 2025-07-09 19:05:49,156 - root - INFO - step: 24960 loss: 16.9154 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.2757 +[titan] 2025-07-09 19:05:49,156 - root - INFO - lr: 2.6282e-04 gnorm: 0.88 [ 5:30:51<16:34:41] +[titan] 2025-07-09 19:05:53,084 - root - INFO - step: 24965 loss: 16.6385 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.0608 +[titan] 2025-07-09 19:05:53,084 - root - INFO - lr: 2.6281e-04 gnorm: 0.77 [ 5:30:55<16:34:37] +[titan] 2025-07-09 19:05:56,981 - root - INFO - step: 24970 loss: 16.7877 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6124 global_avg_mtp_loss: 14.1753 +[titan] 2025-07-09 19:05:56,981 - root - INFO - lr: 2.6279e-04 gnorm: 0.95 [ 5:30:59<16:34:33] +[titan] 2025-07-09 19:06:00,891 - root - INFO - step: 24975 loss: 17.0208 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6568 global_avg_mtp_loss: 14.3641 +[titan] 2025-07-09 19:06:00,892 - root - INFO - lr: 2.6278e-04 gnorm: 0.81 [ 5:31:03<16:34:29] +[titan] 2025-07-09 19:06:04,778 - root - INFO - step: 24980 loss: 16.9153 memory: 44.58GiB(31.99%) tps: 84,320 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.6256 global_avg_mtp_loss: 14.2897 +[titan] 2025-07-09 19:06:04,778 - root - INFO - lr: 2.6276e-04 gnorm: 0.76 [ 5:31:07<16:34:25] +[titan] 2025-07-09 19:06:08,669 - root - INFO - step: 24985 loss: 16.9128 memory: 44.58GiB(31.99%) tps: 84,226 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6333 global_avg_mtp_loss: 14.2795 +[titan] 2025-07-09 19:06:08,669 - root - INFO - lr: 2.6275e-04 gnorm: 0.81 [ 5:31:11<16:34:20] +[titan] 2025-07-09 19:06:12,574 - root - INFO - step: 24990 loss: 16.8152 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.6038 global_avg_mtp_loss: 14.2114 +[titan] 2025-07-09 19:06:12,574 - root - INFO - lr: 2.6273e-04 gnorm: 0.77 [ 5:31:14<16:34:16] +[titan] 2025-07-09 19:06:16,466 - root - INFO - step: 24995 loss: 17.0015 memory: 44.58GiB(31.99%) tps: 84,190 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6411 global_avg_mtp_loss: 14.3604 +[titan] 2025-07-09 19:06:16,467 - root - INFO - lr: 2.6272e-04 gnorm: 0.72 [ 5:31:18<16:34:12] +[titan] 2025-07-09 19:06:19,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:06:20,363 - root - INFO - step: 25000 loss: 16.7962 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.1891 +[titan] 2025-07-09 19:06:20,363 - root - INFO - lr: 2.6270e-04 gnorm: 0.81 [ 5:31:22<16:34:08] +[titan] 2025-07-09 19:06:24,281 - root - INFO - step: 25005 loss: 16.7561 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.1482 +[titan] 2025-07-09 19:06:24,281 - root - INFO - lr: 2.6269e-04 gnorm: 0.76 [ 5:31:26<16:34:04] +[titan] 2025-07-09 19:06:28,179 - root - INFO - step: 25010 loss: 16.8035 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6155 global_avg_mtp_loss: 14.1881 +[titan] 2025-07-09 19:06:28,179 - root - INFO - lr: 2.6267e-04 gnorm: 0.76 [ 5:31:30<16:33:59] +[titan] 2025-07-09 19:06:32,087 - root - INFO - step: 25015 loss: 16.7161 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 19:06:32,087 - root - INFO - lr: 2.6266e-04 gnorm: 0.75 [ 5:31:34<16:33:55] +[titan] 2025-07-09 19:06:35,987 - root - INFO - step: 25020 loss: 16.9127 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6171 global_avg_mtp_loss: 14.2956 +[titan] 2025-07-09 19:06:35,987 - root - INFO - lr: 2.6264e-04 gnorm: 0.76 [ 5:31:38<16:33:51] +[titan] 2025-07-09 19:06:39,881 - root - INFO - step: 25025 loss: 16.8014 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.6140 global_avg_mtp_loss: 14.1874 +[titan] 2025-07-09 19:06:39,881 - root - INFO - lr: 2.6263e-04 gnorm: 0.76 [ 5:31:42<16:33:47] +[titan] 2025-07-09 19:06:43,796 - root - INFO - step: 25030 loss: 16.7938 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.1800 +[titan] 2025-07-09 19:06:43,796 - root - INFO - lr: 2.6261e-04 gnorm: 0.74 [ 5:31:46<16:33:43] +[titan] 2025-07-09 19:06:47,703 - root - INFO - step: 25035 loss: 16.7280 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.1332 +[titan] 2025-07-09 19:06:47,703 - root - INFO - lr: 2.6260e-04 gnorm: 0.75 [ 5:31:50<16:33:38] +[titan] 2025-07-09 19:06:51,601 - root - INFO - step: 25040 loss: 16.8356 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.2215 +[titan] 2025-07-09 19:06:51,601 - root - INFO - lr: 2.6258e-04 gnorm: 0.73 [ 5:31:53<16:33:34] +[titan] 2025-07-09 19:06:55,512 - root - INFO - step: 25045 loss: 17.0250 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6510 global_avg_mtp_loss: 14.3741 +[titan] 2025-07-09 19:06:55,512 - root - INFO - lr: 2.6257e-04 gnorm: 0.76 [ 5:31:57<16:33:30] +[titan] 2025-07-09 19:06:58,624 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:06:59,413 - root - INFO - step: 25050 loss: 17.1715 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.7259 global_avg_mtp_loss: 14.4456 +[titan] 2025-07-09 19:06:59,413 - root - INFO - lr: 2.6255e-04 gnorm: 1.21 [ 5:32:01<16:33:26] +[titan] 2025-07-09 19:07:03,324 - root - INFO - step: 25055 loss: 17.0261 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.3660 +[titan] 2025-07-09 19:07:03,324 - root - INFO - lr: 2.6254e-04 gnorm: 0.87 [ 5:32:05<16:33:22] +[titan] 2025-07-09 19:07:07,272 - root - INFO - step: 25060 loss: 17.1447 memory: 44.58GiB(31.99%) tps: 82,996 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6803 global_avg_mtp_loss: 14.4643 +[titan] 2025-07-09 19:07:07,272 - root - INFO - lr: 2.6252e-04 gnorm: 0.85 [ 5:32:09<16:33:18] +[titan] 2025-07-09 19:07:11,169 - root - INFO - step: 25065 loss: 16.6530 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0767 +[titan] 2025-07-09 19:07:11,169 - root - INFO - lr: 2.6251e-04 gnorm: 0.87 [ 5:32:13<16:33:13] +[titan] 2025-07-09 19:07:15,074 - root - INFO - step: 25070 loss: 16.6227 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0472 +[titan] 2025-07-09 19:07:15,074 - root - INFO - lr: 2.6249e-04 gnorm: 0.78 [ 5:32:17<16:33:09] +[titan] 2025-07-09 19:07:18,987 - root - INFO - step: 25075 loss: 16.9220 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6312 global_avg_mtp_loss: 14.2908 +[titan] 2025-07-09 19:07:18,987 - root - INFO - lr: 2.6248e-04 gnorm: 0.81 [ 5:32:21<16:33:05] +[titan] 2025-07-09 19:07:22,885 - root - INFO - step: 25080 loss: 16.7746 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5976 global_avg_mtp_loss: 14.1770 +[titan] 2025-07-09 19:07:22,885 - root - INFO - lr: 2.6247e-04 gnorm: 0.76 [ 5:32:25<16:33:01] +[titan] 2025-07-09 19:07:26,892 - root - INFO - step: 25085 loss: 16.7499 memory: 44.58GiB(31.99%) tps: 81,786 tflops: 282.26 mfu: 28.54% global_avg_ntp_loss: 2.6003 global_avg_mtp_loss: 14.1496 +[titan] 2025-07-09 19:07:26,892 - root - INFO - lr: 2.6245e-04 gnorm: 0.92 [ 5:32:29<16:32:57] +[titan] 2025-07-09 19:07:29,407 - root - INFO - Dumping profiler traces at step 25088 +[titan] 2025-07-09 19:07:29,439 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:07:31,006 - root - INFO - step: 25090 loss: 16.9117 memory: 44.58GiB(31.99%) tps: 79,648 tflops: 274.88 mfu: 27.79% global_avg_ntp_loss: 2.6212 global_avg_mtp_loss: 14.2906 +[titan] 2025-07-09 19:07:31,006 - root - INFO - lr: 2.6244e-04 gnorm: 0.84 [ 5:32:33<16:32:53] +[titan] 2025-07-09 19:07:34,922 - root - INFO - step: 25095 loss: 16.9396 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.3211 +[titan] 2025-07-09 19:07:34,922 - root - INFO - lr: 2.6242e-04 gnorm: 0.74 [ 5:32:37<16:32:49] +[titan] 2025-07-09 19:07:38,076 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:07:38,871 - root - INFO - step: 25100 loss: 16.7404 memory: 44.58GiB(31.99%) tps: 82,997 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1485 +[titan] 2025-07-09 19:07:38,871 - root - INFO - lr: 2.6241e-04 gnorm: 0.82 [ 5:32:41<16:32:45] +[titan] 2025-07-09 19:07:42,774 - root - INFO - step: 25105 loss: 16.5062 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 13.9464 +[titan] 2025-07-09 19:07:42,774 - root - INFO - lr: 2.6239e-04 gnorm: 0.76 [ 5:32:45<16:32:41] +[titan] 2025-07-09 19:07:46,676 - root - INFO - step: 25110 loss: 16.6576 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.0717 +[titan] 2025-07-09 19:07:46,676 - root - INFO - lr: 2.6238e-04 gnorm: 1.07 [ 5:32:49<16:32:37] +[titan] 2025-07-09 19:07:50,600 - root - INFO - step: 25115 loss: 16.7864 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6147 global_avg_mtp_loss: 14.1717 +[titan] 2025-07-09 19:07:50,600 - root - INFO - lr: 2.6236e-04 gnorm: 0.79 [ 5:32:52<16:32:33] +[titan] 2025-07-09 19:07:54,517 - root - INFO - step: 25120 loss: 16.9407 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6353 global_avg_mtp_loss: 14.3053 +[titan] 2025-07-09 19:07:54,517 - root - INFO - lr: 2.6235e-04 gnorm: 0.80 [ 5:32:56<16:32:28] +[titan] 2025-07-09 19:07:58,430 - root - INFO - step: 25125 loss: 16.8086 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.1871 +[titan] 2025-07-09 19:07:58,430 - root - INFO - lr: 2.6233e-04 gnorm: 0.78 [ 5:33:00<16:32:24] +[titan] 2025-07-09 19:08:02,336 - root - INFO - step: 25130 loss: 16.6743 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5903 global_avg_mtp_loss: 14.0840 +[titan] 2025-07-09 19:08:02,336 - root - INFO - lr: 2.6232e-04 gnorm: 0.80 [ 5:33:04<16:32:20] +[titan] 2025-07-09 19:08:06,236 - root - INFO - step: 25135 loss: 16.4903 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 13.9434 +[titan] 2025-07-09 19:08:06,236 - root - INFO - lr: 2.6230e-04 gnorm: 0.75 [ 5:33:08<16:32:16] +[titan] 2025-07-09 19:08:10,169 - root - INFO - step: 25140 loss: 16.7254 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5958 global_avg_mtp_loss: 14.1296 +[titan] 2025-07-09 19:08:10,170 - root - INFO - lr: 2.6229e-04 gnorm: 0.78 [ 5:33:12<16:32:12] +[titan] 2025-07-09 19:08:14,080 - root - INFO - step: 25145 loss: 17.0221 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3807 +[titan] 2025-07-09 19:08:14,080 - root - INFO - lr: 2.6227e-04 gnorm: 0.77 [ 5:33:16<16:32:08] +[titan] 2025-07-09 19:08:17,205 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:08:17,998 - root - INFO - step: 25150 loss: 16.9033 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.2707 +[titan] 2025-07-09 19:08:17,998 - root - INFO - lr: 2.6226e-04 gnorm: 0.78 [ 5:33:20<16:32:03] +[titan] 2025-07-09 19:08:21,935 - root - INFO - step: 25155 loss: 16.7305 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5990 global_avg_mtp_loss: 14.1315 +[titan] 2025-07-09 19:08:21,935 - root - INFO - lr: 2.6224e-04 gnorm: 0.78 [ 5:33:24<16:31:59] +[titan] 2025-07-09 19:08:25,840 - root - INFO - step: 25160 loss: 16.7255 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1281 +[titan] 2025-07-09 19:08:25,840 - root - INFO - lr: 2.6223e-04 gnorm: 0.78 [ 5:33:28<16:31:55] +[titan] 2025-07-09 19:08:29,735 - root - INFO - step: 25165 loss: 16.8845 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.2632 +[titan] 2025-07-09 19:08:29,736 - root - INFO - lr: 2.6221e-04 gnorm: 0.87 [ 5:33:32<16:31:51] +[titan] 2025-07-09 19:08:33,649 - root - INFO - step: 25170 loss: 16.9831 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.3499 +[titan] 2025-07-09 19:08:33,649 - root - INFO - lr: 2.6220e-04 gnorm: 0.74 [ 5:33:36<16:31:47] +[titan] 2025-07-09 19:08:37,556 - root - INFO - step: 25175 loss: 16.6749 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.0827 +[titan] 2025-07-09 19:08:37,557 - root - INFO - lr: 2.6218e-04 gnorm: 0.85 [ 5:33:39<16:31:43] +[titan] 2025-07-09 19:08:41,439 - root - INFO - step: 25180 loss: 16.9460 memory: 44.58GiB(31.99%) tps: 84,398 tflops: 291.27 mfu: 29.45% global_avg_ntp_loss: 2.6392 global_avg_mtp_loss: 14.3068 +[titan] 2025-07-09 19:08:41,439 - root - INFO - lr: 2.6217e-04 gnorm: 0.84 [ 5:33:43<16:31:38] +[titan] 2025-07-09 19:08:45,364 - root - INFO - step: 25185 loss: 16.8075 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.1965 +[titan] 2025-07-09 19:08:45,364 - root - INFO - lr: 2.6215e-04 gnorm: 0.75 [ 5:33:47<16:31:34] +[titan] 2025-07-09 19:08:49,276 - root - INFO - step: 25190 loss: 16.8879 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.2638 +[titan] 2025-07-09 19:08:49,276 - root - INFO - lr: 2.6214e-04 gnorm: 0.74 [ 5:33:51<16:31:30] +[titan] 2025-07-09 19:08:53,191 - root - INFO - step: 25195 loss: 17.1569 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6843 global_avg_mtp_loss: 14.4726 +[titan] 2025-07-09 19:08:53,192 - root - INFO - lr: 2.6212e-04 gnorm: 0.81 [ 5:33:55<16:31:26] +[titan] 2025-07-09 19:08:56,300 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:08:57,139 - root - INFO - step: 25200 loss: 17.6486 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.8196 global_avg_mtp_loss: 14.8291 +[titan] 2025-07-09 19:08:57,139 - root - INFO - lr: 2.6211e-04 gnorm: 0.95 [ 5:33:59<16:31:22] +[titan] 2025-07-09 19:09:01,045 - root - INFO - step: 25205 loss: 16.7836 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6097 global_avg_mtp_loss: 14.1740 +[titan] 2025-07-09 19:09:01,045 - root - INFO - lr: 2.6209e-04 gnorm: 0.82 [ 5:34:03<16:31:18] +[titan] 2025-07-09 19:09:04,953 - root - INFO - step: 25210 loss: 16.9539 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6429 global_avg_mtp_loss: 14.3110 +[titan] 2025-07-09 19:09:04,954 - root - INFO - lr: 2.6208e-04 gnorm: 0.79 [ 5:34:07<16:31:13] +[titan] 2025-07-09 19:09:08,871 - root - INFO - step: 25215 loss: 16.7724 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.1614 +[titan] 2025-07-09 19:09:08,871 - root - INFO - lr: 2.6206e-04 gnorm: 0.75 [ 5:34:11<16:31:09] +[titan] 2025-07-09 19:09:12,795 - root - INFO - step: 25220 loss: 16.7437 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1495 +[titan] 2025-07-09 19:09:12,795 - root - INFO - lr: 2.6205e-04 gnorm: 0.76 [ 5:34:15<16:31:05] +[titan] 2025-07-09 19:09:16,726 - root - INFO - step: 25225 loss: 16.5630 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 13.9918 +[titan] 2025-07-09 19:09:16,727 - root - INFO - lr: 2.6203e-04 gnorm: 0.77 [ 5:34:19<16:31:01] +[titan] 2025-07-09 19:09:20,666 - root - INFO - step: 25230 loss: 17.2627 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6966 global_avg_mtp_loss: 14.5660 +[titan] 2025-07-09 19:09:20,666 - root - INFO - lr: 2.6202e-04 gnorm: 0.73 [ 5:34:23<16:30:57] +[titan] 2025-07-09 19:09:24,595 - root - INFO - step: 25235 loss: 16.9294 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.3030 +[titan] 2025-07-09 19:09:24,595 - root - INFO - lr: 2.6200e-04 gnorm: 0.80 [ 5:34:26<16:30:53] +[titan] 2025-07-09 19:09:28,537 - root - INFO - step: 25240 loss: 16.8342 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2256 +[titan] 2025-07-09 19:09:28,538 - root - INFO - lr: 2.6199e-04 gnorm: 0.78 [ 5:34:30<16:30:49] +[titan] 2025-07-09 19:09:32,472 - root - INFO - step: 25245 loss: 17.1936 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6861 global_avg_mtp_loss: 14.5075 +[titan] 2025-07-09 19:09:32,473 - root - INFO - lr: 2.6197e-04 gnorm: 0.79 [ 5:34:34<16:30:45] +[titan] 2025-07-09 19:09:35,582 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:09:36,373 - root - INFO - step: 25250 loss: 16.7033 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1185 +[titan] 2025-07-09 19:09:36,373 - root - INFO - lr: 2.6196e-04 gnorm: 0.75 [ 5:34:38<16:30:40] +[titan] 2025-07-09 19:09:40,284 - root - INFO - step: 25255 loss: 16.9127 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6205 global_avg_mtp_loss: 14.2922 +[titan] 2025-07-09 19:09:40,285 - root - INFO - lr: 2.6194e-04 gnorm: 0.82 [ 5:34:42<16:30:36] +[titan] 2025-07-09 19:09:44,183 - root - INFO - step: 25260 loss: 16.9631 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6400 global_avg_mtp_loss: 14.3231 +[titan] 2025-07-09 19:09:44,184 - root - INFO - lr: 2.6193e-04 gnorm: 0.75 [ 5:34:46<16:30:32] +[titan] 2025-07-09 19:09:48,115 - root - INFO - step: 25265 loss: 16.9709 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.3244 +[titan] 2025-07-09 19:09:48,115 - root - INFO - lr: 2.6191e-04 gnorm: 0.76 [ 5:34:50<16:30:28] +[titan] 2025-07-09 19:09:52,021 - root - INFO - step: 25270 loss: 17.0068 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6381 global_avg_mtp_loss: 14.3687 +[titan] 2025-07-09 19:09:52,022 - root - INFO - lr: 2.6190e-04 gnorm: 0.74 [ 5:34:54<16:30:24] +[titan] 2025-07-09 19:09:55,950 - root - INFO - step: 25275 loss: 16.8247 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6122 global_avg_mtp_loss: 14.2125 +[titan] 2025-07-09 19:09:55,950 - root - INFO - lr: 2.6189e-04 gnorm: 0.78 [ 5:34:58<16:30:20] +[titan] 2025-07-09 19:09:59,871 - root - INFO - step: 25280 loss: 16.9151 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.2809 +[titan] 2025-07-09 19:09:59,871 - root - INFO - lr: 2.6187e-04 gnorm: 0.76 [ 5:35:02<16:30:16] +[titan] 2025-07-09 19:10:03,763 - root - INFO - step: 25285 loss: 16.6595 memory: 44.58GiB(31.99%) tps: 84,204 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.0784 +[titan] 2025-07-09 19:10:03,763 - root - INFO - lr: 2.6186e-04 gnorm: 0.75 [ 5:35:06<16:30:11] +[titan] 2025-07-09 19:10:07,677 - root - INFO - step: 25290 loss: 16.6125 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5680 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-09 19:10:07,677 - root - INFO - lr: 2.6184e-04 gnorm: 0.76 [ 5:35:10<16:30:07] +[titan] 2025-07-09 19:10:11,590 - root - INFO - step: 25295 loss: 16.9172 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6416 global_avg_mtp_loss: 14.2755 +[titan] 2025-07-09 19:10:11,590 - root - INFO - lr: 2.6183e-04 gnorm: 0.78 [ 5:35:13<16:30:03] +[titan] 2025-07-09 19:10:14,711 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:10:15,498 - root - INFO - step: 25300 loss: 16.7667 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 19:10:15,498 - root - INFO - lr: 2.6181e-04 gnorm: 0.80 [ 5:35:17<16:29:59] +[titan] 2025-07-09 19:10:19,416 - root - INFO - step: 25305 loss: 16.9874 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6379 global_avg_mtp_loss: 14.3496 +[titan] 2025-07-09 19:10:19,416 - root - INFO - lr: 2.6180e-04 gnorm: 0.75 [ 5:35:21<16:29:55] +[titan] 2025-07-09 19:10:23,328 - root - INFO - step: 25310 loss: 17.0574 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6572 global_avg_mtp_loss: 14.4002 +[titan] 2025-07-09 19:10:23,328 - root - INFO - lr: 2.6178e-04 gnorm: 0.81 [ 5:35:25<16:29:51] +[titan] 2025-07-09 19:10:27,249 - root - INFO - step: 25315 loss: 16.8570 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6309 global_avg_mtp_loss: 14.2260 +[titan] 2025-07-09 19:10:27,249 - root - INFO - lr: 2.6177e-04 gnorm: 0.76 [ 5:35:29<16:29:46] +[titan] 2025-07-09 19:10:31,172 - root - INFO - step: 25320 loss: 17.1865 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6803 global_avg_mtp_loss: 14.5062 +[titan] 2025-07-09 19:10:31,172 - root - INFO - lr: 2.6175e-04 gnorm: 0.76 [ 5:35:33<16:29:42] +[titan] 2025-07-09 19:10:35,097 - root - INFO - step: 25325 loss: 16.9508 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.3086 +[titan] 2025-07-09 19:10:35,097 - root - INFO - lr: 2.6174e-04 gnorm: 0.77 [ 5:35:37<16:29:38] +[titan] 2025-07-09 19:10:39,014 - root - INFO - step: 25330 loss: 16.8077 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5955 global_avg_mtp_loss: 14.2121 +[titan] 2025-07-09 19:10:39,014 - root - INFO - lr: 2.6172e-04 gnorm: 0.76 [ 5:35:41<16:29:34] +[titan] 2025-07-09 19:10:42,925 - root - INFO - step: 25335 loss: 16.9001 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2761 +[titan] 2025-07-09 19:10:42,925 - root - INFO - lr: 2.6171e-04 gnorm: 0.83 [ 5:35:45<16:29:30] +[titan] 2025-07-09 19:10:46,842 - root - INFO - step: 25340 loss: 16.8736 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6295 global_avg_mtp_loss: 14.2442 +[titan] 2025-07-09 19:10:46,842 - root - INFO - lr: 2.6169e-04 gnorm: 0.78 [ 5:35:49<16:29:26] +[titan] 2025-07-09 19:10:50,763 - root - INFO - step: 25345 loss: 16.9118 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6289 global_avg_mtp_loss: 14.2829 +[titan] 2025-07-09 19:10:50,763 - root - INFO - lr: 2.6168e-04 gnorm: 0.79 [ 5:35:53<16:29:21] +[titan] 2025-07-09 19:10:53,884 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:10:54,681 - root - INFO - step: 25350 loss: 16.8321 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6166 global_avg_mtp_loss: 14.2154 +[titan] 2025-07-09 19:10:54,682 - root - INFO - lr: 2.6166e-04 gnorm: 0.85 [ 5:35:57<16:29:17] +[titan] 2025-07-09 19:10:58,585 - root - INFO - step: 25355 loss: 16.8452 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6134 global_avg_mtp_loss: 14.2317 +[titan] 2025-07-09 19:10:58,586 - root - INFO - lr: 2.6165e-04 gnorm: 0.78 [ 5:36:00<16:29:13] +[titan] 2025-07-09 19:11:02,479 - root - INFO - step: 25360 loss: 16.4493 memory: 44.58GiB(31.99%) tps: 84,177 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 13.8942 +[titan] 2025-07-09 19:11:02,479 - root - INFO - lr: 2.6163e-04 gnorm: 0.76 [ 5:36:04<16:29:09] +[titan] 2025-07-09 19:11:06,393 - root - INFO - step: 25365 loss: 16.8687 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2522 +[titan] 2025-07-09 19:11:06,393 - root - INFO - lr: 2.6162e-04 gnorm: 0.80 [ 5:36:08<16:29:05] +[titan] 2025-07-09 19:11:10,296 - root - INFO - step: 25370 loss: 16.8213 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6147 global_avg_mtp_loss: 14.2066 +[titan] 2025-07-09 19:11:10,297 - root - INFO - lr: 2.6160e-04 gnorm: 0.76 [ 5:36:12<16:29:01] +[titan] 2025-07-09 19:11:14,204 - root - INFO - step: 25375 loss: 16.7568 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.1540 +[titan] 2025-07-09 19:11:14,205 - root - INFO - lr: 2.6159e-04 gnorm: 0.80 [ 5:36:16<16:28:56] +[titan] 2025-07-09 19:11:18,111 - root - INFO - step: 25380 loss: 16.9098 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2891 +[titan] 2025-07-09 19:11:18,111 - root - INFO - lr: 2.6157e-04 gnorm: 0.76 [ 5:36:20<16:28:52] +[titan] 2025-07-09 19:11:22,022 - root - INFO - step: 25385 loss: 17.0521 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6649 global_avg_mtp_loss: 14.3872 +[titan] 2025-07-09 19:11:22,023 - root - INFO - lr: 2.6156e-04 gnorm: 0.76 [ 5:36:24<16:28:48] +[titan] 2025-07-09 19:11:25,934 - root - INFO - step: 25390 loss: 16.7818 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5957 global_avg_mtp_loss: 14.1861 +[titan] 2025-07-09 19:11:25,934 - root - INFO - lr: 2.6154e-04 gnorm: 0.81 [ 5:36:28<16:28:44] +[titan] 2025-07-09 19:11:29,854 - root - INFO - step: 25395 loss: 16.8971 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.2807 +[titan] 2025-07-09 19:11:29,854 - root - INFO - lr: 2.6153e-04 gnorm: 0.81 [ 5:36:32<16:28:40] +[titan] 2025-07-09 19:11:33,001 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:11:33,799 - root - INFO - step: 25400 loss: 16.9232 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.6367 global_avg_mtp_loss: 14.2865 +[titan] 2025-07-09 19:11:33,800 - root - INFO - lr: 2.6151e-04 gnorm: 0.80 [ 5:36:36<16:28:36] +[titan] 2025-07-09 19:11:37,695 - root - INFO - step: 25405 loss: 17.0206 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6554 global_avg_mtp_loss: 14.3652 +[titan] 2025-07-09 19:11:37,696 - root - INFO - lr: 2.6150e-04 gnorm: 0.83 [ 5:36:40<16:28:31] +[titan] 2025-07-09 19:11:41,610 - root - INFO - step: 25410 loss: 17.1063 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6636 global_avg_mtp_loss: 14.4426 +[titan] 2025-07-09 19:11:41,610 - root - INFO - lr: 2.6148e-04 gnorm: 0.75 [ 5:36:43<16:28:27] +[titan] 2025-07-09 19:11:45,510 - root - INFO - step: 25415 loss: 16.8345 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.2235 +[titan] 2025-07-09 19:11:45,511 - root - INFO - lr: 2.6147e-04 gnorm: 0.89 [ 5:36:47<16:28:23] +[titan] 2025-07-09 19:11:49,421 - root - INFO - step: 25420 loss: 16.7838 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6054 global_avg_mtp_loss: 14.1785 +[titan] 2025-07-09 19:11:49,421 - root - INFO - lr: 2.6145e-04 gnorm: 0.78 [ 5:36:51<16:28:19] +[titan] 2025-07-09 19:11:53,352 - root - INFO - step: 25425 loss: 16.9640 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.6391 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 19:11:53,352 - root - INFO - lr: 2.6144e-04 gnorm: 0.79 [ 5:36:55<16:28:15] +[titan] 2025-07-09 19:11:57,296 - root - INFO - step: 25430 loss: 16.6101 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.0359 +[titan] 2025-07-09 19:11:57,296 - root - INFO - lr: 2.6142e-04 gnorm: 0.74 [ 5:36:59<16:28:11] +[titan] 2025-07-09 19:12:01,206 - root - INFO - step: 25435 loss: 16.9155 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.2753 +[titan] 2025-07-09 19:12:01,206 - root - INFO - lr: 2.6141e-04 gnorm: 0.81 [ 5:37:03<16:28:07] +[titan] 2025-07-09 19:12:05,141 - root - INFO - step: 25440 loss: 16.6764 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.0873 +[titan] 2025-07-09 19:12:05,141 - root - INFO - lr: 2.6139e-04 gnorm: 0.82 [ 5:37:07<16:28:03] +[titan] 2025-07-09 19:12:09,043 - root - INFO - step: 25445 loss: 16.9535 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.3164 +[titan] 2025-07-09 19:12:09,044 - root - INFO - lr: 2.6138e-04 gnorm: 0.82 [ 5:37:11<16:27:58] +[titan] 2025-07-09 19:12:12,172 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:12:12,963 - root - INFO - step: 25450 loss: 16.8072 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.1850 +[titan] 2025-07-09 19:12:12,963 - root - INFO - lr: 2.6136e-04 gnorm: 0.76 [ 5:37:15<16:27:54] +[titan] 2025-07-09 19:12:16,933 - root - INFO - step: 25455 loss: 16.7406 memory: 44.58GiB(31.99%) tps: 82,553 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.6064 global_avg_mtp_loss: 14.1342 +[titan] 2025-07-09 19:12:16,933 - root - INFO - lr: 2.6135e-04 gnorm: 0.85 [ 5:37:19<16:27:50] +[titan] 2025-07-09 19:12:21,193 - root - INFO - step: 25460 loss: 16.8588 memory: 44.58GiB(31.99%) tps: 76,931 tflops: 265.50 mfu: 26.85% global_avg_ntp_loss: 2.6251 global_avg_mtp_loss: 14.2337 +[titan] 2025-07-09 19:12:21,193 - root - INFO - lr: 2.6133e-04 gnorm: 0.81 [ 5:37:23<16:27:47] +[titan] 2025-07-09 19:12:25,107 - root - INFO - step: 25465 loss: 16.6886 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.0998 +[titan] 2025-07-09 19:12:25,107 - root - INFO - lr: 2.6132e-04 gnorm: 0.77 [ 5:37:27<16:27:43] +[titan] 2025-07-09 19:12:29,062 - root - INFO - step: 25470 loss: 16.4927 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9409 +[titan] 2025-07-09 19:12:29,063 - root - INFO - lr: 2.6130e-04 gnorm: 0.79 [ 5:37:31<16:27:39] +[titan] 2025-07-09 19:12:32,971 - root - INFO - step: 25475 loss: 16.9186 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6336 global_avg_mtp_loss: 14.2850 +[titan] 2025-07-09 19:12:32,971 - root - INFO - lr: 2.6129e-04 gnorm: 0.77 [ 5:37:35<16:27:35] +[titan] 2025-07-09 19:12:36,956 - root - INFO - step: 25480 loss: 16.7372 memory: 44.58GiB(31.99%) tps: 82,230 tflops: 283.79 mfu: 28.69% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1468 +[titan] 2025-07-09 19:12:36,956 - root - INFO - lr: 2.6127e-04 gnorm: 0.76 [ 5:37:39<16:27:31] +[titan] 2025-07-09 19:12:40,869 - root - INFO - step: 25485 loss: 16.9063 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2792 +[titan] 2025-07-09 19:12:40,869 - root - INFO - lr: 2.6126e-04 gnorm: 0.84 [ 5:37:43<16:27:27] +[titan] 2025-07-09 19:12:44,769 - root - INFO - step: 25490 loss: 17.0427 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6535 global_avg_mtp_loss: 14.3892 +[titan] 2025-07-09 19:12:44,769 - root - INFO - lr: 2.6124e-04 gnorm: 0.75 [ 5:37:47<16:27:22] +[titan] 2025-07-09 19:12:48,676 - root - INFO - step: 25495 loss: 16.7609 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6064 global_avg_mtp_loss: 14.1545 +[titan] 2025-07-09 19:12:48,676 - root - INFO - lr: 2.6123e-04 gnorm: 0.70 [ 5:37:51<16:27:18] +[titan] 2025-07-09 19:12:51,819 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:12:52,623 - root - INFO - step: 25500 loss: 16.4750 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-09 19:12:52,623 - root - INFO - lr: 2.6121e-04 gnorm: 0.81 [ 5:37:54<16:27:14] +[titan] 2025-07-09 19:12:56,532 - root - INFO - step: 25505 loss: 16.9486 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3049 +[titan] 2025-07-09 19:12:56,533 - root - INFO - lr: 2.6120e-04 gnorm: 0.78 [ 5:37:58<16:27:10] +[titan] 2025-07-09 19:13:00,471 - root - INFO - step: 25510 loss: 17.0862 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.6618 global_avg_mtp_loss: 14.4244 +[titan] 2025-07-09 19:13:00,472 - root - INFO - lr: 2.6118e-04 gnorm: 0.75 [ 5:38:02<16:27:06] +[titan] 2025-07-09 19:13:04,402 - root - INFO - step: 25515 loss: 16.5170 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 13.9545 +[titan] 2025-07-09 19:13:04,402 - root - INFO - lr: 2.6117e-04 gnorm: 0.75 [ 5:38:06<16:27:02] +[titan] 2025-07-09 19:13:08,299 - root - INFO - step: 25520 loss: 16.8226 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6051 global_avg_mtp_loss: 14.2175 +[titan] 2025-07-09 19:13:08,299 - root - INFO - lr: 2.6115e-04 gnorm: 0.89 [ 5:38:10<16:26:58] +[titan] 2025-07-09 19:13:12,221 - root - INFO - step: 25525 loss: 16.9652 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3096 +[titan] 2025-07-09 19:13:12,221 - root - INFO - lr: 2.6114e-04 gnorm: 0.78 [ 5:38:14<16:26:53] +[titan] 2025-07-09 19:13:16,124 - root - INFO - step: 25530 loss: 16.7993 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.1793 +[titan] 2025-07-09 19:13:16,124 - root - INFO - lr: 2.6112e-04 gnorm: 0.83 [ 5:38:18<16:26:49] +[titan] 2025-07-09 19:13:20,036 - root - INFO - step: 25535 loss: 16.7690 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.1750 +[titan] 2025-07-09 19:13:20,036 - root - INFO - lr: 2.6111e-04 gnorm: 0.82 [ 5:38:22<16:26:45] +[titan] 2025-07-09 19:13:23,936 - root - INFO - step: 25540 loss: 17.0279 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6494 global_avg_mtp_loss: 14.3785 +[titan] 2025-07-09 19:13:23,937 - root - INFO - lr: 2.6109e-04 gnorm: 0.79 [ 5:38:26<16:26:41] +[titan] 2025-07-09 19:13:27,845 - root - INFO - step: 25545 loss: 16.7760 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.1767 +[titan] 2025-07-09 19:13:27,846 - root - INFO - lr: 2.6108e-04 gnorm: 0.75 [ 5:38:30<16:26:37] +[titan] 2025-07-09 19:13:30,974 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:13:31,762 - root - INFO - step: 25550 loss: 16.8529 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6064 global_avg_mtp_loss: 14.2465 +[titan] 2025-07-09 19:13:31,762 - root - INFO - lr: 2.6106e-04 gnorm: 0.81 [ 5:38:34<16:26:33] +[titan] 2025-07-09 19:13:35,715 - root - INFO - step: 25555 loss: 16.8666 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.2553 +[titan] 2025-07-09 19:13:35,715 - root - INFO - lr: 2.6105e-04 gnorm: 0.77 [ 5:38:38<16:26:29] +[titan] 2025-07-09 19:13:39,618 - root - INFO - step: 25560 loss: 16.8485 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6143 global_avg_mtp_loss: 14.2342 +[titan] 2025-07-09 19:13:39,618 - root - INFO - lr: 2.6103e-04 gnorm: 0.81 [ 5:38:41<16:26:24] +[titan] 2025-07-09 19:13:43,504 - root - INFO - step: 25565 loss: 17.0439 memory: 44.58GiB(31.99%) tps: 84,322 tflops: 291.01 mfu: 29.42% global_avg_ntp_loss: 2.6518 global_avg_mtp_loss: 14.3921 +[titan] 2025-07-09 19:13:43,505 - root - INFO - lr: 2.6102e-04 gnorm: 0.77 [ 5:38:45<16:26:20] +[titan] 2025-07-09 19:13:47,418 - root - INFO - step: 25570 loss: 16.8692 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.2551 +[titan] 2025-07-09 19:13:47,418 - root - INFO - lr: 2.6100e-04 gnorm: 0.76 [ 5:38:49<16:26:16] +[titan] 2025-07-09 19:13:51,353 - root - INFO - step: 25575 loss: 16.7397 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.1606 +[titan] 2025-07-09 19:13:51,354 - root - INFO - lr: 2.6099e-04 gnorm: 0.81 [ 5:38:53<16:26:12] +[titan] 2025-07-09 19:13:55,264 - root - INFO - step: 25580 loss: 16.8725 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6112 global_avg_mtp_loss: 14.2613 +[titan] 2025-07-09 19:13:55,264 - root - INFO - lr: 2.6097e-04 gnorm: 0.83 [ 5:38:57<16:26:08] +[titan] 2025-07-09 19:13:59,210 - root - INFO - step: 25585 loss: 16.9353 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.3145 +[titan] 2025-07-09 19:13:59,211 - root - INFO - lr: 2.6096e-04 gnorm: 0.83 [ 5:39:01<16:26:04] +[titan] 2025-07-09 19:14:03,142 - root - INFO - step: 25590 loss: 17.0107 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6603 global_avg_mtp_loss: 14.3504 +[titan] 2025-07-09 19:14:03,142 - root - INFO - lr: 2.6094e-04 gnorm: 0.75 [ 5:39:05<16:26:00] +[titan] 2025-07-09 19:14:07,057 - root - INFO - step: 25595 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5900 global_avg_mtp_loss: 14.0723 +[titan] 2025-07-09 19:14:07,057 - root - INFO - lr: 2.6093e-04 gnorm: 0.77 [ 5:39:09<16:25:55] +[titan] 2025-07-09 19:14:10,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:14:11,088 - root - INFO - step: 25600 loss: 16.8306 memory: 44.58GiB(31.99%) tps: 81,297 tflops: 280.57 mfu: 28.37% global_avg_ntp_loss: 2.6227 global_avg_mtp_loss: 14.2080 +[titan] 2025-07-09 19:14:11,088 - root - INFO - lr: 2.6091e-04 gnorm: 0.77 [ 5:39:13<16:25:52] +[titan] 2025-07-09 19:14:11,245 - root - INFO - Dumping profiler traces at step 25600 +[titan] 2025-07-09 19:14:11,279 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:14:15,182 - root - INFO - step: 25605 loss: 16.9447 memory: 44.58GiB(31.99%) tps: 80,052 tflops: 276.27 mfu: 27.93% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.3209 +[titan] 2025-07-09 19:14:15,182 - root - INFO - lr: 2.6090e-04 gnorm: 0.81 [ 5:39:17<16:25:48] +[titan] 2025-07-09 19:14:19,133 - root - INFO - step: 25610 loss: 16.8279 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.2211 +[titan] 2025-07-09 19:14:19,133 - root - INFO - lr: 2.6088e-04 gnorm: 0.78 [ 5:39:21<16:25:44] +[titan] 2025-07-09 19:14:23,053 - root - INFO - step: 25615 loss: 16.7112 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1216 +[titan] 2025-07-09 19:14:23,053 - root - INFO - lr: 2.6087e-04 gnorm: 0.81 [ 5:39:25<16:25:40] +[titan] 2025-07-09 19:14:26,987 - root - INFO - step: 25620 loss: 16.6900 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 19:14:26,987 - root - INFO - lr: 2.6085e-04 gnorm: 0.75 [ 5:39:29<16:25:36] +[titan] 2025-07-09 19:14:30,876 - root - INFO - step: 25625 loss: 16.7778 memory: 44.58GiB(31.99%) tps: 84,256 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.6139 global_avg_mtp_loss: 14.1638 +[titan] 2025-07-09 19:14:30,877 - root - INFO - lr: 2.6084e-04 gnorm: 0.79 [ 5:39:33<16:25:31] +[titan] 2025-07-09 19:14:34,802 - root - INFO - step: 25630 loss: 16.9524 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.3217 +[titan] 2025-07-09 19:14:34,802 - root - INFO - lr: 2.6082e-04 gnorm: 0.82 [ 5:39:37<16:25:27] +[titan] 2025-07-09 19:14:38,736 - root - INFO - step: 25635 loss: 16.7353 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1430 +[titan] 2025-07-09 19:14:38,736 - root - INFO - lr: 2.6080e-04 gnorm: 0.76 [ 5:39:41<16:25:23] +[titan] 2025-07-09 19:14:42,662 - root - INFO - step: 25640 loss: 16.7663 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1718 +[titan] 2025-07-09 19:14:42,663 - root - INFO - lr: 2.6079e-04 gnorm: 0.82 [ 5:39:44<16:25:19] +[titan] 2025-07-09 19:14:46,588 - root - INFO - step: 25645 loss: 16.9567 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.3276 +[titan] 2025-07-09 19:14:46,588 - root - INFO - lr: 2.6077e-04 gnorm: 0.77 [ 5:39:48<16:25:15] +[titan] 2025-07-09 19:14:49,700 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:14:50,493 - root - INFO - step: 25650 loss: 16.5744 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 14.0181 +[titan] 2025-07-09 19:14:50,493 - root - INFO - lr: 2.6076e-04 gnorm: 0.76 [ 5:39:52<16:25:11] +[titan] 2025-07-09 19:14:54,426 - root - INFO - step: 25655 loss: 16.8988 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2807 +[titan] 2025-07-09 19:14:54,427 - root - INFO - lr: 2.6074e-04 gnorm: 0.75 [ 5:39:56<16:25:07] +[titan] 2025-07-09 19:14:58,346 - root - INFO - step: 25660 loss: 16.9841 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6300 global_avg_mtp_loss: 14.3541 +[titan] 2025-07-09 19:14:58,346 - root - INFO - lr: 2.6073e-04 gnorm: 0.73 [ 5:40:00<16:25:03] +[titan] 2025-07-09 19:15:02,268 - root - INFO - step: 25665 loss: 16.4680 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 13.9102 +[titan] 2025-07-09 19:15:02,268 - root - INFO - lr: 2.6071e-04 gnorm: 0.74 [ 5:40:04<16:24:58] +[titan] 2025-07-09 19:15:06,209 - root - INFO - step: 25670 loss: 16.8296 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2108 +[titan] 2025-07-09 19:15:06,209 - root - INFO - lr: 2.6070e-04 gnorm: 0.72 [ 5:40:08<16:24:54] +[titan] 2025-07-09 19:15:10,124 - root - INFO - step: 25675 loss: 17.1406 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6666 global_avg_mtp_loss: 14.4740 +[titan] 2025-07-09 19:15:10,124 - root - INFO - lr: 2.6068e-04 gnorm: 0.75 [ 5:40:12<16:24:50] +[titan] 2025-07-09 19:15:14,047 - root - INFO - step: 25680 loss: 16.6424 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0679 +[titan] 2025-07-09 19:15:14,048 - root - INFO - lr: 2.6067e-04 gnorm: 0.76 [ 5:40:16<16:24:46] +[titan] 2025-07-09 19:15:17,947 - root - INFO - step: 25685 loss: 16.9011 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6322 global_avg_mtp_loss: 14.2689 +[titan] 2025-07-09 19:15:17,948 - root - INFO - lr: 2.6065e-04 gnorm: 0.76 [ 5:40:20<16:24:42] +[titan] 2025-07-09 19:15:21,892 - root - INFO - step: 25690 loss: 16.8942 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.6428 global_avg_mtp_loss: 14.2514 +[titan] 2025-07-09 19:15:21,893 - root - INFO - lr: 2.6064e-04 gnorm: 0.78 [ 5:40:24<16:24:38] +[titan] 2025-07-09 19:15:25,807 - root - INFO - step: 25695 loss: 16.9697 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6341 global_avg_mtp_loss: 14.3355 +[titan] 2025-07-09 19:15:25,807 - root - INFO - lr: 2.6062e-04 gnorm: 0.78 [ 5:40:28<16:24:34] +[titan] 2025-07-09 19:15:28,937 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:15:29,728 - root - INFO - step: 25700 loss: 16.6655 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.0794 +[titan] 2025-07-09 19:15:29,728 - root - INFO - lr: 2.6061e-04 gnorm: 0.77 [ 5:40:32<16:24:30] +[titan] 2025-07-09 19:15:33,676 - root - INFO - step: 25705 loss: 16.9179 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.3064 +[titan] 2025-07-09 19:15:33,676 - root - INFO - lr: 2.6059e-04 gnorm: 0.77 [ 5:40:35<16:24:25] +[titan] 2025-07-09 19:15:37,592 - root - INFO - step: 25710 loss: 16.8622 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2387 +[titan] 2025-07-09 19:15:37,592 - root - INFO - lr: 2.6058e-04 gnorm: 0.70 [ 5:40:39<16:24:21] +[titan] 2025-07-09 19:15:41,518 - root - INFO - step: 25715 loss: 16.8404 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.2293 +[titan] 2025-07-09 19:15:41,518 - root - INFO - lr: 2.6056e-04 gnorm: 0.79 [ 5:40:43<16:24:17] +[titan] 2025-07-09 19:15:45,436 - root - INFO - step: 25720 loss: 16.6880 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.0968 +[titan] 2025-07-09 19:15:45,436 - root - INFO - lr: 2.6055e-04 gnorm: 0.76 [ 5:40:47<16:24:13] +[titan] 2025-07-09 19:15:49,334 - root - INFO - step: 25725 loss: 17.0062 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.3399 +[titan] 2025-07-09 19:15:49,334 - root - INFO - lr: 2.6053e-04 gnorm: 0.79 [ 5:40:51<16:24:09] +[titan] 2025-07-09 19:15:53,239 - root - INFO - step: 25730 loss: 16.8001 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.1972 +[titan] 2025-07-09 19:15:53,240 - root - INFO - lr: 2.6052e-04 gnorm: 0.80 [ 5:40:55<16:24:05] +[titan] 2025-07-09 19:15:57,154 - root - INFO - step: 25735 loss: 16.8561 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6190 global_avg_mtp_loss: 14.2371 +[titan] 2025-07-09 19:15:57,154 - root - INFO - lr: 2.6050e-04 gnorm: 0.76 [ 5:40:59<16:24:01] +[titan] 2025-07-09 19:16:01,052 - root - INFO - step: 25740 loss: 16.7900 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.1774 +[titan] 2025-07-09 19:16:01,052 - root - INFO - lr: 2.6049e-04 gnorm: 0.74 [ 5:41:03<16:23:56] +[titan] 2025-07-09 19:16:04,992 - root - INFO - step: 25745 loss: 16.7434 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.1367 +[titan] 2025-07-09 19:16:04,992 - root - INFO - lr: 2.6047e-04 gnorm: 0.76 [ 5:41:07<16:23:52] +[titan] 2025-07-09 19:16:08,135 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:16:08,929 - root - INFO - step: 25750 loss: 16.6141 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5732 global_avg_mtp_loss: 14.0409 +[titan] 2025-07-09 19:16:08,930 - root - INFO - lr: 2.6046e-04 gnorm: 0.87 [ 5:41:11<16:23:48] +[titan] 2025-07-09 19:16:12,875 - root - INFO - step: 25755 loss: 16.8872 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.6251 global_avg_mtp_loss: 14.2621 +[titan] 2025-07-09 19:16:12,875 - root - INFO - lr: 2.6044e-04 gnorm: 0.75 [ 5:41:15<16:23:44] +[titan] 2025-07-09 19:16:16,815 - root - INFO - step: 25760 loss: 16.7297 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5917 global_avg_mtp_loss: 14.1380 +[titan] 2025-07-09 19:16:16,815 - root - INFO - lr: 2.6043e-04 gnorm: 0.75 [ 5:41:19<16:23:40] +[titan] 2025-07-09 19:16:20,725 - root - INFO - step: 25765 loss: 16.9724 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3329 +[titan] 2025-07-09 19:16:20,725 - root - INFO - lr: 2.6041e-04 gnorm: 0.78 [ 5:41:23<16:23:36] +[titan] 2025-07-09 19:16:24,629 - root - INFO - step: 25770 loss: 16.7002 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1084 +[titan] 2025-07-09 19:16:24,629 - root - INFO - lr: 2.6040e-04 gnorm: 0.77 [ 5:41:26<16:23:32] +[titan] 2025-07-09 19:16:28,564 - root - INFO - step: 25775 loss: 16.8226 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.1988 +[titan] 2025-07-09 19:16:28,564 - root - INFO - lr: 2.6038e-04 gnorm: 0.76 [ 5:41:30<16:23:28] +[titan] 2025-07-09 19:16:32,505 - root - INFO - step: 25780 loss: 16.8233 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.2100 +[titan] 2025-07-09 19:16:32,505 - root - INFO - lr: 2.6037e-04 gnorm: 0.76 [ 5:41:34<16:23:24] +[titan] 2025-07-09 19:16:36,396 - root - INFO - step: 25785 loss: 17.0108 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6510 global_avg_mtp_loss: 14.3598 +[titan] 2025-07-09 19:16:36,396 - root - INFO - lr: 2.6035e-04 gnorm: 0.75 [ 5:41:38<16:23:19] +[titan] 2025-07-09 19:16:40,298 - root - INFO - step: 25790 loss: 16.7367 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5935 global_avg_mtp_loss: 14.1432 +[titan] 2025-07-09 19:16:40,298 - root - INFO - lr: 2.6034e-04 gnorm: 0.89 [ 5:41:42<16:23:15] +[titan] 2025-07-09 19:16:44,213 - root - INFO - step: 25795 loss: 16.5756 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0080 +[titan] 2025-07-09 19:16:44,213 - root - INFO - lr: 2.6032e-04 gnorm: 0.92 [ 5:41:46<16:23:11] +[titan] 2025-07-09 19:16:47,334 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:16:48,125 - root - INFO - step: 25800 loss: 16.8597 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.6130 global_avg_mtp_loss: 14.2467 +[titan] 2025-07-09 19:16:48,125 - root - INFO - lr: 2.6031e-04 gnorm: 0.80 [ 5:41:50<16:23:07] +[titan] 2025-07-09 19:16:52,038 - root - INFO - step: 25805 loss: 16.6843 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.0932 +[titan] 2025-07-09 19:16:52,038 - root - INFO - lr: 2.6029e-04 gnorm: 0.78 [ 5:41:54<16:23:03] +[titan] 2025-07-09 19:16:55,946 - root - INFO - step: 25810 loss: 16.9097 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.2862 +[titan] 2025-07-09 19:16:55,946 - root - INFO - lr: 2.6028e-04 gnorm: 0.79 [ 5:41:58<16:22:59] +[titan] 2025-07-09 19:16:59,856 - root - INFO - step: 25815 loss: 16.7697 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.1679 +[titan] 2025-07-09 19:16:59,856 - root - INFO - lr: 2.6026e-04 gnorm: 0.80 [ 5:42:02<16:22:54] +[titan] 2025-07-09 19:17:03,768 - root - INFO - step: 25820 loss: 17.0113 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6545 global_avg_mtp_loss: 14.3568 +[titan] 2025-07-09 19:17:03,768 - root - INFO - lr: 2.6024e-04 gnorm: 0.84 [ 5:42:06<16:22:50] +[titan] 2025-07-09 19:17:07,669 - root - INFO - step: 25825 loss: 16.4702 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9277 +[titan] 2025-07-09 19:17:07,669 - root - INFO - lr: 2.6023e-04 gnorm: 0.73 [ 5:42:09<16:22:46] +[titan] 2025-07-09 19:17:11,568 - root - INFO - step: 25830 loss: 16.8057 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5952 global_avg_mtp_loss: 14.2105 +[titan] 2025-07-09 19:17:11,569 - root - INFO - lr: 2.6021e-04 gnorm: 0.83 [ 5:42:13<16:22:42] +[titan] 2025-07-09 19:17:15,484 - root - INFO - step: 25835 loss: 16.9089 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6408 global_avg_mtp_loss: 14.2681 +[titan] 2025-07-09 19:17:15,484 - root - INFO - lr: 2.6020e-04 gnorm: 0.75 [ 5:42:17<16:22:38] +[titan] 2025-07-09 19:17:19,434 - root - INFO - step: 25840 loss: 16.9095 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.2791 +[titan] 2025-07-09 19:17:19,434 - root - INFO - lr: 2.6018e-04 gnorm: 0.84 [ 5:42:21<16:22:34] +[titan] 2025-07-09 19:17:23,344 - root - INFO - step: 25845 loss: 16.7002 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5770 global_avg_mtp_loss: 14.1232 +[titan] 2025-07-09 19:17:23,344 - root - INFO - lr: 2.6017e-04 gnorm: 0.80 [ 5:42:25<16:22:29] +[titan] 2025-07-09 19:17:26,458 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:17:27,250 - root - INFO - step: 25850 loss: 16.8116 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6151 global_avg_mtp_loss: 14.1965 +[titan] 2025-07-09 19:17:27,250 - root - INFO - lr: 2.6015e-04 gnorm: 0.81 [ 5:42:29<16:22:25] +[titan] 2025-07-09 19:17:31,191 - root - INFO - step: 25855 loss: 16.6628 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.0747 +[titan] 2025-07-09 19:17:31,191 - root - INFO - lr: 2.6014e-04 gnorm: 0.77 [ 5:42:33<16:22:21] +[titan] 2025-07-09 19:17:35,123 - root - INFO - step: 25860 loss: 16.5781 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5771 global_avg_mtp_loss: 14.0011 +[titan] 2025-07-09 19:17:35,123 - root - INFO - lr: 2.6012e-04 gnorm: 0.80 [ 5:42:37<16:22:17] +[titan] 2025-07-09 19:17:39,019 - root - INFO - step: 25865 loss: 16.7170 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.1352 +[titan] 2025-07-09 19:17:39,020 - root - INFO - lr: 2.6011e-04 gnorm: 0.75 [ 5:42:41<16:22:13] +[titan] 2025-07-09 19:17:42,953 - root - INFO - step: 25870 loss: 17.0362 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6532 global_avg_mtp_loss: 14.3830 +[titan] 2025-07-09 19:17:42,954 - root - INFO - lr: 2.6009e-04 gnorm: 0.81 [ 5:42:45<16:22:09] +[titan] 2025-07-09 19:17:46,869 - root - INFO - step: 25875 loss: 16.6600 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5812 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-09 19:17:46,869 - root - INFO - lr: 2.6008e-04 gnorm: 0.78 [ 5:42:49<16:22:05] +[titan] 2025-07-09 19:17:50,789 - root - INFO - step: 25880 loss: 16.7817 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.1753 +[titan] 2025-07-09 19:17:50,790 - root - INFO - lr: 2.6006e-04 gnorm: 0.76 [ 5:42:53<16:22:01] +[titan] 2025-07-09 19:17:54,692 - root - INFO - step: 25885 loss: 16.6518 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.0723 +[titan] 2025-07-09 19:17:54,692 - root - INFO - lr: 2.6005e-04 gnorm: 0.85 [ 5:42:56<16:21:56] +[titan] 2025-07-09 19:17:58,603 - root - INFO - step: 25890 loss: 16.9244 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6368 global_avg_mtp_loss: 14.2875 +[titan] 2025-07-09 19:17:58,604 - root - INFO - lr: 2.6003e-04 gnorm: 0.74 [ 5:43:00<16:21:52] +[titan] 2025-07-09 19:18:02,519 - root - INFO - step: 25895 loss: 16.8800 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6275 global_avg_mtp_loss: 14.2525 +[titan] 2025-07-09 19:18:02,519 - root - INFO - lr: 2.6002e-04 gnorm: 0.80 [ 5:43:04<16:21:48] +[titan] 2025-07-09 19:18:05,653 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:18:06,441 - root - INFO - step: 25900 loss: 16.9305 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6391 global_avg_mtp_loss: 14.2914 +[titan] 2025-07-09 19:18:06,441 - root - INFO - lr: 2.6000e-04 gnorm: 0.75 [ 5:43:08<16:21:44] +[titan] 2025-07-09 19:18:10,330 - root - INFO - step: 25905 loss: 17.0878 memory: 44.58GiB(31.99%) tps: 84,261 tflops: 290.80 mfu: 29.40% global_avg_ntp_loss: 2.6526 global_avg_mtp_loss: 14.4352 +[titan] 2025-07-09 19:18:10,330 - root - INFO - lr: 2.5999e-04 gnorm: 0.79 [ 5:43:12<16:21:40] +[titan] 2025-07-09 19:18:14,244 - root - INFO - step: 25910 loss: 17.1030 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6849 global_avg_mtp_loss: 14.4181 +[titan] 2025-07-09 19:18:14,245 - root - INFO - lr: 2.5997e-04 gnorm: 0.80 [ 5:43:16<16:21:36] +[titan] 2025-07-09 19:18:18,152 - root - INFO - step: 25915 loss: 16.7963 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1998 +[titan] 2025-07-09 19:18:18,152 - root - INFO - lr: 2.5996e-04 gnorm: 0.81 [ 5:43:20<16:21:31] +[titan] 2025-07-09 19:18:22,070 - root - INFO - step: 25920 loss: 16.7754 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.1570 +[titan] 2025-07-09 19:18:22,070 - root - INFO - lr: 2.5994e-04 gnorm: 0.77 [ 5:43:24<16:21:27] +[titan] 2025-07-09 19:18:25,967 - root - INFO - step: 25925 loss: 17.0132 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6455 global_avg_mtp_loss: 14.3677 +[titan] 2025-07-09 19:18:25,968 - root - INFO - lr: 2.5993e-04 gnorm: 0.79 [ 5:43:28<16:21:23] +[titan] 2025-07-09 19:18:29,862 - root - INFO - step: 25930 loss: 17.0514 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6505 global_avg_mtp_loss: 14.4010 +[titan] 2025-07-09 19:18:29,862 - root - INFO - lr: 2.5991e-04 gnorm: 0.80 [ 5:43:32<16:21:19] +[titan] 2025-07-09 19:18:33,786 - root - INFO - step: 25935 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6297 global_avg_mtp_loss: 14.3104 +[titan] 2025-07-09 19:18:33,786 - root - INFO - lr: 2.5990e-04 gnorm: 0.78 [ 5:43:36<16:21:15] +[titan] 2025-07-09 19:18:37,733 - root - INFO - step: 25940 loss: 16.6072 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0363 +[titan] 2025-07-09 19:18:37,734 - root - INFO - lr: 2.5988e-04 gnorm: 0.78 [ 5:43:40<16:21:11] +[titan] 2025-07-09 19:18:41,634 - root - INFO - step: 25945 loss: 16.8308 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.2200 +[titan] 2025-07-09 19:18:41,634 - root - INFO - lr: 2.5986e-04 gnorm: 0.81 [ 5:43:43<16:21:06] +[titan] 2025-07-09 19:18:44,787 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:18:45,570 - root - INFO - step: 25950 loss: 17.0240 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6630 global_avg_mtp_loss: 14.3610 +[titan] 2025-07-09 19:18:45,570 - root - INFO - lr: 2.5985e-04 gnorm: 0.80 [ 5:43:47<16:21:02] +[titan] 2025-07-09 19:18:49,477 - root - INFO - step: 25955 loss: 16.8223 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.2193 +[titan] 2025-07-09 19:18:49,477 - root - INFO - lr: 2.5983e-04 gnorm: 0.79 [ 5:43:51<16:20:58] +[titan] 2025-07-09 19:18:53,373 - root - INFO - step: 25960 loss: 16.8930 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6214 global_avg_mtp_loss: 14.2716 +[titan] 2025-07-09 19:18:53,374 - root - INFO - lr: 2.5982e-04 gnorm: 0.78 [ 5:43:55<16:20:54] +[titan] 2025-07-09 19:18:57,306 - root - INFO - step: 25965 loss: 16.8991 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.2778 +[titan] 2025-07-09 19:18:57,306 - root - INFO - lr: 2.5980e-04 gnorm: 0.76 [ 5:43:59<16:20:50] +[titan] 2025-07-09 19:19:01,218 - root - INFO - step: 25970 loss: 16.7607 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1633 +[titan] 2025-07-09 19:19:01,218 - root - INFO - lr: 2.5979e-04 gnorm: 0.75 [ 5:44:03<16:20:46] +[titan] 2025-07-09 19:19:05,119 - root - INFO - step: 25975 loss: 16.7370 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.1472 +[titan] 2025-07-09 19:19:05,119 - root - INFO - lr: 2.5977e-04 gnorm: 0.76 [ 5:44:07<16:20:42] +[titan] 2025-07-09 19:19:09,060 - root - INFO - step: 25980 loss: 16.6925 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.0985 +[titan] 2025-07-09 19:19:09,060 - root - INFO - lr: 2.5976e-04 gnorm: 0.79 [ 5:44:11<16:20:38] +[titan] 2025-07-09 19:19:12,983 - root - INFO - step: 25985 loss: 16.9322 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2987 +[titan] 2025-07-09 19:19:12,983 - root - INFO - lr: 2.5974e-04 gnorm: 0.83 [ 5:44:15<16:20:33] +[titan] 2025-07-09 19:19:16,895 - root - INFO - step: 25990 loss: 16.8447 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6265 global_avg_mtp_loss: 14.2182 +[titan] 2025-07-09 19:19:16,895 - root - INFO - lr: 2.5973e-04 gnorm: 0.77 [ 5:44:19<16:20:29] +[titan] 2025-07-09 19:19:20,842 - root - INFO - step: 25995 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0414 +[titan] 2025-07-09 19:19:20,842 - root - INFO - lr: 2.5971e-04 gnorm: 0.82 [ 5:44:23<16:20:25] +[titan] 2025-07-09 19:19:23,978 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:19:24,772 - root - INFO - step: 26000 loss: 16.8482 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6217 global_avg_mtp_loss: 14.2264 +[titan] 2025-07-09 19:19:24,772 - root - INFO - lr: 2.5970e-04 gnorm: 0.83 [ 5:44:27<16:20:21] +[titan] 2025-07-09 19:19:28,671 - root - INFO - step: 26005 loss: 16.8112 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.2002 +[titan] 2025-07-09 19:19:28,671 - root - INFO - lr: 2.5968e-04 gnorm: 0.77 [ 5:44:30<16:20:17] +[titan] 2025-07-09 19:19:32,589 - root - INFO - step: 26010 loss: 16.7011 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.1017 +[titan] 2025-07-09 19:19:32,589 - root - INFO - lr: 2.5967e-04 gnorm: 0.88 [ 5:44:34<16:20:13] +[titan] 2025-07-09 19:19:36,517 - root - INFO - step: 26015 loss: 16.8343 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2144 +[titan] 2025-07-09 19:19:36,517 - root - INFO - lr: 2.5965e-04 gnorm: 0.78 [ 5:44:38<16:20:09] +[titan] 2025-07-09 19:19:40,442 - root - INFO - step: 26020 loss: 16.9893 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6390 global_avg_mtp_loss: 14.3503 +[titan] 2025-07-09 19:19:40,442 - root - INFO - lr: 2.5964e-04 gnorm: 0.87 [ 5:44:42<16:20:05] +[titan] 2025-07-09 19:19:44,335 - root - INFO - step: 26025 loss: 16.7243 memory: 44.58GiB(31.99%) tps: 84,176 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1389 +[titan] 2025-07-09 19:19:44,336 - root - INFO - lr: 2.5962e-04 gnorm: 0.78 [ 5:44:46<16:20:00] +[titan] 2025-07-09 19:19:48,234 - root - INFO - step: 26030 loss: 16.8782 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.6607 global_avg_mtp_loss: 14.2175 +[titan] 2025-07-09 19:19:48,234 - root - INFO - lr: 2.5961e-04 gnorm: 2.05 [ 5:44:50<16:19:56] +[titan] 2025-07-09 19:19:52,141 - root - INFO - step: 26035 loss: 16.7417 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5968 global_avg_mtp_loss: 14.1449 +[titan] 2025-07-09 19:19:52,141 - root - INFO - lr: 2.5959e-04 gnorm: 0.85 [ 5:44:54<16:19:52] +[titan] 2025-07-09 19:19:56,043 - root - INFO - step: 26040 loss: 16.7345 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1361 +[titan] 2025-07-09 19:19:56,043 - root - INFO - lr: 2.5957e-04 gnorm: 0.79 [ 5:44:58<16:19:48] +[titan] 2025-07-09 19:19:59,943 - root - INFO - step: 26045 loss: 16.7202 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1282 +[titan] 2025-07-09 19:19:59,943 - root - INFO - lr: 2.5956e-04 gnorm: 0.86 [ 5:45:02<16:19:44] +[titan] 2025-07-09 19:20:03,064 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:20:03,855 - root - INFO - step: 26050 loss: 16.5976 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.0188 +[titan] 2025-07-09 19:20:03,856 - root - INFO - lr: 2.5954e-04 gnorm: 0.76 [ 5:45:06<16:19:39] +[titan] 2025-07-09 19:20:07,744 - root - INFO - step: 26055 loss: 16.8783 memory: 44.58GiB(31.99%) tps: 84,278 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2576 +[titan] 2025-07-09 19:20:07,744 - root - INFO - lr: 2.5953e-04 gnorm: 0.72 [ 5:45:10<16:19:35] +[titan] 2025-07-09 19:20:11,671 - root - INFO - step: 26060 loss: 16.5964 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0288 +[titan] 2025-07-09 19:20:11,671 - root - INFO - lr: 2.5951e-04 gnorm: 0.72 [ 5:45:13<16:19:31] +[titan] 2025-07-09 19:20:15,599 - root - INFO - step: 26065 loss: 16.9334 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6376 global_avg_mtp_loss: 14.2957 +[titan] 2025-07-09 19:20:15,599 - root - INFO - lr: 2.5950e-04 gnorm: 0.73 [ 5:45:17<16:19:27] +[titan] 2025-07-09 19:20:19,519 - root - INFO - step: 26070 loss: 16.7347 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1445 +[titan] 2025-07-09 19:20:19,519 - root - INFO - lr: 2.5948e-04 gnorm: 0.74 [ 5:45:21<16:19:23] +[titan] 2025-07-09 19:20:23,417 - root - INFO - step: 26075 loss: 16.7536 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.1519 +[titan] 2025-07-09 19:20:23,418 - root - INFO - lr: 2.5947e-04 gnorm: 0.84 [ 5:45:25<16:19:19] +[titan] 2025-07-09 19:20:27,331 - root - INFO - step: 26080 loss: 16.7807 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5986 global_avg_mtp_loss: 14.1821 +[titan] 2025-07-09 19:20:27,331 - root - INFO - lr: 2.5945e-04 gnorm: 0.75 [ 5:45:29<16:19:15] +[titan] 2025-07-09 19:20:31,265 - root - INFO - step: 26085 loss: 16.7266 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1303 +[titan] 2025-07-09 19:20:31,266 - root - INFO - lr: 2.5944e-04 gnorm: 0.83 [ 5:45:33<16:19:10] +[titan] 2025-07-09 19:20:35,231 - root - INFO - step: 26090 loss: 16.6480 memory: 44.58GiB(31.99%) tps: 82,645 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0770 +[titan] 2025-07-09 19:20:35,231 - root - INFO - lr: 2.5942e-04 gnorm: 0.78 [ 5:45:37<16:19:06] +[titan] 2025-07-09 19:20:39,189 - root - INFO - step: 26095 loss: 16.9248 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.6398 global_avg_mtp_loss: 14.2850 +[titan] 2025-07-09 19:20:39,189 - root - INFO - lr: 2.5941e-04 gnorm: 0.77 [ 5:45:41<16:19:02] +[titan] 2025-07-09 19:20:42,327 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:20:43,111 - root - INFO - step: 26100 loss: 16.9972 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6477 global_avg_mtp_loss: 14.3495 +[titan] 2025-07-09 19:20:43,112 - root - INFO - lr: 2.5939e-04 gnorm: 0.74 [ 5:45:45<16:18:58] +[titan] 2025-07-09 19:20:47,020 - root - INFO - step: 26105 loss: 17.0669 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6534 global_avg_mtp_loss: 14.4134 +[titan] 2025-07-09 19:20:47,020 - root - INFO - lr: 2.5938e-04 gnorm: 0.76 [ 5:45:49<16:18:54] +[titan] 2025-07-09 19:20:51,006 - root - INFO - step: 26110 loss: 17.0629 memory: 44.58GiB(31.99%) tps: 82,215 tflops: 283.74 mfu: 28.69% global_avg_ntp_loss: 2.6595 global_avg_mtp_loss: 14.4034 +[titan] 2025-07-09 19:20:51,006 - root - INFO - lr: 2.5936e-04 gnorm: 0.76 [ 5:45:53<16:18:50] +[titan] 2025-07-09 19:20:52,717 - root - INFO - Dumping profiler traces at step 26112 +[titan] 2025-07-09 19:20:52,748 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:20:55,120 - root - INFO - step: 26115 loss: 17.0764 memory: 44.58GiB(31.99%) tps: 79,656 tflops: 274.91 mfu: 27.80% global_avg_ntp_loss: 2.6736 global_avg_mtp_loss: 14.4028 +[titan] 2025-07-09 19:20:55,120 - root - INFO - lr: 2.5935e-04 gnorm: 0.78 [ 5:45:57<16:18:47] +[titan] 2025-07-09 19:20:59,041 - root - INFO - step: 26120 loss: 16.9932 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6477 global_avg_mtp_loss: 14.3455 +[titan] 2025-07-09 19:20:59,041 - root - INFO - lr: 2.5933e-04 gnorm: 0.75 [ 5:46:01<16:18:43] +[titan] 2025-07-09 19:21:02,946 - root - INFO - step: 26125 loss: 16.8762 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6212 global_avg_mtp_loss: 14.2550 +[titan] 2025-07-09 19:21:02,947 - root - INFO - lr: 2.5931e-04 gnorm: 0.74 [ 5:46:05<16:18:38] +[titan] 2025-07-09 19:21:06,884 - root - INFO - step: 26130 loss: 16.9107 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2883 +[titan] 2025-07-09 19:21:06,884 - root - INFO - lr: 2.5930e-04 gnorm: 0.79 [ 5:46:09<16:18:34] +[titan] 2025-07-09 19:21:10,785 - root - INFO - step: 26135 loss: 16.8142 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2056 +[titan] 2025-07-09 19:21:10,785 - root - INFO - lr: 2.5928e-04 gnorm: 0.79 [ 5:46:13<16:18:30] +[titan] 2025-07-09 19:21:14,718 - root - INFO - step: 26140 loss: 16.9094 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2854 +[titan] 2025-07-09 19:21:14,718 - root - INFO - lr: 2.5927e-04 gnorm: 0.73 [ 5:46:16<16:18:26] +[titan] 2025-07-09 19:21:18,687 - root - INFO - step: 26145 loss: 17.0755 memory: 44.58GiB(31.99%) tps: 82,555 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.6627 global_avg_mtp_loss: 14.4128 +[titan] 2025-07-09 19:21:18,688 - root - INFO - lr: 2.5925e-04 gnorm: 0.80 [ 5:46:20<16:18:22] +[titan] 2025-07-09 19:21:21,837 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:21:22,628 - root - INFO - step: 26150 loss: 16.7840 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.1725 +[titan] 2025-07-09 19:21:22,628 - root - INFO - lr: 2.5924e-04 gnorm: 0.80 [ 5:46:24<16:18:18] +[titan] 2025-07-09 19:21:26,570 - root - INFO - step: 26155 loss: 16.8090 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.1977 +[titan] 2025-07-09 19:21:26,570 - root - INFO - lr: 2.5922e-04 gnorm: 0.78 [ 5:46:28<16:18:14] +[titan] 2025-07-09 19:21:30,496 - root - INFO - step: 26160 loss: 16.7547 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5970 global_avg_mtp_loss: 14.1577 +[titan] 2025-07-09 19:21:30,497 - root - INFO - lr: 2.5921e-04 gnorm: 0.79 [ 5:46:32<16:18:10] +[titan] 2025-07-09 19:21:34,395 - root - INFO - step: 26165 loss: 16.5017 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9574 +[titan] 2025-07-09 19:21:34,396 - root - INFO - lr: 2.5919e-04 gnorm: 0.81 [ 5:46:36<16:18:06] +[titan] 2025-07-09 19:21:38,354 - root - INFO - step: 26170 loss: 16.4861 memory: 44.58GiB(31.99%) tps: 82,773 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 13.9269 +[titan] 2025-07-09 19:21:38,355 - root - INFO - lr: 2.5918e-04 gnorm: 0.74 [ 5:46:40<16:18:02] +[titan] 2025-07-09 19:21:42,261 - root - INFO - step: 26175 loss: 16.8116 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.1944 +[titan] 2025-07-09 19:21:42,261 - root - INFO - lr: 2.5916e-04 gnorm: 0.76 [ 5:46:44<16:17:57] +[titan] 2025-07-09 19:21:46,192 - root - INFO - step: 26180 loss: 16.7283 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5997 global_avg_mtp_loss: 14.1287 +[titan] 2025-07-09 19:21:46,192 - root - INFO - lr: 2.5915e-04 gnorm: 0.73 [ 5:46:48<16:17:53] +[titan] 2025-07-09 19:21:50,103 - root - INFO - step: 26185 loss: 16.9971 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6448 global_avg_mtp_loss: 14.3522 +[titan] 2025-07-09 19:21:50,103 - root - INFO - lr: 2.5913e-04 gnorm: 0.85 [ 5:46:52<16:17:49] +[titan] 2025-07-09 19:21:54,022 - root - INFO - step: 26190 loss: 16.8493 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.2294 +[titan] 2025-07-09 19:21:54,022 - root - INFO - lr: 2.5911e-04 gnorm: 0.78 [ 5:46:56<16:17:45] +[titan] 2025-07-09 19:21:57,911 - root - INFO - step: 26195 loss: 16.7457 memory: 44.58GiB(31.99%) tps: 84,267 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.6191 global_avg_mtp_loss: 14.1266 +[titan] 2025-07-09 19:21:57,911 - root - INFO - lr: 2.5910e-04 gnorm: 0.89 [ 5:47:00<16:17:41] +[titan] 2025-07-09 19:22:01,034 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:22:01,820 - root - INFO - step: 26200 loss: 16.7411 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6203 global_avg_mtp_loss: 14.1208 +[titan] 2025-07-09 19:22:01,820 - root - INFO - lr: 2.5908e-04 gnorm: 0.79 [ 5:47:04<16:17:37] +[titan] 2025-07-09 19:22:05,729 - root - INFO - step: 26205 loss: 16.6034 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.0194 +[titan] 2025-07-09 19:22:05,729 - root - INFO - lr: 2.5907e-04 gnorm: 0.92 [ 5:47:07<16:17:32] +[titan] 2025-07-09 19:22:09,664 - root - INFO - step: 26210 loss: 16.8066 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.2058 +[titan] 2025-07-09 19:22:09,665 - root - INFO - lr: 2.5905e-04 gnorm: 0.81 [ 5:47:11<16:17:28] +[titan] 2025-07-09 19:22:13,582 - root - INFO - step: 26215 loss: 16.7407 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5962 global_avg_mtp_loss: 14.1445 +[titan] 2025-07-09 19:22:13,582 - root - INFO - lr: 2.5904e-04 gnorm: 0.80 [ 5:47:15<16:17:24] +[titan] 2025-07-09 19:22:17,505 - root - INFO - step: 26220 loss: 16.9734 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6379 global_avg_mtp_loss: 14.3355 +[titan] 2025-07-09 19:22:17,505 - root - INFO - lr: 2.5902e-04 gnorm: 0.86 [ 5:47:19<16:17:20] +[titan] 2025-07-09 19:22:21,416 - root - INFO - step: 26225 loss: 16.8928 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.2622 +[titan] 2025-07-09 19:22:21,416 - root - INFO - lr: 2.5901e-04 gnorm: 0.79 [ 5:47:23<16:17:16] +[titan] 2025-07-09 19:22:25,351 - root - INFO - step: 26230 loss: 17.0291 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.6571 global_avg_mtp_loss: 14.3720 +[titan] 2025-07-09 19:22:25,351 - root - INFO - lr: 2.5899e-04 gnorm: 0.81 [ 5:47:27<16:17:12] +[titan] 2025-07-09 19:22:29,311 - root - INFO - step: 26235 loss: 16.8927 memory: 44.58GiB(31.99%) tps: 82,746 tflops: 285.57 mfu: 28.87% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.2642 +[titan] 2025-07-09 19:22:29,312 - root - INFO - lr: 2.5898e-04 gnorm: 0.81 [ 5:47:31<16:17:08] +[titan] 2025-07-09 19:22:33,215 - root - INFO - step: 26240 loss: 16.8460 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.2357 +[titan] 2025-07-09 19:22:33,216 - root - INFO - lr: 2.5896e-04 gnorm: 0.75 [ 5:47:35<16:17:04] +[titan] 2025-07-09 19:22:37,169 - root - INFO - step: 26245 loss: 16.8701 memory: 44.58GiB(31.99%) tps: 82,898 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2564 +[titan] 2025-07-09 19:22:37,169 - root - INFO - lr: 2.5895e-04 gnorm: 0.77 [ 5:47:39<16:17:00] +[titan] 2025-07-09 19:22:40,280 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:22:41,067 - root - INFO - step: 26250 loss: 17.0009 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.12 mfu: 29.34% global_avg_ntp_loss: 2.6495 global_avg_mtp_loss: 14.3514 +[titan] 2025-07-09 19:22:41,067 - root - INFO - lr: 2.5893e-04 gnorm: 0.73 [ 5:47:43<16:16:55] +[titan] 2025-07-09 19:22:44,964 - root - INFO - step: 26255 loss: 16.9930 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6382 global_avg_mtp_loss: 14.3548 +[titan] 2025-07-09 19:22:44,964 - root - INFO - lr: 2.5892e-04 gnorm: 0.74 [ 5:47:47<16:16:51] +[titan] 2025-07-09 19:22:48,879 - root - INFO - step: 26260 loss: 16.9346 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.6289 global_avg_mtp_loss: 14.3057 +[titan] 2025-07-09 19:22:48,879 - root - INFO - lr: 2.5890e-04 gnorm: 0.83 [ 5:47:51<16:16:47] +[titan] 2025-07-09 19:22:52,783 - root - INFO - step: 26265 loss: 16.8372 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.2307 +[titan] 2025-07-09 19:22:52,783 - root - INFO - lr: 2.5888e-04 gnorm: 0.75 [ 5:47:55<16:16:43] +[titan] 2025-07-09 19:22:56,709 - root - INFO - step: 26270 loss: 16.7903 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.1773 +[titan] 2025-07-09 19:22:56,709 - root - INFO - lr: 2.5887e-04 gnorm: 0.76 [ 5:47:58<16:16:39] +[titan] 2025-07-09 19:23:00,627 - root - INFO - step: 26275 loss: 16.6699 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.0897 +[titan] 2025-07-09 19:23:00,628 - root - INFO - lr: 2.5885e-04 gnorm: 0.81 [ 5:48:02<16:16:35] +[titan] 2025-07-09 19:23:04,526 - root - INFO - step: 26280 loss: 16.6504 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0794 +[titan] 2025-07-09 19:23:04,526 - root - INFO - lr: 2.5884e-04 gnorm: 0.73 [ 5:48:06<16:16:31] +[titan] 2025-07-09 19:23:08,453 - root - INFO - step: 26285 loss: 16.6650 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5928 global_avg_mtp_loss: 14.0723 +[titan] 2025-07-09 19:23:08,453 - root - INFO - lr: 2.5882e-04 gnorm: 0.83 [ 5:48:10<16:16:26] +[titan] 2025-07-09 19:23:12,402 - root - INFO - step: 26290 loss: 16.6759 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.0962 +[titan] 2025-07-09 19:23:12,402 - root - INFO - lr: 2.5881e-04 gnorm: 0.79 [ 5:48:14<16:16:22] +[titan] 2025-07-09 19:23:16,306 - root - INFO - step: 26295 loss: 16.8610 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.2528 +[titan] 2025-07-09 19:23:16,306 - root - INFO - lr: 2.5879e-04 gnorm: 0.75 [ 5:48:18<16:16:18] +[titan] 2025-07-09 19:23:19,437 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:23:20,242 - root - INFO - step: 26300 loss: 16.9889 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6432 global_avg_mtp_loss: 14.3457 +[titan] 2025-07-09 19:23:20,242 - root - INFO - lr: 2.5878e-04 gnorm: 0.74 [ 5:48:22<16:16:14] +[titan] 2025-07-09 19:23:24,192 - root - INFO - step: 26305 loss: 16.5658 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5714 global_avg_mtp_loss: 13.9944 +[titan] 2025-07-09 19:23:24,192 - root - INFO - lr: 2.5876e-04 gnorm: 0.77 [ 5:48:26<16:16:10] +[titan] 2025-07-09 19:23:28,124 - root - INFO - step: 26310 loss: 16.8622 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6125 global_avg_mtp_loss: 14.2496 +[titan] 2025-07-09 19:23:28,124 - root - INFO - lr: 2.5875e-04 gnorm: 0.76 [ 5:48:30<16:16:06] +[titan] 2025-07-09 19:23:32,039 - root - INFO - step: 26315 loss: 16.8815 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6224 global_avg_mtp_loss: 14.2591 +[titan] 2025-07-09 19:23:32,040 - root - INFO - lr: 2.5873e-04 gnorm: 0.77 [ 5:48:34<16:16:02] +[titan] 2025-07-09 19:23:35,937 - root - INFO - step: 26320 loss: 16.7310 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6052 global_avg_mtp_loss: 14.1258 +[titan] 2025-07-09 19:23:35,937 - root - INFO - lr: 2.5871e-04 gnorm: 0.81 [ 5:48:38<16:15:58] +[titan] 2025-07-09 19:23:39,828 - root - INFO - step: 26325 loss: 16.5925 memory: 44.58GiB(31.99%) tps: 84,224 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0350 +[titan] 2025-07-09 19:23:39,828 - root - INFO - lr: 2.5870e-04 gnorm: 0.78 [ 5:48:42<16:15:53] +[titan] 2025-07-09 19:23:43,728 - root - INFO - step: 26330 loss: 16.8483 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2417 +[titan] 2025-07-09 19:23:43,729 - root - INFO - lr: 2.5868e-04 gnorm: 0.76 [ 5:48:45<16:15:49] +[titan] 2025-07-09 19:23:47,621 - root - INFO - step: 26335 loss: 16.6678 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0967 +[titan] 2025-07-09 19:23:47,621 - root - INFO - lr: 2.5867e-04 gnorm: 0.73 [ 5:48:49<16:15:45] +[titan] 2025-07-09 19:23:51,529 - root - INFO - step: 26340 loss: 16.9126 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6449 global_avg_mtp_loss: 14.2677 +[titan] 2025-07-09 19:23:51,529 - root - INFO - lr: 2.5865e-04 gnorm: 0.79 [ 5:48:53<16:15:41] +[titan] 2025-07-09 19:23:55,440 - root - INFO - step: 26345 loss: 16.5879 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-09 19:23:55,440 - root - INFO - lr: 2.5864e-04 gnorm: 0.73 [ 5:48:57<16:15:37] +[titan] 2025-07-09 19:23:58,552 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:23:59,340 - root - INFO - step: 26350 loss: 16.8257 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.2067 +[titan] 2025-07-09 19:23:59,340 - root - INFO - lr: 2.5862e-04 gnorm: 0.75 [ 5:49:01<16:15:33] +[titan] 2025-07-09 19:24:03,235 - root - INFO - step: 26355 loss: 17.1135 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.4471 +[titan] 2025-07-09 19:24:03,235 - root - INFO - lr: 2.5861e-04 gnorm: 0.78 [ 5:49:05<16:15:28] +[titan] 2025-07-09 19:24:07,158 - root - INFO - step: 26360 loss: 16.5120 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 13.9618 +[titan] 2025-07-09 19:24:07,159 - root - INFO - lr: 2.5859e-04 gnorm: 0.77 [ 5:49:09<16:15:24] +[titan] 2025-07-09 19:24:11,080 - root - INFO - step: 26365 loss: 16.6429 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0693 +[titan] 2025-07-09 19:24:11,081 - root - INFO - lr: 2.5858e-04 gnorm: 0.75 [ 5:49:13<16:15:20] +[titan] 2025-07-09 19:24:15,017 - root - INFO - step: 26370 loss: 16.6262 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.0400 +[titan] 2025-07-09 19:24:15,017 - root - INFO - lr: 2.5856e-04 gnorm: 0.73 [ 5:49:17<16:15:16] +[titan] 2025-07-09 19:24:18,922 - root - INFO - step: 26375 loss: 16.8445 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6171 global_avg_mtp_loss: 14.2274 +[titan] 2025-07-09 19:24:18,923 - root - INFO - lr: 2.5855e-04 gnorm: 0.74 [ 5:49:21<16:15:12] +[titan] 2025-07-09 19:24:22,832 - root - INFO - step: 26380 loss: 16.9154 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.2853 +[titan] 2025-07-09 19:24:22,832 - root - INFO - lr: 2.5853e-04 gnorm: 0.82 [ 5:49:25<16:15:08] +[titan] 2025-07-09 19:24:26,723 - root - INFO - step: 26385 loss: 16.8136 memory: 44.58GiB(31.99%) tps: 84,231 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.1928 +[titan] 2025-07-09 19:24:26,723 - root - INFO - lr: 2.5851e-04 gnorm: 0.78 [ 5:49:28<16:15:04] +[titan] 2025-07-09 19:24:30,658 - root - INFO - step: 26390 loss: 17.3120 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.7179 global_avg_mtp_loss: 14.5941 +[titan] 2025-07-09 19:24:30,658 - root - INFO - lr: 2.5850e-04 gnorm: 0.84 [ 5:49:32<16:15:00] +[titan] 2025-07-09 19:24:34,580 - root - INFO - step: 26395 loss: 16.8502 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2364 +[titan] 2025-07-09 19:24:34,580 - root - INFO - lr: 2.5848e-04 gnorm: 0.77 [ 5:49:36<16:14:55] +[titan] 2025-07-09 19:24:37,725 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:24:38,510 - root - INFO - step: 26400 loss: 16.9569 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6228 global_avg_mtp_loss: 14.3341 +[titan] 2025-07-09 19:24:38,510 - root - INFO - lr: 2.5847e-04 gnorm: 0.77 [ 5:49:40<16:14:51] +[titan] 2025-07-09 19:24:42,420 - root - INFO - step: 26405 loss: 16.9674 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6340 global_avg_mtp_loss: 14.3334 +[titan] 2025-07-09 19:24:42,421 - root - INFO - lr: 2.5845e-04 gnorm: 0.78 [ 5:49:44<16:14:47] +[titan] 2025-07-09 19:24:46,340 - root - INFO - step: 26410 loss: 16.8965 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.2759 +[titan] 2025-07-09 19:24:46,340 - root - INFO - lr: 2.5844e-04 gnorm: 0.70 [ 5:49:48<16:14:43] +[titan] 2025-07-09 19:24:50,258 - root - INFO - step: 26415 loss: 16.8605 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.2392 +[titan] 2025-07-09 19:24:50,258 - root - INFO - lr: 2.5842e-04 gnorm: 0.84 [ 5:49:52<16:14:39] +[titan] 2025-07-09 19:24:54,200 - root - INFO - step: 26420 loss: 16.8127 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5997 global_avg_mtp_loss: 14.2130 +[titan] 2025-07-09 19:24:54,201 - root - INFO - lr: 2.5841e-04 gnorm: 0.86 [ 5:49:56<16:14:35] +[titan] 2025-07-09 19:24:58,152 - root - INFO - step: 26425 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 82,934 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1085 +[titan] 2025-07-09 19:24:58,152 - root - INFO - lr: 2.5839e-04 gnorm: 0.76 [ 5:50:00<16:14:31] +[titan] 2025-07-09 19:25:02,045 - root - INFO - step: 26430 loss: 17.0915 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.6686 global_avg_mtp_loss: 14.4229 +[titan] 2025-07-09 19:25:02,045 - root - INFO - lr: 2.5838e-04 gnorm: 0.76 [ 5:50:04<16:14:27] +[titan] 2025-07-09 19:25:05,967 - root - INFO - step: 26435 loss: 17.1606 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6707 global_avg_mtp_loss: 14.4899 +[titan] 2025-07-09 19:25:05,967 - root - INFO - lr: 2.5836e-04 gnorm: 0.87 [ 5:50:08<16:14:22] +[titan] 2025-07-09 19:25:09,877 - root - INFO - step: 26440 loss: 16.8187 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6162 global_avg_mtp_loss: 14.2025 +[titan] 2025-07-09 19:25:09,877 - root - INFO - lr: 2.5834e-04 gnorm: 0.83 [ 5:50:12<16:14:18] +[titan] 2025-07-09 19:25:13,822 - root - INFO - step: 26445 loss: 16.5983 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.0207 +[titan] 2025-07-09 19:25:13,822 - root - INFO - lr: 2.5833e-04 gnorm: 0.82 [ 5:50:16<16:14:14] +[titan] 2025-07-09 19:25:16,978 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:25:17,773 - root - INFO - step: 26450 loss: 16.7340 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1422 +[titan] 2025-07-09 19:25:17,773 - root - INFO - lr: 2.5831e-04 gnorm: 0.78 [ 5:50:20<16:14:10] +[titan] 2025-07-09 19:25:21,698 - root - INFO - step: 26455 loss: 16.7558 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.1598 +[titan] 2025-07-09 19:25:21,698 - root - INFO - lr: 2.5830e-04 gnorm: 0.76 [ 5:50:23<16:14:06] +[titan] 2025-07-09 19:25:25,633 - root - INFO - step: 26460 loss: 16.7907 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6006 global_avg_mtp_loss: 14.1901 +[titan] 2025-07-09 19:25:25,633 - root - INFO - lr: 2.5828e-04 gnorm: 0.72 [ 5:50:27<16:14:02] +[titan] 2025-07-09 19:25:29,556 - root - INFO - step: 26465 loss: 16.8717 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.2523 +[titan] 2025-07-09 19:25:29,556 - root - INFO - lr: 2.5827e-04 gnorm: 0.74 [ 5:50:31<16:13:58] +[titan] 2025-07-09 19:25:33,479 - root - INFO - step: 26470 loss: 16.6828 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.1029 +[titan] 2025-07-09 19:25:33,479 - root - INFO - lr: 2.5825e-04 gnorm: 0.75 [ 5:50:35<16:13:54] +[titan] 2025-07-09 19:25:37,381 - root - INFO - step: 26475 loss: 16.7086 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5957 global_avg_mtp_loss: 14.1129 +[titan] 2025-07-09 19:25:37,382 - root - INFO - lr: 2.5824e-04 gnorm: 0.78 [ 5:50:39<16:13:50] +[titan] 2025-07-09 19:25:41,315 - root - INFO - step: 26480 loss: 16.8163 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.6002 global_avg_mtp_loss: 14.2161 +[titan] 2025-07-09 19:25:41,315 - root - INFO - lr: 2.5822e-04 gnorm: 0.81 [ 5:50:43<16:13:46] +[titan] 2025-07-09 19:25:45,219 - root - INFO - step: 26485 loss: 17.1071 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4410 +[titan] 2025-07-09 19:25:45,219 - root - INFO - lr: 2.5820e-04 gnorm: 0.81 [ 5:50:47<16:13:41] +[titan] 2025-07-09 19:25:49,127 - root - INFO - step: 26490 loss: 16.7165 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.1390 +[titan] 2025-07-09 19:25:49,127 - root - INFO - lr: 2.5819e-04 gnorm: 0.83 [ 5:50:51<16:13:37] +[titan] 2025-07-09 19:25:53,026 - root - INFO - step: 26495 loss: 17.0182 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3763 +[titan] 2025-07-09 19:25:53,027 - root - INFO - lr: 2.5817e-04 gnorm: 0.72 [ 5:50:55<16:13:33] +[titan] 2025-07-09 19:25:56,129 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:25:56,926 - root - INFO - step: 26500 loss: 16.8238 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.2085 +[titan] 2025-07-09 19:25:56,926 - root - INFO - lr: 2.5816e-04 gnorm: 0.75 [ 5:50:59<16:13:29] +[titan] 2025-07-09 19:26:00,874 - root - INFO - step: 26505 loss: 17.0475 memory: 44.58GiB(31.99%) tps: 82,997 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.6513 global_avg_mtp_loss: 14.3962 +[titan] 2025-07-09 19:26:00,875 - root - INFO - lr: 2.5814e-04 gnorm: 0.77 [ 5:51:03<16:13:25] +[titan] 2025-07-09 19:26:04,779 - root - INFO - step: 26510 loss: 16.8522 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.2325 +[titan] 2025-07-09 19:26:04,779 - root - INFO - lr: 2.5813e-04 gnorm: 0.79 [ 5:51:07<16:13:21] +[titan] 2025-07-09 19:26:08,679 - root - INFO - step: 26515 loss: 16.6111 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.0246 +[titan] 2025-07-09 19:26:08,679 - root - INFO - lr: 2.5811e-04 gnorm: 0.87 [ 5:51:10<16:13:16] +[titan] 2025-07-09 19:26:12,582 - root - INFO - step: 26520 loss: 16.7100 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1181 +[titan] 2025-07-09 19:26:12,582 - root - INFO - lr: 2.5810e-04 gnorm: 0.81 [ 5:51:14<16:13:12] +[titan] 2025-07-09 19:26:16,510 - root - INFO - step: 26525 loss: 16.7038 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5814 global_avg_mtp_loss: 14.1224 +[titan] 2025-07-09 19:26:16,510 - root - INFO - lr: 2.5808e-04 gnorm: 0.86 [ 5:51:18<16:13:08] +[titan] 2025-07-09 19:26:20,422 - root - INFO - step: 26530 loss: 16.7470 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.1404 +[titan] 2025-07-09 19:26:20,422 - root - INFO - lr: 2.5807e-04 gnorm: 0.78 [ 5:51:22<16:13:04] +[titan] 2025-07-09 19:26:24,331 - root - INFO - step: 26535 loss: 16.8041 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6006 global_avg_mtp_loss: 14.2035 +[titan] 2025-07-09 19:26:24,332 - root - INFO - lr: 2.5805e-04 gnorm: 0.73 [ 5:51:26<16:13:00] +[titan] 2025-07-09 19:26:28,241 - root - INFO - step: 26540 loss: 16.8378 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6386 global_avg_mtp_loss: 14.1992 +[titan] 2025-07-09 19:26:28,241 - root - INFO - lr: 2.5803e-04 gnorm: 0.81 [ 5:51:30<16:12:56] +[titan] 2025-07-09 19:26:32,176 - root - INFO - step: 26545 loss: 16.9669 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3255 +[titan] 2025-07-09 19:26:32,176 - root - INFO - lr: 2.5802e-04 gnorm: 0.80 [ 5:51:34<16:12:52] +[titan] 2025-07-09 19:26:35,296 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:26:36,085 - root - INFO - step: 26550 loss: 16.8899 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6277 global_avg_mtp_loss: 14.2622 +[titan] 2025-07-09 19:26:36,086 - root - INFO - lr: 2.5800e-04 gnorm: 0.89 [ 5:51:38<16:12:48] +[titan] 2025-07-09 19:26:40,010 - root - INFO - step: 26555 loss: 16.5957 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0343 +[titan] 2025-07-09 19:26:40,011 - root - INFO - lr: 2.5799e-04 gnorm: 0.73 [ 5:51:42<16:12:43] +[titan] 2025-07-09 19:26:43,936 - root - INFO - step: 26560 loss: 16.9487 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.3067 +[titan] 2025-07-09 19:26:43,936 - root - INFO - lr: 2.5797e-04 gnorm: 0.74 [ 5:51:46<16:12:39] +[titan] 2025-07-09 19:26:47,858 - root - INFO - step: 26565 loss: 16.8544 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6142 global_avg_mtp_loss: 14.2402 +[titan] 2025-07-09 19:26:47,858 - root - INFO - lr: 2.5796e-04 gnorm: 0.82 [ 5:51:50<16:12:35] +[titan] 2025-07-09 19:26:51,750 - root - INFO - step: 26570 loss: 16.6109 memory: 44.58GiB(31.99%) tps: 84,198 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5653 global_avg_mtp_loss: 14.0456 +[titan] 2025-07-09 19:26:51,751 - root - INFO - lr: 2.5794e-04 gnorm: 0.77 [ 5:51:53<16:12:31] +[titan] 2025-07-09 19:26:55,650 - root - INFO - step: 26575 loss: 17.1166 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.4565 +[titan] 2025-07-09 19:26:55,651 - root - INFO - lr: 2.5793e-04 gnorm: 0.75 [ 5:51:57<16:12:27] +[titan] 2025-07-09 19:26:59,540 - root - INFO - step: 26580 loss: 16.7090 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1201 +[titan] 2025-07-09 19:26:59,541 - root - INFO - lr: 2.5791e-04 gnorm: 0.76 [ 5:52:01<16:12:23] +[titan] 2025-07-09 19:27:03,443 - root - INFO - step: 26585 loss: 16.9492 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.3142 +[titan] 2025-07-09 19:27:03,443 - root - INFO - lr: 2.5789e-04 gnorm: 0.78 [ 5:52:05<16:12:18] +[titan] 2025-07-09 19:27:07,346 - root - INFO - step: 26590 loss: 16.5016 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 13.9432 +[titan] 2025-07-09 19:27:07,347 - root - INFO - lr: 2.5788e-04 gnorm: 0.88 [ 5:52:09<16:12:14] +[titan] 2025-07-09 19:27:11,234 - root - INFO - step: 26595 loss: 16.7540 memory: 44.58GiB(31.99%) tps: 84,297 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1671 +[titan] 2025-07-09 19:27:11,234 - root - INFO - lr: 2.5786e-04 gnorm: 0.81 [ 5:52:13<16:12:10] +[titan] 2025-07-09 19:27:14,377 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:27:15,160 - root - INFO - step: 26600 loss: 16.8062 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.1827 +[titan] 2025-07-09 19:27:15,160 - root - INFO - lr: 2.5785e-04 gnorm: 0.79 [ 5:52:17<16:12:06] +[titan] 2025-07-09 19:27:19,073 - root - INFO - step: 26605 loss: 16.9119 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6334 global_avg_mtp_loss: 14.2785 +[titan] 2025-07-09 19:27:19,073 - root - INFO - lr: 2.5783e-04 gnorm: 0.75 [ 5:52:21<16:12:02] +[titan] 2025-07-09 19:27:22,966 - root - INFO - step: 26610 loss: 16.6732 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.0861 +[titan] 2025-07-09 19:27:22,966 - root - INFO - lr: 2.5782e-04 gnorm: 0.69 [ 5:52:25<16:11:58] +[titan] 2025-07-09 19:27:26,868 - root - INFO - step: 26615 loss: 16.7357 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5935 global_avg_mtp_loss: 14.1422 +[titan] 2025-07-09 19:27:26,868 - root - INFO - lr: 2.5780e-04 gnorm: 0.77 [ 5:52:29<16:11:53] +[titan] 2025-07-09 19:27:30,795 - root - INFO - step: 26620 loss: 16.6968 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.1101 +[titan] 2025-07-09 19:27:30,796 - root - INFO - lr: 2.5779e-04 gnorm: 0.75 [ 5:52:33<16:11:49] +[titan] 2025-07-09 19:27:34,159 - root - INFO - Dumping profiler traces at step 26624 +[titan] 2025-07-09 19:27:34,193 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:27:34,989 - root - INFO - step: 26625 loss: 16.7865 memory: 44.58GiB(31.99%) tps: 78,144 tflops: 269.69 mfu: 27.27% global_avg_ntp_loss: 2.6096 global_avg_mtp_loss: 14.1769 +[titan] 2025-07-09 19:27:34,989 - root - INFO - lr: 2.5777e-04 gnorm: 0.84 [ 5:52:37<16:11:46] +[titan] 2025-07-09 19:27:38,882 - root - INFO - step: 26630 loss: 16.8918 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.2638 +[titan] 2025-07-09 19:27:38,882 - root - INFO - lr: 2.5775e-04 gnorm: 0.79 [ 5:52:41<16:11:42] +[titan] 2025-07-09 19:27:42,770 - root - INFO - step: 26635 loss: 16.8529 memory: 44.58GiB(31.99%) tps: 84,284 tflops: 290.88 mfu: 29.41% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.2420 +[titan] 2025-07-09 19:27:42,770 - root - INFO - lr: 2.5774e-04 gnorm: 0.72 [ 5:52:44<16:11:38] +[titan] 2025-07-09 19:27:46,672 - root - INFO - step: 26640 loss: 16.8914 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.2800 +[titan] 2025-07-09 19:27:46,673 - root - INFO - lr: 2.5772e-04 gnorm: 0.79 [ 5:52:48<16:11:33] +[titan] 2025-07-09 19:27:50,605 - root - INFO - step: 26645 loss: 16.9850 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6451 global_avg_mtp_loss: 14.3399 +[titan] 2025-07-09 19:27:50,605 - root - INFO - lr: 2.5771e-04 gnorm: 0.74 [ 5:52:52<16:11:29] +[titan] 2025-07-09 19:27:53,741 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:27:54,539 - root - INFO - step: 26650 loss: 16.8194 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.6293 global_avg_mtp_loss: 14.1902 +[titan] 2025-07-09 19:27:54,540 - root - INFO - lr: 2.5769e-04 gnorm: 0.77 [ 5:52:56<16:11:25] +[titan] 2025-07-09 19:27:58,471 - root - INFO - step: 26655 loss: 16.7188 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6094 global_avg_mtp_loss: 14.1093 +[titan] 2025-07-09 19:27:58,471 - root - INFO - lr: 2.5768e-04 gnorm: 0.76 [ 5:53:00<16:11:21] +[titan] 2025-07-09 19:28:02,407 - root - INFO - step: 26660 loss: 16.5466 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 13.9858 +[titan] 2025-07-09 19:28:02,407 - root - INFO - lr: 2.5766e-04 gnorm: 0.97 [ 5:53:04<16:11:17] +[titan] 2025-07-09 19:28:06,315 - root - INFO - step: 26665 loss: 16.7338 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1504 +[titan] 2025-07-09 19:28:06,316 - root - INFO - lr: 2.5765e-04 gnorm: 0.73 [ 5:53:08<16:11:13] +[titan] 2025-07-09 19:28:10,232 - root - INFO - step: 26670 loss: 16.9755 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6415 global_avg_mtp_loss: 14.3341 +[titan] 2025-07-09 19:28:10,232 - root - INFO - lr: 2.5763e-04 gnorm: 0.75 [ 5:53:12<16:11:09] +[titan] 2025-07-09 19:28:14,170 - root - INFO - step: 26675 loss: 16.7728 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.6012 global_avg_mtp_loss: 14.1716 +[titan] 2025-07-09 19:28:14,170 - root - INFO - lr: 2.5761e-04 gnorm: 0.77 [ 5:53:16<16:11:05] +[titan] 2025-07-09 19:28:18,077 - root - INFO - step: 26680 loss: 16.8630 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2494 +[titan] 2025-07-09 19:28:18,077 - root - INFO - lr: 2.5760e-04 gnorm: 0.81 [ 5:53:20<16:11:01] +[titan] 2025-07-09 19:28:21,998 - root - INFO - step: 26685 loss: 16.8058 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.1991 +[titan] 2025-07-09 19:28:21,999 - root - INFO - lr: 2.5758e-04 gnorm: 0.78 [ 5:53:24<16:10:56] +[titan] 2025-07-09 19:28:25,900 - root - INFO - step: 26690 loss: 16.7130 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5863 global_avg_mtp_loss: 14.1267 +[titan] 2025-07-09 19:28:25,900 - root - INFO - lr: 2.5757e-04 gnorm: 0.77 [ 5:53:28<16:10:52] +[titan] 2025-07-09 19:28:29,805 - root - INFO - step: 26695 loss: 16.8896 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.2681 +[titan] 2025-07-09 19:28:29,805 - root - INFO - lr: 2.5755e-04 gnorm: 0.75 [ 5:53:32<16:10:48] +[titan] 2025-07-09 19:28:32,919 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:28:33,710 - root - INFO - step: 26700 loss: 16.8747 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6368 global_avg_mtp_loss: 14.2379 +[titan] 2025-07-09 19:28:33,710 - root - INFO - lr: 2.5754e-04 gnorm: 0.79 [ 5:53:35<16:10:44] +[titan] 2025-07-09 19:28:37,615 - root - INFO - step: 26705 loss: 16.9077 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6220 global_avg_mtp_loss: 14.2858 +[titan] 2025-07-09 19:28:37,615 - root - INFO - lr: 2.5752e-04 gnorm: 0.75 [ 5:53:39<16:10:40] +[titan] 2025-07-09 19:28:41,512 - root - INFO - step: 26710 loss: 16.8568 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6228 global_avg_mtp_loss: 14.2340 +[titan] 2025-07-09 19:28:41,512 - root - INFO - lr: 2.5751e-04 gnorm: 0.74 [ 5:53:43<16:10:36] +[titan] 2025-07-09 19:28:45,460 - root - INFO - step: 26715 loss: 16.6002 memory: 44.58GiB(31.99%) tps: 82,991 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 19:28:45,461 - root - INFO - lr: 2.5749e-04 gnorm: 1.11 [ 5:53:47<16:10:32] +[titan] 2025-07-09 19:28:49,396 - root - INFO - step: 26720 loss: 16.8831 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.6270 global_avg_mtp_loss: 14.2562 +[titan] 2025-07-09 19:28:49,396 - root - INFO - lr: 2.5747e-04 gnorm: 0.78 [ 5:53:51<16:10:27] +[titan] 2025-07-09 19:28:53,329 - root - INFO - step: 26725 loss: 16.7197 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5954 global_avg_mtp_loss: 14.1243 +[titan] 2025-07-09 19:28:53,329 - root - INFO - lr: 2.5746e-04 gnorm: 0.73 [ 5:53:55<16:10:23] +[titan] 2025-07-09 19:28:57,211 - root - INFO - step: 26730 loss: 16.7839 memory: 44.58GiB(31.99%) tps: 84,428 tflops: 291.37 mfu: 29.46% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.1687 +[titan] 2025-07-09 19:28:57,211 - root - INFO - lr: 2.5744e-04 gnorm: 1.14 [ 5:53:59<16:10:19] +[titan] 2025-07-09 19:29:01,115 - root - INFO - step: 26735 loss: 17.1603 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6688 global_avg_mtp_loss: 14.4915 +[titan] 2025-07-09 19:29:01,115 - root - INFO - lr: 2.5743e-04 gnorm: 0.73 [ 5:54:03<16:10:15] +[titan] 2025-07-09 19:29:05,030 - root - INFO - step: 26740 loss: 16.7186 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1342 +[titan] 2025-07-09 19:29:05,031 - root - INFO - lr: 2.5741e-04 gnorm: 0.80 [ 5:54:07<16:10:11] +[titan] 2025-07-09 19:29:08,925 - root - INFO - step: 26745 loss: 17.0180 memory: 44.58GiB(31.99%) tps: 84,151 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.3688 +[titan] 2025-07-09 19:29:08,925 - root - INFO - lr: 2.5740e-04 gnorm: 0.76 [ 5:54:11<16:10:07] +[titan] 2025-07-09 19:29:12,038 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:29:12,860 - root - INFO - step: 26750 loss: 16.7826 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.6171 global_avg_mtp_loss: 14.1655 +[titan] 2025-07-09 19:29:12,860 - root - INFO - lr: 2.5738e-04 gnorm: 0.79 [ 5:54:15<16:10:03] +[titan] 2025-07-09 19:29:16,761 - root - INFO - step: 26755 loss: 16.8976 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6340 global_avg_mtp_loss: 14.2636 +[titan] 2025-07-09 19:29:16,762 - root - INFO - lr: 2.5736e-04 gnorm: 0.75 [ 5:54:18<16:09:58] +[titan] 2025-07-09 19:29:20,680 - root - INFO - step: 26760 loss: 16.9901 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6399 global_avg_mtp_loss: 14.3503 +[titan] 2025-07-09 19:29:20,680 - root - INFO - lr: 2.5735e-04 gnorm: 0.73 [ 5:54:22<16:09:54] +[titan] 2025-07-09 19:29:24,579 - root - INFO - step: 26765 loss: 16.9105 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6374 global_avg_mtp_loss: 14.2731 +[titan] 2025-07-09 19:29:24,580 - root - INFO - lr: 2.5733e-04 gnorm: 0.77 [ 5:54:26<16:09:50] +[titan] 2025-07-09 19:29:28,483 - root - INFO - step: 26770 loss: 17.0348 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6479 global_avg_mtp_loss: 14.3869 +[titan] 2025-07-09 19:29:28,483 - root - INFO - lr: 2.5732e-04 gnorm: 0.74 [ 5:54:30<16:09:46] +[titan] 2025-07-09 19:29:32,408 - root - INFO - step: 26775 loss: 16.9695 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.6426 global_avg_mtp_loss: 14.3269 +[titan] 2025-07-09 19:29:32,408 - root - INFO - lr: 2.5730e-04 gnorm: 0.77 [ 5:54:34<16:09:42] +[titan] 2025-07-09 19:29:36,303 - root - INFO - step: 26780 loss: 16.6078 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 14.0470 +[titan] 2025-07-09 19:29:36,303 - root - INFO - lr: 2.5729e-04 gnorm: 0.74 [ 5:54:38<16:09:38] +[titan] 2025-07-09 19:29:40,197 - root - INFO - step: 26785 loss: 16.4959 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 13.9476 +[titan] 2025-07-09 19:29:40,197 - root - INFO - lr: 2.5727e-04 gnorm: 0.78 [ 5:54:42<16:09:34] +[titan] 2025-07-09 19:29:44,099 - root - INFO - step: 26790 loss: 16.9081 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.2607 +[titan] 2025-07-09 19:29:44,100 - root - INFO - lr: 2.5726e-04 gnorm: 0.82 [ 5:54:46<16:09:29] +[titan] 2025-07-09 19:29:48,021 - root - INFO - step: 26795 loss: 16.8339 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.2033 +[titan] 2025-07-09 19:29:48,022 - root - INFO - lr: 2.5724e-04 gnorm: 0.80 [ 5:54:50<16:09:25] +[titan] 2025-07-09 19:29:51,167 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:29:51,970 - root - INFO - step: 26800 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.0965 +[titan] 2025-07-09 19:29:51,970 - root - INFO - lr: 2.5722e-04 gnorm: 0.79 [ 5:54:54<16:09:21] +[titan] 2025-07-09 19:29:55,906 - root - INFO - step: 26805 loss: 16.8210 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.2215 +[titan] 2025-07-09 19:29:55,906 - root - INFO - lr: 2.5721e-04 gnorm: 0.77 [ 5:54:58<16:09:17] +[titan] 2025-07-09 19:29:59,818 - root - INFO - step: 26810 loss: 16.7538 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5959 global_avg_mtp_loss: 14.1579 +[titan] 2025-07-09 19:29:59,818 - root - INFO - lr: 2.5719e-04 gnorm: 0.82 [ 5:55:02<16:09:13] +[titan] 2025-07-09 19:30:03,746 - root - INFO - step: 26815 loss: 16.8172 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.1991 +[titan] 2025-07-09 19:30:03,746 - root - INFO - lr: 2.5718e-04 gnorm: 0.90 [ 5:55:05<16:09:09] +[titan] 2025-07-09 19:30:07,711 - root - INFO - step: 26820 loss: 16.7720 memory: 44.58GiB(31.99%) tps: 82,642 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.6334 global_avg_mtp_loss: 14.1386 +[titan] 2025-07-09 19:30:07,712 - root - INFO - lr: 2.5716e-04 gnorm: 0.80 [ 5:55:09<16:09:05] +[titan] 2025-07-09 19:30:11,641 - root - INFO - step: 26825 loss: 16.6504 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.0716 +[titan] 2025-07-09 19:30:11,641 - root - INFO - lr: 2.5715e-04 gnorm: 0.75 [ 5:55:13<16:09:01] +[titan] 2025-07-09 19:30:15,594 - root - INFO - step: 26830 loss: 16.6810 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.1029 +[titan] 2025-07-09 19:30:15,594 - root - INFO - lr: 2.5713e-04 gnorm: 0.73 [ 5:55:17<16:08:57] +[titan] 2025-07-09 19:30:19,500 - root - INFO - step: 26835 loss: 16.8486 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6224 global_avg_mtp_loss: 14.2262 +[titan] 2025-07-09 19:30:19,500 - root - INFO - lr: 2.5711e-04 gnorm: 0.71 [ 5:55:21<16:08:53] +[titan] 2025-07-09 19:30:23,553 - root - INFO - step: 26840 loss: 16.6707 memory: 44.58GiB(31.99%) tps: 80,857 tflops: 279.05 mfu: 28.22% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.0972 +[titan] 2025-07-09 19:30:23,553 - root - INFO - lr: 2.5710e-04 gnorm: 0.83 [ 5:55:25<16:08:49] +[titan] 2025-07-09 19:30:27,460 - root - INFO - step: 26845 loss: 16.8844 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2524 +[titan] 2025-07-09 19:30:27,461 - root - INFO - lr: 2.5708e-04 gnorm: 0.79 [ 5:55:29<16:08:45] +[titan] 2025-07-09 19:30:30,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:30:31,397 - root - INFO - step: 26850 loss: 16.5792 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5568 global_avg_mtp_loss: 14.0224 +[titan] 2025-07-09 19:30:31,397 - root - INFO - lr: 2.5707e-04 gnorm: 0.75 [ 5:55:33<16:08:41] +[titan] 2025-07-09 19:30:35,315 - root - INFO - step: 26855 loss: 16.9849 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6378 global_avg_mtp_loss: 14.3471 +[titan] 2025-07-09 19:30:35,315 - root - INFO - lr: 2.5705e-04 gnorm: 0.75 [ 5:55:37<16:08:37] +[titan] 2025-07-09 19:30:39,266 - root - INFO - step: 26860 loss: 16.6951 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1062 +[titan] 2025-07-09 19:30:39,266 - root - INFO - lr: 2.5704e-04 gnorm: 0.84 [ 5:55:41<16:08:32] +[titan] 2025-07-09 19:30:43,177 - root - INFO - step: 26865 loss: 16.8596 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6190 global_avg_mtp_loss: 14.2406 +[titan] 2025-07-09 19:30:43,177 - root - INFO - lr: 2.5702e-04 gnorm: 0.83 [ 5:55:45<16:08:28] +[titan] 2025-07-09 19:30:47,081 - root - INFO - step: 26870 loss: 16.9976 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.6356 global_avg_mtp_loss: 14.3621 +[titan] 2025-07-09 19:30:47,081 - root - INFO - lr: 2.5700e-04 gnorm: 0.77 [ 5:55:49<16:08:24] +[titan] 2025-07-09 19:30:51,034 - root - INFO - step: 26875 loss: 16.7856 memory: 44.58GiB(31.99%) tps: 82,895 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1947 +[titan] 2025-07-09 19:30:51,034 - root - INFO - lr: 2.5699e-04 gnorm: 0.75 [ 5:55:53<16:08:20] +[titan] 2025-07-09 19:30:54,951 - root - INFO - step: 26880 loss: 16.7886 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.1805 +[titan] 2025-07-09 19:30:54,951 - root - INFO - lr: 2.5697e-04 gnorm: 0.77 [ 5:55:57<16:08:16] +[titan] 2025-07-09 19:30:58,835 - root - INFO - step: 26885 loss: 16.8032 memory: 44.58GiB(31.99%) tps: 84,367 tflops: 291.16 mfu: 29.44% global_avg_ntp_loss: 2.6055 global_avg_mtp_loss: 14.1977 +[titan] 2025-07-09 19:30:58,835 - root - INFO - lr: 2.5696e-04 gnorm: 0.73 [ 5:56:01<16:08:12] +[titan] 2025-07-09 19:31:02,742 - root - INFO - step: 26890 loss: 16.6954 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.0958 +[titan] 2025-07-09 19:31:02,742 - root - INFO - lr: 2.5694e-04 gnorm: 0.74 [ 5:56:04<16:08:08] +[titan] 2025-07-09 19:31:06,642 - root - INFO - step: 26895 loss: 16.5571 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 13.9898 +[titan] 2025-07-09 19:31:06,642 - root - INFO - lr: 2.5693e-04 gnorm: 0.82 [ 5:56:08<16:08:03] +[titan] 2025-07-09 19:31:09,749 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:31:10,538 - root - INFO - step: 26900 loss: 16.6241 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.0439 +[titan] 2025-07-09 19:31:10,538 - root - INFO - lr: 2.5691e-04 gnorm: 0.75 [ 5:56:12<16:07:59] +[titan] 2025-07-09 19:31:14,432 - root - INFO - step: 26905 loss: 17.0303 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.6535 global_avg_mtp_loss: 14.3768 +[titan] 2025-07-09 19:31:14,433 - root - INFO - lr: 2.5690e-04 gnorm: 0.71 [ 5:56:16<16:07:55] +[titan] 2025-07-09 19:31:18,334 - root - INFO - step: 26910 loss: 16.4479 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 13.8943 +[titan] 2025-07-09 19:31:18,334 - root - INFO - lr: 2.5688e-04 gnorm: 0.75 [ 5:56:20<16:07:51] +[titan] 2025-07-09 19:31:22,278 - root - INFO - step: 26915 loss: 16.8345 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.2367 +[titan] 2025-07-09 19:31:22,278 - root - INFO - lr: 2.5686e-04 gnorm: 0.79 [ 5:56:24<16:07:47] +[titan] 2025-07-09 19:31:26,224 - root - INFO - step: 26920 loss: 16.8028 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.6129 global_avg_mtp_loss: 14.1898 +[titan] 2025-07-09 19:31:26,224 - root - INFO - lr: 2.5685e-04 gnorm: 0.77 [ 5:56:28<16:07:43] +[titan] 2025-07-09 19:31:30,136 - root - INFO - step: 26925 loss: 16.9871 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6405 global_avg_mtp_loss: 14.3466 +[titan] 2025-07-09 19:31:30,136 - root - INFO - lr: 2.5683e-04 gnorm: 0.75 [ 5:56:32<16:07:39] +[titan] 2025-07-09 19:31:34,042 - root - INFO - step: 26930 loss: 16.9085 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6295 global_avg_mtp_loss: 14.2790 +[titan] 2025-07-09 19:31:34,042 - root - INFO - lr: 2.5682e-04 gnorm: 0.82 [ 5:56:36<16:07:35] +[titan] 2025-07-09 19:31:37,949 - root - INFO - step: 26935 loss: 16.8744 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.2613 +[titan] 2025-07-09 19:31:37,949 - root - INFO - lr: 2.5680e-04 gnorm: 0.77 [ 5:56:40<16:07:30] +[titan] 2025-07-09 19:31:41,864 - root - INFO - step: 26940 loss: 16.8725 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.2446 +[titan] 2025-07-09 19:31:41,864 - root - INFO - lr: 2.5679e-04 gnorm: 0.75 [ 5:56:44<16:07:26] +[titan] 2025-07-09 19:31:45,802 - root - INFO - step: 26945 loss: 16.6681 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.0833 +[titan] 2025-07-09 19:31:45,803 - root - INFO - lr: 2.5677e-04 gnorm: 0.80 [ 5:56:47<16:07:22] +[titan] 2025-07-09 19:31:48,931 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:31:49,723 - root - INFO - step: 26950 loss: 16.7952 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6031 global_avg_mtp_loss: 14.1921 +[titan] 2025-07-09 19:31:49,723 - root - INFO - lr: 2.5675e-04 gnorm: 0.81 [ 5:56:51<16:07:18] +[titan] 2025-07-09 19:31:53,635 - root - INFO - step: 26955 loss: 16.8046 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.1906 +[titan] 2025-07-09 19:31:53,636 - root - INFO - lr: 2.5674e-04 gnorm: 0.74 [ 5:56:55<16:07:14] +[titan] 2025-07-09 19:31:57,546 - root - INFO - step: 26960 loss: 16.7209 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6031 global_avg_mtp_loss: 14.1177 +[titan] 2025-07-09 19:31:57,546 - root - INFO - lr: 2.5672e-04 gnorm: 0.75 [ 5:56:59<16:07:10] +[titan] 2025-07-09 19:32:01,446 - root - INFO - step: 26965 loss: 16.7340 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6027 global_avg_mtp_loss: 14.1313 +[titan] 2025-07-09 19:32:01,446 - root - INFO - lr: 2.5671e-04 gnorm: 0.78 [ 5:57:03<16:07:06] +[titan] 2025-07-09 19:32:05,407 - root - INFO - step: 26970 loss: 16.7552 memory: 44.58GiB(31.99%) tps: 82,732 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.1536 +[titan] 2025-07-09 19:32:05,407 - root - INFO - lr: 2.5669e-04 gnorm: 0.85 [ 5:57:07<16:07:02] +[titan] 2025-07-09 19:32:09,344 - root - INFO - step: 26975 loss: 16.7395 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.1513 +[titan] 2025-07-09 19:32:09,344 - root - INFO - lr: 2.5668e-04 gnorm: 0.76 [ 5:57:11<16:06:58] +[titan] 2025-07-09 19:32:13,253 - root - INFO - step: 26980 loss: 16.9671 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6355 global_avg_mtp_loss: 14.3315 +[titan] 2025-07-09 19:32:13,253 - root - INFO - lr: 2.5666e-04 gnorm: 0.76 [ 5:57:15<16:06:53] +[titan] 2025-07-09 19:32:17,145 - root - INFO - step: 26985 loss: 16.7293 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.6239 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 19:32:17,145 - root - INFO - lr: 2.5664e-04 gnorm: 0.76 [ 5:57:19<16:06:49] +[titan] 2025-07-09 19:32:21,078 - root - INFO - step: 26990 loss: 16.6926 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1059 +[titan] 2025-07-09 19:32:21,079 - root - INFO - lr: 2.5663e-04 gnorm: 0.78 [ 5:57:23<16:06:45] +[titan] 2025-07-09 19:32:24,973 - root - INFO - step: 26995 loss: 16.8375 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.2190 +[titan] 2025-07-09 19:32:24,974 - root - INFO - lr: 2.5661e-04 gnorm: 0.79 [ 5:57:27<16:06:41] +[titan] 2025-07-09 19:32:28,088 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:32:28,880 - root - INFO - step: 27000 loss: 16.7853 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6163 global_avg_mtp_loss: 14.1690 +[titan] 2025-07-09 19:32:28,880 - root - INFO - lr: 2.5660e-04 gnorm: 0.86 [ 5:57:31<16:06:37] +[titan] 2025-07-09 19:32:32,808 - root - INFO - step: 27005 loss: 16.7563 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.1524 +[titan] 2025-07-09 19:32:32,808 - root - INFO - lr: 2.5658e-04 gnorm: 0.72 [ 5:57:34<16:06:33] +[titan] 2025-07-09 19:32:36,722 - root - INFO - step: 27010 loss: 16.7737 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5971 global_avg_mtp_loss: 14.1766 +[titan] 2025-07-09 19:32:36,722 - root - INFO - lr: 2.5656e-04 gnorm: 0.82 [ 5:57:38<16:06:29] +[titan] 2025-07-09 19:32:40,643 - root - INFO - step: 27015 loss: 16.9457 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.6266 global_avg_mtp_loss: 14.3191 +[titan] 2025-07-09 19:32:40,643 - root - INFO - lr: 2.5655e-04 gnorm: 0.82 [ 5:57:42<16:06:24] +[titan] 2025-07-09 19:32:44,551 - root - INFO - step: 27020 loss: 16.8828 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6342 global_avg_mtp_loss: 14.2486 +[titan] 2025-07-09 19:32:44,552 - root - INFO - lr: 2.5653e-04 gnorm: 0.73 [ 5:57:46<16:06:20] +[titan] 2025-07-09 19:32:48,486 - root - INFO - step: 27025 loss: 16.8919 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6147 global_avg_mtp_loss: 14.2772 +[titan] 2025-07-09 19:32:48,486 - root - INFO - lr: 2.5652e-04 gnorm: 0.74 [ 5:57:50<16:06:16] +[titan] 2025-07-09 19:32:52,410 - root - INFO - step: 27030 loss: 16.9926 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6478 global_avg_mtp_loss: 14.3447 +[titan] 2025-07-09 19:32:52,410 - root - INFO - lr: 2.5650e-04 gnorm: 0.75 [ 5:57:54<16:06:12] +[titan] 2025-07-09 19:32:56,315 - root - INFO - step: 27035 loss: 16.8747 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.2509 +[titan] 2025-07-09 19:32:56,316 - root - INFO - lr: 2.5649e-04 gnorm: 0.77 [ 5:57:58<16:06:08] +[titan] 2025-07-09 19:33:00,236 - root - INFO - step: 27040 loss: 16.3587 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.8361 +[titan] 2025-07-09 19:33:00,237 - root - INFO - lr: 2.5647e-04 gnorm: 0.84 [ 5:58:02<16:06:04] +[titan] 2025-07-09 19:33:04,140 - root - INFO - step: 27045 loss: 16.9766 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6316 global_avg_mtp_loss: 14.3451 +[titan] 2025-07-09 19:33:04,141 - root - INFO - lr: 2.5645e-04 gnorm: 0.78 [ 5:58:06<16:06:00] +[titan] 2025-07-09 19:33:07,269 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:33:08,054 - root - INFO - step: 27050 loss: 16.8565 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.2280 +[titan] 2025-07-09 19:33:08,055 - root - INFO - lr: 2.5644e-04 gnorm: 0.79 [ 5:58:10<16:05:56] +[titan] 2025-07-09 19:33:11,942 - root - INFO - step: 27055 loss: 16.5465 memory: 44.58GiB(31.99%) tps: 84,298 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 13.9907 +[titan] 2025-07-09 19:33:11,942 - root - INFO - lr: 2.5642e-04 gnorm: 0.91 [ 5:58:14<16:05:51] +[titan] 2025-07-09 19:33:15,848 - root - INFO - step: 27060 loss: 16.9186 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6347 global_avg_mtp_loss: 14.2839 +[titan] 2025-07-09 19:33:15,848 - root - INFO - lr: 2.5641e-04 gnorm: 0.72 [ 5:58:18<16:05:47] +[titan] 2025-07-09 19:33:19,774 - root - INFO - step: 27065 loss: 16.8693 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.6296 global_avg_mtp_loss: 14.2398 +[titan] 2025-07-09 19:33:19,775 - root - INFO - lr: 2.5639e-04 gnorm: 0.77 [ 5:58:21<16:05:43] +[titan] 2025-07-09 19:33:23,681 - root - INFO - step: 27070 loss: 16.4768 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 13.9271 +[titan] 2025-07-09 19:33:23,681 - root - INFO - lr: 2.5638e-04 gnorm: 0.75 [ 5:58:25<16:05:39] +[titan] 2025-07-09 19:33:27,626 - root - INFO - step: 27075 loss: 16.7682 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1698 +[titan] 2025-07-09 19:33:27,626 - root - INFO - lr: 2.5636e-04 gnorm: 0.74 [ 5:58:29<16:05:35] +[titan] 2025-07-09 19:33:31,539 - root - INFO - step: 27080 loss: 16.7645 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1680 +[titan] 2025-07-09 19:33:31,540 - root - INFO - lr: 2.5634e-04 gnorm: 0.80 [ 5:58:33<16:05:31] +[titan] 2025-07-09 19:33:35,449 - root - INFO - step: 27085 loss: 16.8101 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6057 global_avg_mtp_loss: 14.2044 +[titan] 2025-07-09 19:33:35,449 - root - INFO - lr: 2.5633e-04 gnorm: 0.79 [ 5:58:37<16:05:27] +[titan] 2025-07-09 19:33:39,348 - root - INFO - step: 27090 loss: 17.0326 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6567 global_avg_mtp_loss: 14.3758 +[titan] 2025-07-09 19:33:39,348 - root - INFO - lr: 2.5631e-04 gnorm: 0.83 [ 5:58:41<16:05:23] +[titan] 2025-07-09 19:33:43,262 - root - INFO - step: 27095 loss: 16.7199 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.1408 +[titan] 2025-07-09 19:33:43,262 - root - INFO - lr: 2.5630e-04 gnorm: 0.72 [ 5:58:45<16:05:18] +[titan] 2025-07-09 19:33:46,384 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:33:47,180 - root - INFO - step: 27100 loss: 17.0417 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6532 global_avg_mtp_loss: 14.3885 +[titan] 2025-07-09 19:33:47,180 - root - INFO - lr: 2.5628e-04 gnorm: 0.79 [ 5:58:49<16:05:14] +[titan] 2025-07-09 19:33:51,076 - root - INFO - step: 27105 loss: 16.8844 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.2352 +[titan] 2025-07-09 19:33:51,076 - root - INFO - lr: 2.5627e-04 gnorm: 0.81 [ 5:58:53<16:05:10] +[titan] 2025-07-09 19:33:54,989 - root - INFO - step: 27110 loss: 16.6829 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 19:33:54,989 - root - INFO - lr: 2.5625e-04 gnorm: 0.79 [ 5:58:57<16:05:06] +[titan] 2025-07-09 19:33:58,887 - root - INFO - step: 27115 loss: 16.7570 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.1538 +[titan] 2025-07-09 19:33:58,887 - root - INFO - lr: 2.5623e-04 gnorm: 0.80 [ 5:59:01<16:05:02] +[titan] 2025-07-09 19:34:02,834 - root - INFO - step: 27120 loss: 16.8433 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.6073 global_avg_mtp_loss: 14.2360 +[titan] 2025-07-09 19:34:02,834 - root - INFO - lr: 2.5622e-04 gnorm: 0.78 [ 5:59:05<16:04:58] +[titan] 2025-07-09 19:34:06,741 - root - INFO - step: 27125 loss: 16.9454 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.6492 global_avg_mtp_loss: 14.2961 +[titan] 2025-07-09 19:34:06,741 - root - INFO - lr: 2.5620e-04 gnorm: 0.81 [ 5:59:08<16:04:54] +[titan] 2025-07-09 19:34:10,650 - root - INFO - step: 27130 loss: 17.0282 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.3762 +[titan] 2025-07-09 19:34:10,650 - root - INFO - lr: 2.5619e-04 gnorm: 0.77 [ 5:59:12<16:04:49] +[titan] 2025-07-09 19:34:14,636 - root - INFO - step: 27135 loss: 16.9768 memory: 44.58GiB(31.99%) tps: 82,204 tflops: 283.70 mfu: 28.69% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.3405 +[titan] 2025-07-09 19:34:14,637 - root - INFO - lr: 2.5617e-04 gnorm: 0.79 [ 5:59:16<16:04:46] +[titan] 2025-07-09 19:34:15,584 - root - INFO - Dumping profiler traces at step 27136 +[titan] 2025-07-09 19:34:15,615 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:34:18,758 - root - INFO - step: 27140 loss: 16.5682 memory: 44.58GiB(31.99%) tps: 79,507 tflops: 274.39 mfu: 27.74% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 13.9998 +[titan] 2025-07-09 19:34:18,759 - root - INFO - lr: 2.5615e-04 gnorm: 0.81 [ 5:59:20<16:04:42] +[titan] 2025-07-09 19:34:22,671 - root - INFO - step: 27145 loss: 16.9133 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6265 global_avg_mtp_loss: 14.2869 +[titan] 2025-07-09 19:34:22,671 - root - INFO - lr: 2.5614e-04 gnorm: 0.80 [ 5:59:24<16:04:38] +[titan] 2025-07-09 19:34:25,783 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:34:26,578 - root - INFO - step: 27150 loss: 16.8227 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6033 global_avg_mtp_loss: 14.2194 +[titan] 2025-07-09 19:34:26,578 - root - INFO - lr: 2.5612e-04 gnorm: 0.74 [ 5:59:28<16:04:34] +[titan] 2025-07-09 19:34:30,520 - root - INFO - step: 27155 loss: 16.6404 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.0479 +[titan] 2025-07-09 19:34:30,520 - root - INFO - lr: 2.5611e-04 gnorm: 0.79 [ 5:59:32<16:04:30] +[titan] 2025-07-09 19:34:34,409 - root - INFO - step: 27160 loss: 17.1138 memory: 44.58GiB(31.99%) tps: 84,261 tflops: 290.80 mfu: 29.40% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.4439 +[titan] 2025-07-09 19:34:34,409 - root - INFO - lr: 2.5609e-04 gnorm: 0.77 [ 5:59:36<16:04:25] +[titan] 2025-07-09 19:34:38,308 - root - INFO - step: 27165 loss: 16.7786 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1727 +[titan] 2025-07-09 19:34:38,308 - root - INFO - lr: 2.5608e-04 gnorm: 0.79 [ 5:59:40<16:04:21] +[titan] 2025-07-09 19:34:42,194 - root - INFO - step: 27170 loss: 16.5998 memory: 44.58GiB(31.99%) tps: 84,319 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.0269 +[titan] 2025-07-09 19:34:42,195 - root - INFO - lr: 2.5606e-04 gnorm: 0.74 [ 5:59:44<16:04:17] +[titan] 2025-07-09 19:34:46,100 - root - INFO - step: 27175 loss: 16.8294 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6091 global_avg_mtp_loss: 14.2203 +[titan] 2025-07-09 19:34:46,100 - root - INFO - lr: 2.5604e-04 gnorm: 0.82 [ 5:59:48<16:04:13] +[titan] 2025-07-09 19:34:50,043 - root - INFO - step: 27180 loss: 16.8563 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6267 global_avg_mtp_loss: 14.2296 +[titan] 2025-07-09 19:34:50,043 - root - INFO - lr: 2.5603e-04 gnorm: 0.79 [ 5:59:52<16:04:09] +[titan] 2025-07-09 19:34:53,955 - root - INFO - step: 27185 loss: 16.9280 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6361 global_avg_mtp_loss: 14.2920 +[titan] 2025-07-09 19:34:53,955 - root - INFO - lr: 2.5601e-04 gnorm: 0.75 [ 5:59:56<16:04:05] +[titan] 2025-07-09 19:34:57,866 - root - INFO - step: 27190 loss: 16.7905 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.1722 +[titan] 2025-07-09 19:34:57,866 - root - INFO - lr: 2.5600e-04 gnorm: 0.83 [ 6:00:00<16:04:01] +[titan] 2025-07-09 19:35:01,781 - root - INFO - step: 27195 loss: 16.8505 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2332 +[titan] 2025-07-09 19:35:01,782 - root - INFO - lr: 2.5598e-04 gnorm: 0.77 [ 6:00:03<16:03:56] +[titan] 2025-07-09 19:35:04,928 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:35:05,722 - root - INFO - step: 27200 loss: 16.8282 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.2241 +[titan] 2025-07-09 19:35:05,722 - root - INFO - lr: 2.5597e-04 gnorm: 0.79 [ 6:00:07<16:03:52] +[titan] 2025-07-09 19:35:09,647 - root - INFO - step: 27205 loss: 16.7572 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6014 global_avg_mtp_loss: 14.1558 +[titan] 2025-07-09 19:35:09,647 - root - INFO - lr: 2.5595e-04 gnorm: 0.86 [ 6:00:11<16:03:48] +[titan] 2025-07-09 19:35:13,574 - root - INFO - step: 27210 loss: 16.8349 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6139 global_avg_mtp_loss: 14.2210 +[titan] 2025-07-09 19:35:13,575 - root - INFO - lr: 2.5593e-04 gnorm: 0.75 [ 6:00:15<16:03:44] +[titan] 2025-07-09 19:35:17,495 - root - INFO - step: 27215 loss: 16.8466 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6097 global_avg_mtp_loss: 14.2369 +[titan] 2025-07-09 19:35:17,495 - root - INFO - lr: 2.5592e-04 gnorm: 0.90 [ 6:00:19<16:03:40] +[titan] 2025-07-09 19:35:21,402 - root - INFO - step: 27220 loss: 16.9194 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6361 global_avg_mtp_loss: 14.2833 +[titan] 2025-07-09 19:35:21,402 - root - INFO - lr: 2.5590e-04 gnorm: 0.80 [ 6:00:23<16:03:36] +[titan] 2025-07-09 19:35:25,293 - root - INFO - step: 27225 loss: 16.7477 memory: 44.58GiB(31.99%) tps: 84,211 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 19:35:25,293 - root - INFO - lr: 2.5589e-04 gnorm: 0.80 [ 6:00:27<16:03:32] +[titan] 2025-07-09 19:35:29,215 - root - INFO - step: 27230 loss: 16.8306 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2220 +[titan] 2025-07-09 19:35:29,215 - root - INFO - lr: 2.5587e-04 gnorm: 0.83 [ 6:00:31<16:03:28] +[titan] 2025-07-09 19:35:33,122 - root - INFO - step: 27235 loss: 16.7196 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.1441 +[titan] 2025-07-09 19:35:33,122 - root - INFO - lr: 2.5585e-04 gnorm: 0.77 [ 6:00:35<16:03:23] +[titan] 2025-07-09 19:35:37,038 - root - INFO - step: 27240 loss: 16.7549 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5931 global_avg_mtp_loss: 14.1618 +[titan] 2025-07-09 19:35:37,038 - root - INFO - lr: 2.5584e-04 gnorm: 0.92 [ 6:00:39<16:03:19] +[titan] 2025-07-09 19:35:40,935 - root - INFO - step: 27245 loss: 16.8476 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.2216 +[titan] 2025-07-09 19:35:40,935 - root - INFO - lr: 2.5582e-04 gnorm: 0.72 [ 6:00:43<16:03:15] +[titan] 2025-07-09 19:35:44,059 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:35:44,864 - root - INFO - step: 27250 loss: 16.8621 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.2419 +[titan] 2025-07-09 19:35:44,864 - root - INFO - lr: 2.5581e-04 gnorm: 0.77 [ 6:00:47<16:03:11] +[titan] 2025-07-09 19:35:48,772 - root - INFO - step: 27255 loss: 16.8258 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.2264 +[titan] 2025-07-09 19:35:48,772 - root - INFO - lr: 2.5579e-04 gnorm: 0.74 [ 6:00:50<16:03:07] +[titan] 2025-07-09 19:35:52,719 - root - INFO - step: 27260 loss: 16.9785 memory: 44.58GiB(31.99%) tps: 83,037 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.6285 global_avg_mtp_loss: 14.3500 +[titan] 2025-07-09 19:35:52,719 - root - INFO - lr: 2.5578e-04 gnorm: 0.71 [ 6:00:54<16:03:03] +[titan] 2025-07-09 19:35:56,634 - root - INFO - step: 27265 loss: 16.7150 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.1356 +[titan] 2025-07-09 19:35:56,634 - root - INFO - lr: 2.5576e-04 gnorm: 0.76 [ 6:00:58<16:02:59] +[titan] 2025-07-09 19:36:00,549 - root - INFO - step: 27270 loss: 16.7628 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1683 +[titan] 2025-07-09 19:36:00,550 - root - INFO - lr: 2.5574e-04 gnorm: 0.78 [ 6:01:02<16:02:55] +[titan] 2025-07-09 19:36:04,444 - root - INFO - step: 27275 loss: 16.9344 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6465 global_avg_mtp_loss: 14.2879 +[titan] 2025-07-09 19:36:04,445 - root - INFO - lr: 2.5573e-04 gnorm: 0.78 [ 6:01:06<16:02:50] +[titan] 2025-07-09 19:36:08,348 - root - INFO - step: 27280 loss: 16.8581 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.2391 +[titan] 2025-07-09 19:36:08,348 - root - INFO - lr: 2.5571e-04 gnorm: 0.75 [ 6:01:10<16:02:46] +[titan] 2025-07-09 19:36:12,269 - root - INFO - step: 27285 loss: 16.6502 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0745 +[titan] 2025-07-09 19:36:12,269 - root - INFO - lr: 2.5570e-04 gnorm: 0.83 [ 6:01:14<16:02:42] +[titan] 2025-07-09 19:36:16,202 - root - INFO - step: 27290 loss: 17.0374 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6714 global_avg_mtp_loss: 14.3660 +[titan] 2025-07-09 19:36:16,202 - root - INFO - lr: 2.5568e-04 gnorm: 0.83 [ 6:01:18<16:02:38] +[titan] 2025-07-09 19:36:20,109 - root - INFO - step: 27295 loss: 16.7076 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1192 +[titan] 2025-07-09 19:36:20,109 - root - INFO - lr: 2.5566e-04 gnorm: 0.81 [ 6:01:22<16:02:34][39m +[titan] 2025-07-09 19:36:23,270 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:36:24,065 - root - INFO - step: 27300 loss: 17.2454 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.7018 global_avg_mtp_loss: 14.5436 +[titan] 2025-07-09 19:36:24,065 - root - INFO - lr: 2.5565e-04 gnorm: 0.77 [ 6:01:26<16:02:30] +[titan] 2025-07-09 19:36:27,960 - root - INFO - step: 27305 loss: 16.8749 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6244 global_avg_mtp_loss: 14.2506 +[titan] 2025-07-09 19:36:27,960 - root - INFO - lr: 2.5563e-04 gnorm: 0.74 [ 6:01:30<16:02:26] +[titan] 2025-07-09 19:36:31,860 - root - INFO - step: 27310 loss: 16.8631 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6089 global_avg_mtp_loss: 14.2542 +[titan] 2025-07-09 19:36:31,860 - root - INFO - lr: 2.5562e-04 gnorm: 0.78 [ 6:01:34<16:02:22] +[titan] 2025-07-09 19:36:35,769 - root - INFO - step: 27315 loss: 16.8189 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5985 global_avg_mtp_loss: 14.2203 +[titan] 2025-07-09 19:36:35,769 - root - INFO - lr: 2.5560e-04 gnorm: 0.76 [ 6:01:37<16:02:18] +[titan] 2025-07-09 19:36:39,729 - root - INFO - step: 27320 loss: 16.7348 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5958 global_avg_mtp_loss: 14.1390 +[titan] 2025-07-09 19:36:39,729 - root - INFO - lr: 2.5558e-04 gnorm: 0.77 [ 6:01:41<16:02:14] +[titan] 2025-07-09 19:36:43,698 - root - INFO - step: 27325 loss: 16.7027 memory: 44.58GiB(31.99%) tps: 82,564 tflops: 284.94 mfu: 28.81% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1131 +[titan] 2025-07-09 19:36:43,699 - root - INFO - lr: 2.5557e-04 gnorm: 0.74 [ 6:01:45<16:02:10] +[titan] 2025-07-09 19:36:47,662 - root - INFO - step: 27330 loss: 16.7718 memory: 44.58GiB(31.99%) tps: 82,674 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.1720 +[titan] 2025-07-09 19:36:47,663 - root - INFO - lr: 2.5555e-04 gnorm: 0.77 [ 6:01:49<16:02:06] +[titan] 2025-07-09 19:36:51,570 - root - INFO - step: 27335 loss: 16.9634 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6434 global_avg_mtp_loss: 14.3201 +[titan] 2025-07-09 19:36:51,570 - root - INFO - lr: 2.5554e-04 gnorm: 0.79 [ 6:01:53<16:02:01] +[titan] 2025-07-09 19:36:55,484 - root - INFO - step: 27340 loss: 16.7237 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1319 +[titan] 2025-07-09 19:36:55,485 - root - INFO - lr: 2.5552e-04 gnorm: 0.77 [ 6:01:57<16:01:57] +[titan] 2025-07-09 19:36:59,404 - root - INFO - step: 27345 loss: 16.6555 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.0784 +[titan] 2025-07-09 19:36:59,404 - root - INFO - lr: 2.5551e-04 gnorm: 0.77 [ 6:02:01<16:01:53] +[titan] 2025-07-09 19:37:02,536 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:37:03,330 - root - INFO - step: 27350 loss: 16.7016 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1181 +[titan] 2025-07-09 19:37:03,330 - root - INFO - lr: 2.5549e-04 gnorm: 0.72 [ 6:02:05<16:01:49] +[titan] 2025-07-09 19:37:07,265 - root - INFO - step: 27355 loss: 16.7583 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5979 global_avg_mtp_loss: 14.1605 +[titan] 2025-07-09 19:37:07,265 - root - INFO - lr: 2.5547e-04 gnorm: 0.77 [ 6:02:09<16:01:45] +[titan] 2025-07-09 19:37:11,186 - root - INFO - step: 27360 loss: 16.6956 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1108 +[titan] 2025-07-09 19:37:11,186 - root - INFO - lr: 2.5546e-04 gnorm: 0.78 [ 6:02:13<16:01:41] +[titan] 2025-07-09 19:37:15,113 - root - INFO - step: 27365 loss: 16.8596 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2411 +[titan] 2025-07-09 19:37:15,113 - root - INFO - lr: 2.5544e-04 gnorm: 0.81 [ 6:02:17<16:01:37] +[titan] 2025-07-09 19:37:19,088 - root - INFO - step: 27370 loss: 16.8160 memory: 44.58GiB(31.99%) tps: 82,452 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.6112 global_avg_mtp_loss: 14.2049 +[titan] 2025-07-09 19:37:19,088 - root - INFO - lr: 2.5543e-04 gnorm: 0.82 [ 6:02:21<16:01:33] +[titan] 2025-07-09 19:37:22,996 - root - INFO - step: 27375 loss: 16.8403 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.2214 +[titan] 2025-07-09 19:37:22,997 - root - INFO - lr: 2.5541e-04 gnorm: 0.78 [ 6:02:25<16:01:29] +[titan] 2025-07-09 19:37:26,924 - root - INFO - step: 27380 loss: 16.5459 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 13.9846 +[titan] 2025-07-09 19:37:26,924 - root - INFO - lr: 2.5539e-04 gnorm: 0.82 [ 6:02:29<16:01:25] +[titan] 2025-07-09 19:37:30,855 - root - INFO - step: 27385 loss: 16.9677 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.3313 +[titan] 2025-07-09 19:37:30,856 - root - INFO - lr: 2.5538e-04 gnorm: 0.72 [ 6:02:33<16:01:21] +[titan] 2025-07-09 19:37:34,837 - root - INFO - step: 27390 loss: 16.9174 memory: 44.58GiB(31.99%) tps: 82,315 tflops: 284.08 mfu: 28.72% global_avg_ntp_loss: 2.6389 global_avg_mtp_loss: 14.2785 +[titan] 2025-07-09 19:37:34,837 - root - INFO - lr: 2.5536e-04 gnorm: 0.83 [ 6:02:36<16:01:17] +[titan] 2025-07-09 19:37:38,780 - root - INFO - step: 27395 loss: 16.9178 memory: 44.58GiB(31.99%) tps: 83,108 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.6283 global_avg_mtp_loss: 14.2896 +[titan] 2025-07-09 19:37:38,780 - root - INFO - lr: 2.5535e-04 gnorm: 0.78 [ 6:02:40<16:01:13] +[titan] 2025-07-09 19:37:41,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:37:42,722 - root - INFO - step: 27400 loss: 16.6907 memory: 44.58GiB(31.99%) tps: 83,129 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5806 global_avg_mtp_loss: 14.1101 +[titan] 2025-07-09 19:37:42,722 - root - INFO - lr: 2.5533e-04 gnorm: 0.77 [ 6:02:44<16:01:08] +[titan] 2025-07-09 19:37:46,678 - root - INFO - step: 27405 loss: 16.6755 memory: 44.58GiB(31.99%) tps: 82,840 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1031 +[titan] 2025-07-09 19:37:46,678 - root - INFO - lr: 2.5531e-04 gnorm: 0.80 [ 6:02:48<16:01:04] +[titan] 2025-07-09 19:37:50,595 - root - INFO - step: 27410 loss: 16.9237 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2902 +[titan] 2025-07-09 19:37:50,595 - root - INFO - lr: 2.5530e-04 gnorm: 0.81 [ 6:02:52<16:01:00] +[titan] 2025-07-09 19:37:54,564 - root - INFO - step: 27415 loss: 16.7597 memory: 44.58GiB(31.99%) tps: 82,562 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.1511 +[titan] 2025-07-09 19:37:54,564 - root - INFO - lr: 2.5528e-04 gnorm: 0.77 [ 6:02:56<16:00:56] +[titan] 2025-07-09 19:37:58,481 - root - INFO - step: 27420 loss: 16.7601 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1657 +[titan] 2025-07-09 19:37:58,481 - root - INFO - lr: 2.5527e-04 gnorm: 0.81 [ 6:03:00<16:00:52] +[titan] 2025-07-09 19:38:02,404 - root - INFO - step: 27425 loss: 16.4397 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 13.8908 +[titan] 2025-07-09 19:38:02,404 - root - INFO - lr: 2.5525e-04 gnorm: 0.73 [ 6:03:04<16:00:48] +[titan] 2025-07-09 19:38:06,325 - root - INFO - step: 27430 loss: 16.7658 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.1628 +[titan] 2025-07-09 19:38:06,325 - root - INFO - lr: 2.5523e-04 gnorm: 0.82 [ 6:03:08<16:00:44] +[titan] 2025-07-09 19:38:10,230 - root - INFO - step: 27435 loss: 16.8066 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.2037 +[titan] 2025-07-09 19:38:10,230 - root - INFO - lr: 2.5522e-04 gnorm: 0.81 [ 6:03:12<16:00:40] +[titan] 2025-07-09 19:38:14,143 - root - INFO - step: 27440 loss: 16.9933 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6342 global_avg_mtp_loss: 14.3591 +[titan] 2025-07-09 19:38:14,143 - root - INFO - lr: 2.5520e-04 gnorm: 0.77 [ 6:03:16<16:00:36] +[titan] 2025-07-09 19:38:18,034 - root - INFO - step: 27445 loss: 16.6712 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.0827 +[titan] 2025-07-09 19:38:18,034 - root - INFO - lr: 2.5519e-04 gnorm: 0.74 [ 6:03:20<16:00:32] +[titan] 2025-07-09 19:38:21,163 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:38:21,949 - root - INFO - step: 27450 loss: 16.8827 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.2655 +[titan] 2025-07-09 19:38:21,949 - root - INFO - lr: 2.5517e-04 gnorm: 0.72 [ 6:03:24<16:00:27] +[titan] 2025-07-09 19:38:25,850 - root - INFO - step: 27455 loss: 16.7215 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1293 +[titan] 2025-07-09 19:38:25,850 - root - INFO - lr: 2.5516e-04 gnorm: 0.72 [ 6:03:27<16:00:23] +[titan] 2025-07-09 19:38:29,756 - root - INFO - step: 27460 loss: 16.5341 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 13.9687 +[titan] 2025-07-09 19:38:29,757 - root - INFO - lr: 2.5514e-04 gnorm: 0.76 [ 6:03:31<16:00:19] +[titan] 2025-07-09 19:38:33,678 - root - INFO - step: 27465 loss: 16.7668 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.6089 global_avg_mtp_loss: 14.1578 +[titan] 2025-07-09 19:38:33,678 - root - INFO - lr: 2.5512e-04 gnorm: 0.81 [ 6:03:35<16:00:15] +[titan] 2025-07-09 19:38:37,571 - root - INFO - step: 27470 loss: 16.5900 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.0112 +[titan] 2025-07-09 19:38:37,572 - root - INFO - lr: 2.5511e-04 gnorm: 0.81 [ 6:03:39<16:00:11] +[titan] 2025-07-09 19:38:41,480 - root - INFO - step: 27475 loss: 16.9510 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.3179 +[titan] 2025-07-09 19:38:41,480 - root - INFO - lr: 2.5509e-04 gnorm: 0.78 [ 6:03:43<16:00:07] +[titan] 2025-07-09 19:38:45,400 - root - INFO - step: 27480 loss: 16.5739 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0229 +[titan] 2025-07-09 19:38:45,401 - root - INFO - lr: 2.5508e-04 gnorm: 0.81 [ 6:03:47<16:00:03] +[titan] 2025-07-09 19:38:49,308 - root - INFO - step: 27485 loss: 16.7694 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.1699 +[titan] 2025-07-09 19:38:49,309 - root - INFO - lr: 2.5506e-04 gnorm: 0.81 [ 6:03:51<15:59:58] +[titan] 2025-07-09 19:38:53,198 - root - INFO - step: 27490 loss: 16.8999 memory: 44.58GiB(31.99%) tps: 84,264 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.6268 global_avg_mtp_loss: 14.2731 +[titan] 2025-07-09 19:38:53,198 - root - INFO - lr: 2.5504e-04 gnorm: 0.77 [ 6:03:55<15:59:54] +[titan] 2025-07-09 19:38:57,141 - root - INFO - step: 27495 loss: 16.6574 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.0863 +[titan] 2025-07-09 19:38:57,141 - root - INFO - lr: 2.5503e-04 gnorm: 0.79 [ 6:03:59<15:59:50] +[titan] 2025-07-09 19:39:00,276 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:39:01,060 - root - INFO - step: 27500 loss: 16.9042 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.2795 +[titan] 2025-07-09 19:39:01,060 - root - INFO - lr: 2.5501e-04 gnorm: 0.75 [ 6:04:03<15:59:46] +[titan] 2025-07-09 19:39:04,955 - root - INFO - step: 27505 loss: 16.5799 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0102 +[titan] 2025-07-09 19:39:04,956 - root - INFO - lr: 2.5500e-04 gnorm: 0.77 [ 6:04:07<15:59:42] +[titan] 2025-07-09 19:39:08,896 - root - INFO - step: 27510 loss: 17.0155 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.6434 global_avg_mtp_loss: 14.3721 +[titan] 2025-07-09 19:39:08,896 - root - INFO - lr: 2.5498e-04 gnorm: 0.82 [ 6:04:11<15:59:38] +[titan] 2025-07-09 19:39:12,822 - root - INFO - step: 27515 loss: 16.8007 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.1915 +[titan] 2025-07-09 19:39:12,822 - root - INFO - lr: 2.5496e-04 gnorm: 0.89 [ 6:04:14<15:59:34] +[titan] 2025-07-09 19:39:16,744 - root - INFO - step: 27520 loss: 16.8620 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.6177 global_avg_mtp_loss: 14.2443 +[titan] 2025-07-09 19:39:16,744 - root - INFO - lr: 2.5495e-04 gnorm: 0.75 [ 6:04:18<15:59:30] +[titan] 2025-07-09 19:39:20,632 - root - INFO - step: 27525 loss: 16.6886 memory: 44.58GiB(31.99%) tps: 84,286 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.5827 global_avg_mtp_loss: 14.1060 +[titan] 2025-07-09 19:39:20,632 - root - INFO - lr: 2.5493e-04 gnorm: 0.85 [ 6:04:22<15:59:25] +[titan] 2025-07-09 19:39:24,533 - root - INFO - step: 27530 loss: 16.7296 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1457 +[titan] 2025-07-09 19:39:24,534 - root - INFO - lr: 2.5492e-04 gnorm: 0.84 [ 6:04:26<15:59:21] +[titan] 2025-07-09 19:39:28,425 - root - INFO - step: 27535 loss: 16.7533 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.5981 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 19:39:28,425 - root - INFO - lr: 2.5490e-04 gnorm: 0.85 [ 6:04:30<15:59:17] +[titan] 2025-07-09 19:39:32,339 - root - INFO - step: 27540 loss: 17.1589 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6787 global_avg_mtp_loss: 14.4802 +[titan] 2025-07-09 19:39:32,339 - root - INFO - lr: 2.5488e-04 gnorm: 0.76 [ 6:04:34<15:59:13] +[titan] 2025-07-09 19:39:36,320 - root - INFO - step: 27545 loss: 16.5724 memory: 44.58GiB(31.99%) tps: 82,306 tflops: 284.05 mfu: 28.72% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0027 +[titan] 2025-07-09 19:39:36,321 - root - INFO - lr: 2.5487e-04 gnorm: 0.76 [ 6:04:38<15:59:09] +[titan] 2025-07-09 19:39:39,436 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:39:40,236 - root - INFO - step: 27550 loss: 16.8559 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.2496 +[titan] 2025-07-09 19:39:40,236 - root - INFO - lr: 2.5485e-04 gnorm: 0.76 [ 6:04:42<15:59:05] +[titan] 2025-07-09 19:39:44,173 - root - INFO - step: 27555 loss: 16.8537 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.6089 global_avg_mtp_loss: 14.2448 +[titan] 2025-07-09 19:39:44,173 - root - INFO - lr: 2.5484e-04 gnorm: 0.75 [ 6:04:46<15:59:01] +[titan] 2025-07-09 19:39:48,066 - root - INFO - step: 27560 loss: 16.5253 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 13.9505 +[titan] 2025-07-09 19:39:48,067 - root - INFO - lr: 2.5482e-04 gnorm: 0.78 [ 6:04:50<15:58:57] +[titan] 2025-07-09 19:39:51,996 - root - INFO - step: 27565 loss: 16.4677 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 13.9098 +[titan] 2025-07-09 19:39:51,996 - root - INFO - lr: 2.5480e-04 gnorm: 0.78 [ 6:04:54<15:58:53] +[titan] 2025-07-09 19:39:55,913 - root - INFO - step: 27570 loss: 16.8698 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2420 +[titan] 2025-07-09 19:39:55,913 - root - INFO - lr: 2.5479e-04 gnorm: 0.74 [ 6:04:58<15:58:49] +[titan] 2025-07-09 19:39:59,817 - root - INFO - step: 27575 loss: 16.6187 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.0329 +[titan] 2025-07-09 19:39:59,817 - root - INFO - lr: 2.5477e-04 gnorm: 0.86 [ 6:05:01<15:58:44] +[titan] 2025-07-09 19:40:03,721 - root - INFO - step: 27580 loss: 16.7339 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1420 +[titan] 2025-07-09 19:40:03,722 - root - INFO - lr: 2.5476e-04 gnorm: 0.78 [ 6:05:05<15:58:40] +[titan] 2025-07-09 19:40:07,657 - root - INFO - step: 27585 loss: 16.5865 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.0179 +[titan] 2025-07-09 19:40:07,657 - root - INFO - lr: 2.5474e-04 gnorm: 0.81 [ 6:05:09<15:58:36] +[titan] 2025-07-09 19:40:11,564 - root - INFO - step: 27590 loss: 16.9035 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6295 global_avg_mtp_loss: 14.2740 +[titan] 2025-07-09 19:40:11,564 - root - INFO - lr: 2.5472e-04 gnorm: 0.80 [ 6:05:13<15:58:32] +[titan] 2025-07-09 19:40:15,483 - root - INFO - step: 27595 loss: 17.0809 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6786 global_avg_mtp_loss: 14.4023 +[titan] 2025-07-09 19:40:15,483 - root - INFO - lr: 2.5471e-04 gnorm: 0.94 [ 6:05:17<15:58:28] +[titan] 2025-07-09 19:40:18,610 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:40:19,396 - root - INFO - step: 27600 loss: 16.9677 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.3333 +[titan] 2025-07-09 19:40:19,396 - root - INFO - lr: 2.5469e-04 gnorm: 0.75 [ 6:05:21<15:58:24] +[titan] 2025-07-09 19:40:23,310 - root - INFO - step: 27605 loss: 16.9788 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.3416 +[titan] 2025-07-09 19:40:23,311 - root - INFO - lr: 2.5468e-04 gnorm: 0.72 [ 6:05:25<15:58:20] +[titan] 2025-07-09 19:40:27,266 - root - INFO - step: 27610 loss: 16.8996 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.2665 +[titan] 2025-07-09 19:40:27,267 - root - INFO - lr: 2.5466e-04 gnorm: 0.81 [ 6:05:29<15:58:16] +[titan] 2025-07-09 19:40:31,198 - root - INFO - step: 27615 loss: 16.7386 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.1470 +[titan] 2025-07-09 19:40:31,198 - root - INFO - lr: 2.5464e-04 gnorm: 0.79 [ 6:05:33<15:58:12] +[titan] 2025-07-09 19:40:35,109 - root - INFO - step: 27620 loss: 16.5871 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0161 +[titan] 2025-07-09 19:40:35,110 - root - INFO - lr: 2.5463e-04 gnorm: 0.76 [ 6:05:37<15:58:07] +[titan] 2025-07-09 19:40:39,025 - root - INFO - step: 27625 loss: 16.9775 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6470 global_avg_mtp_loss: 14.3305 +[titan] 2025-07-09 19:40:39,026 - root - INFO - lr: 2.5461e-04 gnorm: 0.78 [ 6:05:41<15:58:03] +[titan] 2025-07-09 19:40:42,934 - root - INFO - step: 27630 loss: 16.8612 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.2415 +[titan] 2025-07-09 19:40:42,934 - root - INFO - lr: 2.5460e-04 gnorm: 0.76 [ 6:05:45<15:57:59] +[titan] 2025-07-09 19:40:46,850 - root - INFO - step: 27635 loss: 16.8839 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.2706 +[titan] 2025-07-09 19:40:46,850 - root - INFO - lr: 2.5458e-04 gnorm: 0.75 [ 6:05:48<15:57:55] +[titan] 2025-07-09 19:40:50,772 - root - INFO - step: 27640 loss: 16.5619 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 13.9919 +[titan] 2025-07-09 19:40:50,773 - root - INFO - lr: 2.5456e-04 gnorm: 0.74 [ 6:05:52<15:57:51] +[titan] 2025-07-09 19:40:54,749 - root - INFO - step: 27645 loss: 16.7216 memory: 44.58GiB(31.99%) tps: 82,412 tflops: 284.42 mfu: 28.76% global_avg_ntp_loss: 2.5812 global_avg_mtp_loss: 14.1404 +[titan] 2025-07-09 19:40:54,749 - root - INFO - lr: 2.5455e-04 gnorm: 0.77 [ 6:05:56<15:57:47] +[titan] 2025-07-09 19:40:57,257 - root - INFO - Dumping profiler traces at step 27648 +[titan] 2025-07-09 19:40:57,288 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:40:58,067 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:40:58,853 - root - INFO - step: 27650 loss: 17.1555 memory: 44.58GiB(31.99%) tps: 79,848 tflops: 275.57 mfu: 27.86% global_avg_ntp_loss: 2.6792 global_avg_mtp_loss: 14.4762 +[titan] 2025-07-09 19:40:58,853 - root - INFO - lr: 2.5453e-04 gnorm: 0.79 [ 6:06:00<15:57:43] +[titan] 2025-07-09 19:41:02,772 - root - INFO - step: 27655 loss: 16.7768 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6028 global_avg_mtp_loss: 14.1740 +[titan] 2025-07-09 19:41:02,773 - root - INFO - lr: 2.5452e-04 gnorm: 0.76 [ 6:06:04<15:57:39] +[titan] 2025-07-09 19:41:06,663 - root - INFO - step: 27660 loss: 16.8903 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.6251 global_avg_mtp_loss: 14.2652 +[titan] 2025-07-09 19:41:06,664 - root - INFO - lr: 2.5450e-04 gnorm: 0.77 [ 6:06:08<15:57:35] +[titan] 2025-07-09 19:41:10,561 - root - INFO - step: 27665 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0433 +[titan] 2025-07-09 19:41:10,561 - root - INFO - lr: 2.5448e-04 gnorm: 0.81 [ 6:06:12<15:57:31] +[titan] 2025-07-09 19:41:14,457 - root - INFO - step: 27670 loss: 17.0658 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6592 global_avg_mtp_loss: 14.4066 +[titan] 2025-07-09 19:41:14,457 - root - INFO - lr: 2.5447e-04 gnorm: 0.76 [ 6:06:16<15:57:27] +[titan] 2025-07-09 19:41:18,366 - root - INFO - step: 27675 loss: 16.4457 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.9057 +[titan] 2025-07-09 19:41:18,367 - root - INFO - lr: 2.5445e-04 gnorm: 0.87 [ 6:06:20<15:57:23] +[titan] 2025-07-09 19:41:22,259 - root - INFO - step: 27680 loss: 16.9138 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2867 +[titan] 2025-07-09 19:41:22,259 - root - INFO - lr: 2.5444e-04 gnorm: 0.72 [ 6:06:24<15:57:18] +[titan] 2025-07-09 19:41:26,162 - root - INFO - step: 27685 loss: 16.7223 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1315 +[titan] 2025-07-09 19:41:26,162 - root - INFO - lr: 2.5442e-04 gnorm: 0.83 [ 6:06:28<15:57:14] +[titan] 2025-07-09 19:41:30,067 - root - INFO - step: 27690 loss: 16.6064 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5680 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-09 19:41:30,067 - root - INFO - lr: 2.5440e-04 gnorm: 0.83 [ 6:06:32<15:57:10] +[titan] 2025-07-09 19:41:33,969 - root - INFO - step: 27695 loss: 17.0505 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6456 global_avg_mtp_loss: 14.4049 +[titan] 2025-07-09 19:41:33,970 - root - INFO - lr: 2.5439e-04 gnorm: 0.85 [ 6:06:36<15:57:06] +[titan] 2025-07-09 19:41:37,134 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:41:37,943 - root - INFO - step: 27700 loss: 16.7359 memory: 44.58GiB(31.99%) tps: 82,473 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.1498 +[titan] 2025-07-09 19:41:37,943 - root - INFO - lr: 2.5437e-04 gnorm: 0.78 [ 6:06:40<15:57:02] +[titan] 2025-07-09 19:41:41,891 - root - INFO - step: 27705 loss: 17.0140 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.6441 global_avg_mtp_loss: 14.3699 +[titan] 2025-07-09 19:41:41,891 - root - INFO - lr: 2.5436e-04 gnorm: 0.82 [ 6:06:44<15:56:58] +[titan] 2025-07-09 19:41:45,819 - root - INFO - step: 27710 loss: 17.0233 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6652 global_avg_mtp_loss: 14.3581 +[titan] 2025-07-09 19:41:45,820 - root - INFO - lr: 2.5434e-04 gnorm: 0.79 [ 6:06:47<15:56:54] +[titan] 2025-07-09 19:41:49,770 - root - INFO - step: 27715 loss: 16.7656 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.6070 global_avg_mtp_loss: 14.1586 +[titan] 2025-07-09 19:41:49,770 - root - INFO - lr: 2.5432e-04 gnorm: 0.96 [ 6:06:51<15:56:50] +[titan] 2025-07-09 19:41:53,719 - root - INFO - step: 27720 loss: 16.8493 memory: 44.58GiB(31.99%) tps: 82,986 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.2385 +[titan] 2025-07-09 19:41:53,719 - root - INFO - lr: 2.5431e-04 gnorm: 0.80 [ 6:06:55<15:56:46] +[titan] 2025-07-09 19:41:57,626 - root - INFO - step: 27725 loss: 17.2657 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.7045 global_avg_mtp_loss: 14.5613 +[titan] 2025-07-09 19:41:57,626 - root - INFO - lr: 2.5429e-04 gnorm: 0.83 [ 6:06:59<15:56:42] +[titan] 2025-07-09 19:42:01,564 - root - INFO - step: 27730 loss: 16.7453 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5981 global_avg_mtp_loss: 14.1473 +[titan] 2025-07-09 19:42:01,564 - root - INFO - lr: 2.5427e-04 gnorm: 0.78 [ 6:07:03<15:56:38] +[titan] 2025-07-09 19:42:05,533 - root - INFO - step: 27735 loss: 17.7261 memory: 44.58GiB(31.99%) tps: 82,555 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.8552 global_avg_mtp_loss: 14.8710 +[titan] 2025-07-09 19:42:05,534 - root - INFO - lr: 2.5426e-04 gnorm: 10.14 [ 6:07:07<15:56:34] +[titan] 2025-07-09 19:42:09,473 - root - INFO - step: 27740 loss: 16.9111 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.2708 +[titan] 2025-07-09 19:42:09,473 - root - INFO - lr: 2.5424e-04 gnorm: 0.92 [ 6:07:11<15:56:30] +[titan] 2025-07-09 19:42:13,388 - root - INFO - step: 27745 loss: 16.6849 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.0977 +[titan] 2025-07-09 19:42:13,388 - root - INFO - lr: 2.5423e-04 gnorm: 0.82 [ 6:07:15<15:56:26] +[titan] 2025-07-09 19:42:16,510 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:42:17,299 - root - INFO - step: 27750 loss: 16.7092 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5967 global_avg_mtp_loss: 14.1125 +[titan] 2025-07-09 19:42:17,299 - root - INFO - lr: 2.5421e-04 gnorm: 0.80 [ 6:07:19<15:56:21] +[titan] 2025-07-09 19:42:21,198 - root - INFO - step: 27755 loss: 16.6724 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.0840 +[titan] 2025-07-09 19:42:21,198 - root - INFO - lr: 2.5419e-04 gnorm: 0.77 [ 6:07:23<15:56:17] +[titan] 2025-07-09 19:42:25,128 - root - INFO - step: 27760 loss: 16.7489 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1599 +[titan] 2025-07-09 19:42:25,128 - root - INFO - lr: 2.5418e-04 gnorm: 0.83 [ 6:07:27<15:56:13] +[titan] 2025-07-09 19:42:29,063 - root - INFO - step: 27765 loss: 16.7860 memory: 44.58GiB(31.99%) tps: 83,277 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.1732 +[titan] 2025-07-09 19:42:29,063 - root - INFO - lr: 2.5416e-04 gnorm: 0.80 [ 6:07:31<15:56:09] +[titan] 2025-07-09 19:42:32,963 - root - INFO - step: 27770 loss: 17.0685 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6529 global_avg_mtp_loss: 14.4156 +[titan] 2025-07-09 19:42:32,963 - root - INFO - lr: 2.5415e-04 gnorm: 0.86 [ 6:07:35<15:56:05] +[titan] 2025-07-09 19:42:36,879 - root - INFO - step: 27775 loss: 16.6844 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.0984 +[titan] 2025-07-09 19:42:36,880 - root - INFO - lr: 2.5413e-04 gnorm: 0.78 [ 6:07:38<15:56:01] +[titan] 2025-07-09 19:42:40,803 - root - INFO - step: 27780 loss: 16.7770 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.1721 +[titan] 2025-07-09 19:42:40,803 - root - INFO - lr: 2.5411e-04 gnorm: 0.77 [ 6:07:42<15:55:57] +[titan] 2025-07-09 19:42:44,724 - root - INFO - step: 27785 loss: 16.9197 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.2956 +[titan] 2025-07-09 19:42:44,724 - root - INFO - lr: 2.5410e-04 gnorm: 0.75 [ 6:07:46<15:55:53] +[titan] 2025-07-09 19:42:48,621 - root - INFO - step: 27790 loss: 16.9133 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.2828 +[titan] 2025-07-09 19:42:48,622 - root - INFO - lr: 2.5408e-04 gnorm: 0.73 [ 6:07:50<15:55:48] +[titan] 2025-07-09 19:42:52,530 - root - INFO - step: 27795 loss: 17.0192 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6506 global_avg_mtp_loss: 14.3686 +[titan] 2025-07-09 19:42:52,531 - root - INFO - lr: 2.5407e-04 gnorm: 0.70 [ 6:07:54<15:55:44] +[titan] 2025-07-09 19:42:55,649 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:42:56,434 - root - INFO - step: 27800 loss: 16.9262 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6273 global_avg_mtp_loss: 14.2989 +[titan] 2025-07-09 19:42:56,434 - root - INFO - lr: 2.5405e-04 gnorm: 0.74 [ 6:07:58<15:55:40] +[titan] 2025-07-09 19:43:00,329 - root - INFO - step: 27805 loss: 16.6669 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-09 19:43:00,329 - root - INFO - lr: 2.5403e-04 gnorm: 0.76 [ 6:08:02<15:55:36] +[titan] 2025-07-09 19:43:04,263 - root - INFO - step: 27810 loss: 16.5390 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5635 global_avg_mtp_loss: 13.9755 +[titan] 2025-07-09 19:43:04,263 - root - INFO - lr: 2.5402e-04 gnorm: 0.78 [ 6:08:06<15:55:32] +[titan] 2025-07-09 19:43:08,163 - root - INFO - step: 27815 loss: 16.8339 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.6125 global_avg_mtp_loss: 14.2214 +[titan] 2025-07-09 19:43:08,163 - root - INFO - lr: 2.5400e-04 gnorm: 0.79 [ 6:08:10<15:55:28] +[titan] 2025-07-09 19:43:12,073 - root - INFO - step: 27820 loss: 16.7599 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6080 global_avg_mtp_loss: 14.1519 +[titan] 2025-07-09 19:43:12,073 - root - INFO - lr: 2.5399e-04 gnorm: 0.79 [ 6:08:14<15:55:24] +[titan] 2025-07-09 19:43:16,022 - root - INFO - step: 27825 loss: 16.7719 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.5976 global_avg_mtp_loss: 14.1742 +[titan] 2025-07-09 19:43:16,022 - root - INFO - lr: 2.5397e-04 gnorm: 0.83 [ 6:08:18<15:55:20] +[titan] 2025-07-09 19:43:19,955 - root - INFO - step: 27830 loss: 16.7829 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.1763 +[titan] 2025-07-09 19:43:19,955 - root - INFO - lr: 2.5395e-04 gnorm: 0.77 [ 6:08:22<15:55:16] +[titan] 2025-07-09 19:43:23,894 - root - INFO - step: 27835 loss: 17.0433 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.6556 global_avg_mtp_loss: 14.3877 +[titan] 2025-07-09 19:43:23,894 - root - INFO - lr: 2.5394e-04 gnorm: 0.74 [ 6:08:26<15:55:11] +[titan] 2025-07-09 19:43:27,853 - root - INFO - step: 27840 loss: 16.8593 memory: 44.58GiB(31.99%) tps: 82,767 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.6216 global_avg_mtp_loss: 14.2377 +[titan] 2025-07-09 19:43:27,854 - root - INFO - lr: 2.5392e-04 gnorm: 0.77 [ 6:08:29<15:55:07] +[titan] 2025-07-09 19:43:31,755 - root - INFO - step: 27845 loss: 16.8133 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.1979 +[titan] 2025-07-09 19:43:31,755 - root - INFO - lr: 2.5390e-04 gnorm: 0.80 [ 6:08:33<15:55:03] +[titan] 2025-07-09 19:43:34,889 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:43:35,696 - root - INFO - step: 27850 loss: 16.8733 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.6218 global_avg_mtp_loss: 14.2515 +[titan] 2025-07-09 19:43:35,696 - root - INFO - lr: 2.5389e-04 gnorm: 0.73 [ 6:08:37<15:54:59] +[titan] 2025-07-09 19:43:39,622 - root - INFO - step: 27855 loss: 16.9787 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6324 global_avg_mtp_loss: 14.3463 +[titan] 2025-07-09 19:43:39,622 - root - INFO - lr: 2.5387e-04 gnorm: 0.77 [ 6:08:41<15:54:55] +[titan] 2025-07-09 19:43:43,546 - root - INFO - step: 27860 loss: 16.8364 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2172 +[titan] 2025-07-09 19:43:43,546 - root - INFO - lr: 2.5386e-04 gnorm: 0.81 [ 6:08:45<15:54:51] +[titan] 2025-07-09 19:43:47,441 - root - INFO - step: 27865 loss: 16.6437 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.0723 +[titan] 2025-07-09 19:43:47,441 - root - INFO - lr: 2.5384e-04 gnorm: 0.78 [ 6:08:49<15:54:47] +[titan] 2025-07-09 19:43:51,345 - root - INFO - step: 27870 loss: 16.9553 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6453 global_avg_mtp_loss: 14.3100 +[titan] 2025-07-09 19:43:51,345 - root - INFO - lr: 2.5382e-04 gnorm: 0.82 [ 6:08:53<15:54:43] +[titan] 2025-07-09 19:43:55,239 - root - INFO - step: 27875 loss: 16.6402 memory: 44.58GiB(31.99%) tps: 84,151 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.0602 +[titan] 2025-07-09 19:43:55,240 - root - INFO - lr: 2.5381e-04 gnorm: 0.76 [ 6:08:57<15:54:39] +[titan] 2025-07-09 19:43:59,152 - root - INFO - step: 27880 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1690 +[titan] 2025-07-09 19:43:59,152 - root - INFO - lr: 2.5379e-04 gnorm: 0.79 [ 6:09:01<15:54:35] +[titan] 2025-07-09 19:44:03,087 - root - INFO - step: 27885 loss: 16.8931 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.6221 global_avg_mtp_loss: 14.2710 +[titan] 2025-07-09 19:44:03,088 - root - INFO - lr: 2.5378e-04 gnorm: 0.80 [ 6:09:05<15:54:30] +[titan] 2025-07-09 19:44:07,017 - root - INFO - step: 27890 loss: 16.8215 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.6120 global_avg_mtp_loss: 14.2095 +[titan] 2025-07-09 19:44:07,017 - root - INFO - lr: 2.5376e-04 gnorm: 0.79 [ 6:09:09<15:54:26] +[titan] 2025-07-09 19:44:10,923 - root - INFO - step: 27895 loss: 16.9722 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6315 global_avg_mtp_loss: 14.3406 +[titan] 2025-07-09 19:44:10,924 - root - INFO - lr: 2.5374e-04 gnorm: 0.74 [ 6:09:13<15:54:22] +[titan] 2025-07-09 19:44:14,066 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:44:14,856 - root - INFO - step: 27900 loss: 16.7866 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6045 global_avg_mtp_loss: 14.1821 +[titan] 2025-07-09 19:44:14,856 - root - INFO - lr: 2.5373e-04 gnorm: 0.77 [ 6:09:16<15:54:18] +[titan] 2025-07-09 19:44:18,756 - root - INFO - step: 27905 loss: 16.8154 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.2149 +[titan] 2025-07-09 19:44:18,756 - root - INFO - lr: 2.5371e-04 gnorm: 0.77 [ 6:09:20<15:54:14] +[titan] 2025-07-09 19:44:22,681 - root - INFO - step: 27910 loss: 16.9848 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6495 global_avg_mtp_loss: 14.3353 +[titan] 2025-07-09 19:44:22,681 - root - INFO - lr: 2.5369e-04 gnorm: 0.71 [ 6:09:24<15:54:10] +[titan] 2025-07-09 19:44:26,627 - root - INFO - step: 27915 loss: 17.1700 memory: 44.58GiB(31.99%) tps: 83,046 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.6774 global_avg_mtp_loss: 14.4926 +[titan] 2025-07-09 19:44:26,627 - root - INFO - lr: 2.5368e-04 gnorm: 0.78 [ 6:09:28<15:54:06] +[titan] 2025-07-09 19:44:30,560 - root - INFO - step: 27920 loss: 16.9484 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.6340 global_avg_mtp_loss: 14.3144 +[titan] 2025-07-09 19:44:30,560 - root - INFO - lr: 2.5366e-04 gnorm: 0.78 [ 6:09:32<15:54:02] +[titan] 2025-07-09 19:44:34,503 - root - INFO - step: 27925 loss: 16.3839 memory: 44.58GiB(31.99%) tps: 83,113 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.8570 +[titan] 2025-07-09 19:44:34,503 - root - INFO - lr: 2.5365e-04 gnorm: 0.77 [ 6:09:36<15:53:58] +[titan] 2025-07-09 19:44:38,406 - root - INFO - step: 27930 loss: 16.7733 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.1773 +[titan] 2025-07-09 19:44:38,407 - root - INFO - lr: 2.5363e-04 gnorm: 0.76 [ 6:09:40<15:53:54] +[titan] 2025-07-09 19:44:42,293 - root - INFO - step: 27935 loss: 17.0312 memory: 44.58GiB(31.99%) tps: 84,310 tflops: 290.97 mfu: 29.42% global_avg_ntp_loss: 2.6497 global_avg_mtp_loss: 14.3815 +[titan] 2025-07-09 19:44:42,294 - root - INFO - lr: 2.5361e-04 gnorm: 0.74 [ 6:09:44<15:53:49] +[titan] 2025-07-09 19:44:46,210 - root - INFO - step: 27940 loss: 16.8934 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6275 global_avg_mtp_loss: 14.2659 +[titan] 2025-07-09 19:44:46,210 - root - INFO - lr: 2.5360e-04 gnorm: 0.75 [ 6:09:48<15:53:45] +[titan] 2025-07-09 19:44:50,122 - root - INFO - step: 27945 loss: 16.6535 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0823 +[titan] 2025-07-09 19:44:50,122 - root - INFO - lr: 2.5358e-04 gnorm: 0.76 [ 6:09:52<15:53:41] +[titan] 2025-07-09 19:44:53,239 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:44:54,025 - root - INFO - step: 27950 loss: 16.7766 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.1744 +[titan] 2025-07-09 19:44:54,025 - root - INFO - lr: 2.5357e-04 gnorm: 0.81 [ 6:09:56<15:53:37] +[titan] 2025-07-09 19:44:57,945 - root - INFO - step: 27955 loss: 16.8828 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6234 global_avg_mtp_loss: 14.2594 +[titan] 2025-07-09 19:44:57,945 - root - INFO - lr: 2.5355e-04 gnorm: 0.82 [ 6:10:00<15:53:33] +[titan] 2025-07-09 19:45:01,856 - root - INFO - step: 27960 loss: 16.7310 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1386 +[titan] 2025-07-09 19:45:01,857 - root - INFO - lr: 2.5353e-04 gnorm: 0.77 [ 6:10:03<15:53:29] +[titan] 2025-07-09 19:45:05,773 - root - INFO - step: 27965 loss: 16.9340 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.6275 global_avg_mtp_loss: 14.3065 +[titan] 2025-07-09 19:45:05,773 - root - INFO - lr: 2.5352e-04 gnorm: 0.74 [ 6:10:07<15:53:25] +[titan] 2025-07-09 19:45:09,709 - root - INFO - step: 27970 loss: 16.7365 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.6179 global_avg_mtp_loss: 14.1186 +[titan] 2025-07-09 19:45:09,710 - root - INFO - lr: 2.5350e-04 gnorm: 0.71 [ 6:10:11<15:53:21] +[titan] 2025-07-09 19:45:13,608 - root - INFO - step: 27975 loss: 16.4904 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 13.9346 +[titan] 2025-07-09 19:45:13,608 - root - INFO - lr: 2.5348e-04 gnorm: 0.78 [ 6:10:15<15:53:16] +[titan] 2025-07-09 19:45:17,535 - root - INFO - step: 27980 loss: 16.6654 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.0789 +[titan] 2025-07-09 19:45:17,535 - root - INFO - lr: 2.5347e-04 gnorm: 0.71 [ 6:10:19<15:53:12] +[titan] 2025-07-09 19:45:21,476 - root - INFO - step: 27985 loss: 16.8134 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.2002 +[titan] 2025-07-09 19:45:21,477 - root - INFO - lr: 2.5345e-04 gnorm: 0.76 [ 6:10:23<15:53:08] +[titan] 2025-07-09 19:45:25,421 - root - INFO - step: 27990 loss: 16.7819 memory: 44.58GiB(31.99%) tps: 83,069 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5975 global_avg_mtp_loss: 14.1844 +[titan] 2025-07-09 19:45:25,422 - root - INFO - lr: 2.5344e-04 gnorm: 0.76 [ 6:10:27<15:53:04] +[titan] 2025-07-09 19:45:29,356 - root - INFO - step: 27995 loss: 17.0431 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6413 global_avg_mtp_loss: 14.4018 +[titan] 2025-07-09 19:45:29,357 - root - INFO - lr: 2.5342e-04 gnorm: 0.83 [ 6:10:31<15:53:00] +[titan] 2025-07-09 19:45:32,477 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:45:33,270 - root - INFO - step: 28000 loss: 16.7727 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6077 global_avg_mtp_loss: 14.1650 +[titan] 2025-07-09 19:45:33,270 - root - INFO - lr: 2.5340e-04 gnorm: 0.80 [ 6:10:35<15:52:56] +[titan] 2025-07-09 19:45:37,188 - root - INFO - step: 28005 loss: 16.8922 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.2799 +[titan] 2025-07-09 19:45:37,189 - root - INFO - lr: 2.5339e-04 gnorm: 0.77 [ 6:10:39<15:52:52] +[titan] 2025-07-09 19:45:41,102 - root - INFO - step: 28010 loss: 16.6597 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.0758 +[titan] 2025-07-09 19:45:41,102 - root - INFO - lr: 2.5337e-04 gnorm: 0.75 [ 6:10:43<15:52:48] +[titan] 2025-07-09 19:45:45,057 - root - INFO - step: 28015 loss: 16.9001 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.6217 global_avg_mtp_loss: 14.2785 +[titan] 2025-07-09 19:45:45,057 - root - INFO - lr: 2.5336e-04 gnorm: 0.74 [ 6:10:47<15:52:44] +[titan] 2025-07-09 19:45:48,955 - root - INFO - step: 28020 loss: 16.5679 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0016 +[titan] 2025-07-09 19:45:48,956 - root - INFO - lr: 2.5334e-04 gnorm: 0.75 [ 6:10:51<15:52:40] +[titan] 2025-07-09 19:45:52,864 - root - INFO - step: 28025 loss: 16.8838 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.2554 +[titan] 2025-07-09 19:45:52,865 - root - INFO - lr: 2.5332e-04 gnorm: 0.77 [ 6:10:54<15:52:36] +[titan] 2025-07-09 19:45:56,769 - root - INFO - step: 28030 loss: 16.4175 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.8830 +[titan] 2025-07-09 19:45:56,769 - root - INFO - lr: 2.5331e-04 gnorm: 0.77 [ 6:10:58<15:52:31] +[titan] 2025-07-09 19:46:00,733 - root - INFO - step: 28035 loss: 16.8193 memory: 44.58GiB(31.99%) tps: 82,680 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.2171 +[titan] 2025-07-09 19:46:00,733 - root - INFO - lr: 2.5329e-04 gnorm: 0.80 [ 6:11:02<15:52:27] +[titan] 2025-07-09 19:46:04,643 - root - INFO - step: 28040 loss: 16.7638 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5983 global_avg_mtp_loss: 14.1654 +[titan] 2025-07-09 19:46:04,643 - root - INFO - lr: 2.5327e-04 gnorm: 0.73 [ 6:11:06<15:52:23] +[titan] 2025-07-09 19:46:08,582 - root - INFO - step: 28045 loss: 16.9151 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.2859 +[titan] 2025-07-09 19:46:08,582 - root - INFO - lr: 2.5326e-04 gnorm: 0.78 [ 6:11:10<15:52:19] +[titan] 2025-07-09 19:46:11,700 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:46:12,501 - root - INFO - step: 28050 loss: 16.6979 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1112 +[titan] 2025-07-09 19:46:12,501 - root - INFO - lr: 2.5324e-04 gnorm: 0.75 [ 6:11:14<15:52:15] +[titan] 2025-07-09 19:46:16,399 - root - INFO - step: 28055 loss: 16.9780 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6385 global_avg_mtp_loss: 14.3395 +[titan] 2025-07-09 19:46:16,400 - root - INFO - lr: 2.5323e-04 gnorm: 0.78 [ 6:11:18<15:52:11] +[titan] 2025-07-09 19:46:20,302 - root - INFO - step: 28060 loss: 16.7125 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1277 +[titan] 2025-07-09 19:46:20,303 - root - INFO - lr: 2.5321e-04 gnorm: 0.76 [ 6:11:22<15:52:07] +[titan] 2025-07-09 19:46:24,230 - root - INFO - step: 28065 loss: 16.8233 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.2300 +[titan] 2025-07-09 19:46:24,230 - root - INFO - lr: 2.5319e-04 gnorm: 0.76 [ 6:11:26<15:52:03] +[titan] 2025-07-09 19:46:28,148 - root - INFO - step: 28070 loss: 16.7943 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.1914 +[titan] 2025-07-09 19:46:28,149 - root - INFO - lr: 2.5318e-04 gnorm: 0.77 [ 6:11:30<15:51:59] +[titan] 2025-07-09 19:46:32,046 - root - INFO - step: 28075 loss: 17.1525 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6839 global_avg_mtp_loss: 14.4686 +[titan] 2025-07-09 19:46:32,046 - root - INFO - lr: 2.5316e-04 gnorm: 0.72 [ 6:11:34<15:51:55] +[titan] 2025-07-09 19:46:35,970 - root - INFO - step: 28080 loss: 16.9627 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6428 global_avg_mtp_loss: 14.3199 +[titan] 2025-07-09 19:46:35,970 - root - INFO - lr: 2.5314e-04 gnorm: 0.76 [ 6:11:38<15:51:50] +[titan] 2025-07-09 19:46:39,909 - root - INFO - step: 28085 loss: 17.0673 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.6718 global_avg_mtp_loss: 14.3954 +[titan] 2025-07-09 19:46:39,909 - root - INFO - lr: 2.5313e-04 gnorm: 0.76 [ 6:11:41<15:51:46] +[titan] 2025-07-09 19:46:43,848 - root - INFO - step: 28090 loss: 16.8705 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.6316 global_avg_mtp_loss: 14.2389 +[titan] 2025-07-09 19:46:43,848 - root - INFO - lr: 2.5311e-04 gnorm: 0.84 [ 6:11:45<15:51:42] +[titan] 2025-07-09 19:46:47,772 - root - INFO - step: 28095 loss: 16.8052 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6117 global_avg_mtp_loss: 14.1935 +[titan] 2025-07-09 19:46:47,772 - root - INFO - lr: 2.5310e-04 gnorm: 0.77 [ 6:11:49<15:51:38] +[titan] 2025-07-09 19:46:50,912 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:46:51,707 - root - INFO - step: 28100 loss: 16.7366 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.1476 +[titan] 2025-07-09 19:46:51,708 - root - INFO - lr: 2.5308e-04 gnorm: 0.84 [ 6:11:53<15:51:34] +[titan] 2025-07-09 19:46:55,618 - root - INFO - step: 28105 loss: 16.9068 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2797 +[titan] 2025-07-09 19:46:55,618 - root - INFO - lr: 2.5306e-04 gnorm: 0.88 [ 6:11:57<15:51:30] +[titan] 2025-07-09 19:46:59,524 - root - INFO - step: 28110 loss: 17.0343 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.3877 +[titan] 2025-07-09 19:46:59,524 - root - INFO - lr: 2.5305e-04 gnorm: 0.73 [ 6:12:01<15:51:26] +[titan] 2025-07-09 19:47:03,445 - root - INFO - step: 28115 loss: 17.0905 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.6504 global_avg_mtp_loss: 14.4401 +[titan] 2025-07-09 19:47:03,445 - root - INFO - lr: 2.5303e-04 gnorm: 0.83 [ 6:12:05<15:51:22] +[titan] 2025-07-09 19:47:07,356 - root - INFO - step: 28120 loss: 16.6855 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5887 global_avg_mtp_loss: 14.0968 +[titan] 2025-07-09 19:47:07,356 - root - INFO - lr: 2.5301e-04 gnorm: 0.83 [ 6:12:09<15:51:18] +[titan] 2025-07-09 19:47:11,277 - root - INFO - step: 28125 loss: 16.4043 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 13.8555 +[titan] 2025-07-09 19:47:11,277 - root - INFO - lr: 2.5300e-04 gnorm: 0.76 [ 6:12:13<15:51:14] +[titan] 2025-07-09 19:47:15,178 - root - INFO - step: 28130 loss: 16.9107 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.6231 global_avg_mtp_loss: 14.2875 +[titan] 2025-07-09 19:47:15,178 - root - INFO - lr: 2.5298e-04 gnorm: 0.73 [ 6:12:17<15:51:09] +[titan] 2025-07-09 19:47:19,136 - root - INFO - step: 28135 loss: 16.7611 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1763 +[titan] 2025-07-09 19:47:19,137 - root - INFO - lr: 2.5297e-04 gnorm: 0.73 [ 6:12:21<15:51:05] +[titan] 2025-07-09 19:47:23,037 - root - INFO - step: 28140 loss: 16.9851 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.6381 global_avg_mtp_loss: 14.3471 +[titan] 2025-07-09 19:47:23,037 - root - INFO - lr: 2.5295e-04 gnorm: 0.84 [ 6:12:25<15:51:01] +[titan] 2025-07-09 19:47:26,923 - root - INFO - step: 28145 loss: 16.7680 memory: 44.58GiB(31.99%) tps: 84,316 tflops: 290.99 mfu: 29.42% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1713 +[titan] 2025-07-09 19:47:26,924 - root - INFO - lr: 2.5293e-04 gnorm: 0.74 [ 6:12:29<15:50:57] +[titan] 2025-07-09 19:47:30,037 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:47:30,869 - root - INFO - step: 28150 loss: 16.9190 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.6216 global_avg_mtp_loss: 14.2975 +[titan] 2025-07-09 19:47:30,869 - root - INFO - lr: 2.5292e-04 gnorm: 0.77 [ 6:12:32<15:50:53] +[titan] 2025-07-09 19:47:34,790 - root - INFO - step: 28155 loss: 16.5283 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 13.9637 +[titan] 2025-07-09 19:47:34,790 - root - INFO - lr: 2.5290e-04 gnorm: 0.79 [ 6:12:36<15:50:49] +[titan] 2025-07-09 19:47:38,833 - root - INFO - step: 28160 loss: 16.3789 memory: 44.58GiB(31.99%) tps: 81,045 tflops: 279.70 mfu: 28.28% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-09 19:47:38,833 - root - INFO - lr: 2.5288e-04 gnorm: 0.76 [ 6:12:40<15:50:45] +[titan] 2025-07-09 19:47:38,989 - root - INFO - Dumping profiler traces at step 28160 +[titan] 2025-07-09 19:47:39,023 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:47:42,956 - root - INFO - step: 28165 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 79,484 tflops: 274.31 mfu: 27.74% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 13.9795 +[titan] 2025-07-09 19:47:42,956 - root - INFO - lr: 2.5287e-04 gnorm: 0.74 [ 6:12:45<15:50:42] +[titan] 2025-07-09 19:47:46,888 - root - INFO - step: 28170 loss: 16.7553 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1647 +[titan] 2025-07-09 19:47:46,889 - root - INFO - lr: 2.5285e-04 gnorm: 0.73 [ 6:12:48<15:50:38] +[titan] 2025-07-09 19:47:50,807 - root - INFO - step: 28175 loss: 16.7405 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6002 global_avg_mtp_loss: 14.1403 +[titan] 2025-07-09 19:47:50,807 - root - INFO - lr: 2.5284e-04 gnorm: 0.75 [ 6:12:52<15:50:33] +[titan] 2025-07-09 19:47:54,707 - root - INFO - step: 28180 loss: 16.7832 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5972 global_avg_mtp_loss: 14.1860 +[titan] 2025-07-09 19:47:54,707 - root - INFO - lr: 2.5282e-04 gnorm: 0.77 [ 6:12:56<15:50:29] +[titan] 2025-07-09 19:47:58,653 - root - INFO - step: 28185 loss: 16.7120 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1250 +[titan] 2025-07-09 19:47:58,653 - root - INFO - lr: 2.5280e-04 gnorm: 0.80 [ 6:13:00<15:50:25] +[titan] 2025-07-09 19:48:02,564 - root - INFO - step: 28190 loss: 16.8990 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2782 +[titan] 2025-07-09 19:48:02,564 - root - INFO - lr: 2.5279e-04 gnorm: 0.79 [ 6:13:04<15:50:21] +[titan] 2025-07-09 19:48:06,504 - root - INFO - step: 28195 loss: 16.3450 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.8074 +[titan] 2025-07-09 19:48:06,504 - root - INFO - lr: 2.5277e-04 gnorm: 0.84 [ 6:13:08<15:50:17] +[titan] 2025-07-09 19:48:09,642 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:48:10,438 - root - INFO - step: 28200 loss: 16.8399 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.2289 +[titan] 2025-07-09 19:48:10,439 - root - INFO - lr: 2.5275e-04 gnorm: 0.76 [ 6:13:12<15:50:13] +[titan] 2025-07-09 19:48:14,350 - root - INFO - step: 28205 loss: 16.8647 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6231 global_avg_mtp_loss: 14.2416 +[titan] 2025-07-09 19:48:14,351 - root - INFO - lr: 2.5274e-04 gnorm: 0.80 [ 6:13:16<15:50:09] +[titan] 2025-07-09 19:48:18,260 - root - INFO - step: 28210 loss: 16.6536 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0756 +[titan] 2025-07-09 19:48:18,261 - root - INFO - lr: 2.5272e-04 gnorm: 1.55 [ 6:13:20<15:50:05] +[titan] 2025-07-09 19:48:22,208 - root - INFO - step: 28215 loss: 16.4645 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 13.9213 +[titan] 2025-07-09 19:48:22,208 - root - INFO - lr: 2.5271e-04 gnorm: 0.83 [ 6:13:24<15:50:01] +[titan] 2025-07-09 19:48:26,110 - root - INFO - step: 28220 loss: 16.7650 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.1605 +[titan] 2025-07-09 19:48:26,110 - root - INFO - lr: 2.5269e-04 gnorm: 0.82 [ 6:13:28<15:49:57] +[titan] 2025-07-09 19:48:30,020 - root - INFO - step: 28225 loss: 16.9420 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6341 global_avg_mtp_loss: 14.3080 +[titan] 2025-07-09 19:48:30,020 - root - INFO - lr: 2.5267e-04 gnorm: 0.79 [ 6:13:32<15:49:53] +[titan] 2025-07-09 19:48:33,939 - root - INFO - step: 28230 loss: 16.9013 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6245 global_avg_mtp_loss: 14.2768 +[titan] 2025-07-09 19:48:33,939 - root - INFO - lr: 2.5266e-04 gnorm: 0.76 [ 6:13:36<15:49:48] +[titan] 2025-07-09 19:48:37,853 - root - INFO - step: 28235 loss: 16.7036 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6028 global_avg_mtp_loss: 14.1008 +[titan] 2025-07-09 19:48:37,853 - root - INFO - lr: 2.5264e-04 gnorm: 0.78 [ 6:13:39<15:49:44] +[titan] 2025-07-09 19:48:41,782 - root - INFO - step: 28240 loss: 16.8240 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.6035 global_avg_mtp_loss: 14.2205 +[titan] 2025-07-09 19:48:41,782 - root - INFO - lr: 2.5262e-04 gnorm: 0.84 [ 6:13:43<15:49:40] +[titan] 2025-07-09 19:48:45,745 - root - INFO - step: 28245 loss: 16.8906 memory: 44.58GiB(31.99%) tps: 82,700 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.2775 +[titan] 2025-07-09 19:48:45,745 - root - INFO - lr: 2.5261e-04 gnorm: 0.74 [ 6:13:47<15:49:36] +[titan] 2025-07-09 19:48:48,924 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:48:49,712 - root - INFO - step: 28250 loss: 16.8484 memory: 44.58GiB(31.99%) tps: 82,610 tflops: 285.10 mfu: 28.83% global_avg_ntp_loss: 2.6170 global_avg_mtp_loss: 14.2314 +[titan] 2025-07-09 19:48:49,712 - root - INFO - lr: 2.5259e-04 gnorm: 0.76 [ 6:13:51<15:49:32] +[titan] 2025-07-09 19:48:53,644 - root - INFO - step: 28255 loss: 16.7468 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5971 global_avg_mtp_loss: 14.1497 +[titan] 2025-07-09 19:48:53,644 - root - INFO - lr: 2.5258e-04 gnorm: 0.84 [ 6:13:55<15:49:28] +[titan] 2025-07-09 19:48:57,578 - root - INFO - step: 28260 loss: 16.9964 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3569 +[titan] 2025-07-09 19:48:57,578 - root - INFO - lr: 2.5256e-04 gnorm: 0.78 [ 6:13:59<15:49:24] +[titan] 2025-07-09 19:49:01,516 - root - INFO - step: 28265 loss: 16.8160 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.2172 +[titan] 2025-07-09 19:49:01,516 - root - INFO - lr: 2.5254e-04 gnorm: 0.73 [ 6:14:03<15:49:20] +[titan] 2025-07-09 19:49:05,439 - root - INFO - step: 28270 loss: 16.8698 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.2485 +[titan] 2025-07-09 19:49:05,439 - root - INFO - lr: 2.5253e-04 gnorm: 0.80 [ 6:14:07<15:49:16] +[titan] 2025-07-09 19:49:09,357 - root - INFO - step: 28275 loss: 16.7339 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.1346 +[titan] 2025-07-09 19:49:09,357 - root - INFO - lr: 2.5251e-04 gnorm: 0.77 [ 6:14:11<15:49:12] +[titan] 2025-07-09 19:49:13,256 - root - INFO - step: 28280 loss: 16.7023 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1189 +[titan] 2025-07-09 19:49:13,256 - root - INFO - lr: 2.5249e-04 gnorm: 0.76 [ 6:14:15<15:49:08] +[titan] 2025-07-09 19:49:17,166 - root - INFO - step: 28285 loss: 16.7749 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6104 global_avg_mtp_loss: 14.1645 +[titan] 2025-07-09 19:49:17,166 - root - INFO - lr: 2.5248e-04 gnorm: 0.73 [ 6:14:19<15:49:04] +[titan] 2025-07-09 19:49:21,078 - root - INFO - step: 28290 loss: 16.5210 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 13.9626 +[titan] 2025-07-09 19:49:21,078 - root - INFO - lr: 2.5246e-04 gnorm: 0.75 [ 6:14:23<15:48:59] +[titan] 2025-07-09 19:49:24,980 - root - INFO - step: 28295 loss: 16.7868 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.1859 +[titan] 2025-07-09 19:49:24,980 - root - INFO - lr: 2.5244e-04 gnorm: 0.79 [ 6:14:27<15:48:55] +[titan] 2025-07-09 19:49:28,110 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:49:28,904 - root - INFO - step: 28300 loss: 16.8284 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2059 +[titan] 2025-07-09 19:49:28,905 - root - INFO - lr: 2.5243e-04 gnorm: 0.79 [ 6:14:30<15:48:51] +[titan] 2025-07-09 19:49:32,803 - root - INFO - step: 28305 loss: 16.9601 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6442 global_avg_mtp_loss: 14.3160 +[titan] 2025-07-09 19:49:32,803 - root - INFO - lr: 2.5241e-04 gnorm: 0.79 [ 6:14:34<15:48:47] +[titan] 2025-07-09 19:49:36,698 - root - INFO - step: 28310 loss: 16.8632 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.6163 global_avg_mtp_loss: 14.2469 +[titan] 2025-07-09 19:49:36,698 - root - INFO - lr: 2.5240e-04 gnorm: 0.76 [ 6:14:38<15:48:43] +[titan] 2025-07-09 19:49:40,597 - root - INFO - step: 28315 loss: 16.7959 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.1734 +[titan] 2025-07-09 19:49:40,598 - root - INFO - lr: 2.5238e-04 gnorm: 0.84 [ 6:14:42<15:48:39] +[titan] 2025-07-09 19:49:44,505 - root - INFO - step: 28320 loss: 16.8868 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2694 +[titan] 2025-07-09 19:49:44,505 - root - INFO - lr: 2.5236e-04 gnorm: 0.74 [ 6:14:46<15:48:35] +[titan] 2025-07-09 19:49:48,433 - root - INFO - step: 28325 loss: 16.8072 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2006 +[titan] 2025-07-09 19:49:48,434 - root - INFO - lr: 2.5235e-04 gnorm: 0.78 [ 6:14:50<15:48:31] +[titan] 2025-07-09 19:49:52,352 - root - INFO - step: 28330 loss: 16.7037 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6000 global_avg_mtp_loss: 14.1037 +[titan] 2025-07-09 19:49:52,352 - root - INFO - lr: 2.5233e-04 gnorm: 0.77 [ 6:14:54<15:48:26] +[titan] 2025-07-09 19:49:56,270 - root - INFO - step: 28335 loss: 16.5667 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-09 19:49:56,270 - root - INFO - lr: 2.5231e-04 gnorm: 0.81 [ 6:14:58<15:48:22] +[titan] 2025-07-09 19:50:00,165 - root - INFO - step: 28340 loss: 16.6868 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1104 +[titan] 2025-07-09 19:50:00,165 - root - INFO - lr: 2.5230e-04 gnorm: 0.84 [ 6:15:02<15:48:18] +[titan] 2025-07-09 19:50:04,096 - root - INFO - step: 28345 loss: 16.9473 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.3061 +[titan] 2025-07-09 19:50:04,096 - root - INFO - lr: 2.5228e-04 gnorm: 0.80 [ 6:15:06<15:48:14] +[titan] 2025-07-09 19:50:07,251 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:50:08,039 - root - INFO - step: 28350 loss: 16.7884 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6037 global_avg_mtp_loss: 14.1847 +[titan] 2025-07-09 19:50:08,040 - root - INFO - lr: 2.5227e-04 gnorm: 0.81 [ 6:15:10<15:48:10] +[titan] 2025-07-09 19:50:11,969 - root - INFO - step: 28355 loss: 16.5983 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.0241 +[titan] 2025-07-09 19:50:11,969 - root - INFO - lr: 2.5225e-04 gnorm: 0.71 [ 6:15:14<15:48:06] +[titan] 2025-07-09 19:50:15,900 - root - INFO - step: 28360 loss: 16.7980 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.2128 +[titan] 2025-07-09 19:50:15,900 - root - INFO - lr: 2.5223e-04 gnorm: 0.83 [ 6:15:17<15:48:02] +[titan] 2025-07-09 19:50:19,819 - root - INFO - step: 28365 loss: 16.7320 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1412 +[titan] 2025-07-09 19:50:19,819 - root - INFO - lr: 2.5222e-04 gnorm: 0.75 [ 6:15:21<15:47:58] +[titan] 2025-07-09 19:50:23,718 - root - INFO - step: 28370 loss: 16.7650 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6043 global_avg_mtp_loss: 14.1607 +[titan] 2025-07-09 19:50:23,718 - root - INFO - lr: 2.5220e-04 gnorm: 0.78 [ 6:15:25<15:47:54] +[titan] 2025-07-09 19:50:27,612 - root - INFO - step: 28375 loss: 16.7817 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5961 global_avg_mtp_loss: 14.1856 +[titan] 2025-07-09 19:50:27,612 - root - INFO - lr: 2.5218e-04 gnorm: 0.72 [ 6:15:29<15:47:50] +[titan] 2025-07-09 19:50:31,542 - root - INFO - step: 28380 loss: 16.6958 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1065 +[titan] 2025-07-09 19:50:31,542 - root - INFO - lr: 2.5217e-04 gnorm: 0.77 [ 6:15:33<15:47:45] +[titan] 2025-07-09 19:50:35,441 - root - INFO - step: 28385 loss: 16.9137 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.2929 +[titan] 2025-07-09 19:50:35,441 - root - INFO - lr: 2.5215e-04 gnorm: 0.74 [ 6:15:37<15:47:41] +[titan] 2025-07-09 19:50:39,353 - root - INFO - step: 28390 loss: 16.8706 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6150 global_avg_mtp_loss: 14.2556 +[titan] 2025-07-09 19:50:39,353 - root - INFO - lr: 2.5213e-04 gnorm: 0.73 [ 6:15:41<15:47:37] +[titan] 2025-07-09 19:50:43,286 - root - INFO - step: 28395 loss: 17.0046 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6596 global_avg_mtp_loss: 14.3450 +[titan] 2025-07-09 19:50:43,286 - root - INFO - lr: 2.5212e-04 gnorm: 0.82 [ 6:15:45<15:47:33] +[titan] 2025-07-09 19:50:46,392 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:50:47,182 - root - INFO - step: 28400 loss: 16.6463 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.0620 +[titan] 2025-07-09 19:50:47,183 - root - INFO - lr: 2.5210e-04 gnorm: 0.77 [ 6:15:49<15:47:29] +[titan] 2025-07-09 19:50:51,067 - root - INFO - step: 28405 loss: 16.9336 memory: 44.58GiB(31.99%) tps: 84,352 tflops: 291.11 mfu: 29.43% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.3130 +[titan] 2025-07-09 19:50:51,068 - root - INFO - lr: 2.5209e-04 gnorm: 0.84 [ 6:15:53<15:47:25] +[titan] 2025-07-09 19:50:54,957 - root - INFO - step: 28410 loss: 16.7633 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5876 global_avg_mtp_loss: 14.1758 +[titan] 2025-07-09 19:50:54,958 - root - INFO - lr: 2.5207e-04 gnorm: 0.73 [ 6:15:57<15:47:21] +[titan] 2025-07-09 19:50:58,899 - root - INFO - step: 28415 loss: 16.6938 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.1183 +[titan] 2025-07-09 19:50:58,900 - root - INFO - lr: 2.5205e-04 gnorm: 0.74 [ 6:16:00<15:47:17] +[titan] 2025-07-09 19:51:02,802 - root - INFO - step: 28420 loss: 16.9425 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6405 global_avg_mtp_loss: 14.3020 +[titan] 2025-07-09 19:51:02,802 - root - INFO - lr: 2.5204e-04 gnorm: 0.77 [ 6:16:04<15:47:12] +[titan] 2025-07-09 19:51:06,727 - root - INFO - step: 28425 loss: 16.6064 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 19:51:06,728 - root - INFO - lr: 2.5202e-04 gnorm: 0.75 [ 6:16:08<15:47:08] +[titan] 2025-07-09 19:51:10,624 - root - INFO - step: 28430 loss: 17.0707 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6543 global_avg_mtp_loss: 14.4164 +[titan] 2025-07-09 19:51:10,624 - root - INFO - lr: 2.5200e-04 gnorm: 0.72 [ 6:16:12<15:47:04] +[titan] 2025-07-09 19:51:14,518 - root - INFO - step: 28435 loss: 16.5574 memory: 44.58GiB(31.99%) tps: 84,144 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0055 +[titan] 2025-07-09 19:51:14,519 - root - INFO - lr: 2.5199e-04 gnorm: 0.82 [ 6:16:16<15:47:00] +[titan] 2025-07-09 19:51:18,449 - root - INFO - step: 28440 loss: 16.8678 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.2518 +[titan] 2025-07-09 19:51:18,449 - root - INFO - lr: 2.5197e-04 gnorm: 0.78 [ 6:16:20<15:46:56] +[titan] 2025-07-09 19:51:22,355 - root - INFO - step: 28445 loss: 16.6920 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.1109 +[titan] 2025-07-09 19:51:22,355 - root - INFO - lr: 2.5195e-04 gnorm: 0.80 [ 6:16:24<15:46:52] +[titan] 2025-07-09 19:51:25,466 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:51:26,270 - root - INFO - step: 28450 loss: 16.9168 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6215 global_avg_mtp_loss: 14.2953 +[titan] 2025-07-09 19:51:26,270 - root - INFO - lr: 2.5194e-04 gnorm: 0.70 [ 6:16:28<15:46:48] +[titan] 2025-07-09 19:51:30,232 - root - INFO - step: 28455 loss: 16.5590 memory: 44.58GiB(31.99%) tps: 82,704 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0004 +[titan] 2025-07-09 19:51:30,233 - root - INFO - lr: 2.5192e-04 gnorm: 0.79 [ 6:16:32<15:46:44] +[titan] 2025-07-09 19:51:34,140 - root - INFO - step: 28460 loss: 16.8199 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.2151 +[titan] 2025-07-09 19:51:34,140 - root - INFO - lr: 2.5191e-04 gnorm: 0.81 [ 6:16:36<15:46:40] +[titan] 2025-07-09 19:51:38,046 - root - INFO - step: 28465 loss: 16.9230 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6358 global_avg_mtp_loss: 14.2872 +[titan] 2025-07-09 19:51:38,046 - root - INFO - lr: 2.5189e-04 gnorm: 0.72 [ 6:16:40<15:46:35] +[titan] 2025-07-09 19:51:41,962 - root - INFO - step: 28470 loss: 16.5724 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0253 +[titan] 2025-07-09 19:51:41,962 - root - INFO - lr: 2.5187e-04 gnorm: 0.79 [ 6:16:44<15:46:31] +[titan] 2025-07-09 19:51:45,939 - root - INFO - step: 28475 loss: 16.5950 memory: 44.58GiB(31.99%) tps: 82,407 tflops: 284.40 mfu: 28.76% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0280 +[titan] 2025-07-09 19:51:45,939 - root - INFO - lr: 2.5186e-04 gnorm: 0.80 [ 6:16:47<15:46:27] +[titan] 2025-07-09 19:51:49,905 - root - INFO - step: 28480 loss: 16.8061 memory: 44.58GiB(31.99%) tps: 82,628 tflops: 285.16 mfu: 28.83% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.2057 +[titan] 2025-07-09 19:51:49,905 - root - INFO - lr: 2.5184e-04 gnorm: 0.73 [ 6:16:51<15:46:23] +[titan] 2025-07-09 19:51:53,812 - root - INFO - step: 28485 loss: 16.8226 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6079 global_avg_mtp_loss: 14.2147 +[titan] 2025-07-09 19:51:53,812 - root - INFO - lr: 2.5182e-04 gnorm: 0.81 [ 6:16:55<15:46:19] +[titan] 2025-07-09 19:51:57,747 - root - INFO - step: 28490 loss: 17.0968 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6648 global_avg_mtp_loss: 14.4321 +[titan] 2025-07-09 19:51:57,748 - root - INFO - lr: 2.5181e-04 gnorm: 0.81 [ 6:16:59<15:46:15] +[titan] 2025-07-09 19:52:01,665 - root - INFO - step: 28495 loss: 16.8706 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.2665 +[titan] 2025-07-09 19:52:01,665 - root - INFO - lr: 2.5179e-04 gnorm: 0.79 [ 6:17:03<15:46:11] +[titan] 2025-07-09 19:52:04,817 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:52:05,619 - root - INFO - step: 28500 loss: 17.0714 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.6664 global_avg_mtp_loss: 14.4050 +[titan] 2025-07-09 19:52:05,620 - root - INFO - lr: 2.5177e-04 gnorm: 0.83 [ 6:17:07<15:46:07] +[titan] 2025-07-09 19:52:09,548 - root - INFO - step: 28505 loss: 17.1272 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6546 global_avg_mtp_loss: 14.4725 +[titan] 2025-07-09 19:52:09,548 - root - INFO - lr: 2.5176e-04 gnorm: 0.75 [ 6:17:11<15:46:03] +[titan] 2025-07-09 19:52:13,466 - root - INFO - step: 28510 loss: 16.8725 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.2675 +[titan] 2025-07-09 19:52:13,466 - root - INFO - lr: 2.5174e-04 gnorm: 0.80 [ 6:17:15<15:45:59] +[titan] 2025-07-09 19:52:17,404 - root - INFO - step: 28515 loss: 16.8911 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.2703 +[titan] 2025-07-09 19:52:17,405 - root - INFO - lr: 2.5172e-04 gnorm: 0.76 [ 6:17:19<15:45:55] +[titan] 2025-07-09 19:52:21,341 - root - INFO - step: 28520 loss: 16.9891 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.6421 global_avg_mtp_loss: 14.3470 +[titan] 2025-07-09 19:52:21,342 - root - INFO - lr: 2.5171e-04 gnorm: 0.74 [ 6:17:23<15:45:51] +[titan] 2025-07-09 19:52:25,264 - root - INFO - step: 28525 loss: 16.6283 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0546 +[titan] 2025-07-09 19:52:25,264 - root - INFO - lr: 2.5169e-04 gnorm: 0.83 [ 6:17:27<15:45:47] +[titan] 2025-07-09 19:52:29,205 - root - INFO - step: 28530 loss: 16.7300 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.1232 +[titan] 2025-07-09 19:52:29,205 - root - INFO - lr: 2.5168e-04 gnorm: 0.76 [ 6:17:31<15:45:43] +[titan] 2025-07-09 19:52:33,122 - root - INFO - step: 28535 loss: 16.6308 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0568 +[titan] 2025-07-09 19:52:33,122 - root - INFO - lr: 2.5166e-04 gnorm: 0.75 [ 6:17:35<15:45:39] +[titan] 2025-07-09 19:52:37,021 - root - INFO - step: 28540 loss: 16.5937 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.0109 +[titan] 2025-07-09 19:52:37,022 - root - INFO - lr: 2.5164e-04 gnorm: 0.75 [ 6:17:39<15:45:34] +[titan] 2025-07-09 19:52:40,939 - root - INFO - step: 28545 loss: 16.7743 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6161 global_avg_mtp_loss: 14.1581 +[titan] 2025-07-09 19:52:40,939 - root - INFO - lr: 2.5163e-04 gnorm: 0.80 [ 6:17:42<15:45:30] +[titan] 2025-07-09 19:52:44,075 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:52:44,880 - root - INFO - step: 28550 loss: 16.7455 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1510 +[titan] 2025-07-09 19:52:44,880 - root - INFO - lr: 2.5161e-04 gnorm: 0.74 [ 6:17:46<15:45:26] +[titan] 2025-07-09 19:52:48,816 - root - INFO - step: 28555 loss: 16.6684 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5706 global_avg_mtp_loss: 14.0978 +[titan] 2025-07-09 19:52:48,816 - root - INFO - lr: 2.5159e-04 gnorm: 0.78 [ 6:17:50<15:45:22] +[titan] 2025-07-09 19:52:52,746 - root - INFO - step: 28560 loss: 16.6481 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.0576 +[titan] 2025-07-09 19:52:52,746 - root - INFO - lr: 2.5158e-04 gnorm: 0.71 [ 6:17:54<15:45:18] +[titan] 2025-07-09 19:52:56,655 - root - INFO - step: 28565 loss: 16.7800 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6002 global_avg_mtp_loss: 14.1798 +[titan] 2025-07-09 19:52:56,656 - root - INFO - lr: 2.5156e-04 gnorm: 0.77 [ 6:17:58<15:45:14] +[titan] 2025-07-09 19:53:00,635 - root - INFO - step: 28570 loss: 16.8983 memory: 44.58GiB(31.99%) tps: 82,336 tflops: 284.15 mfu: 28.73% global_avg_ntp_loss: 2.6311 global_avg_mtp_loss: 14.2672 +[titan] 2025-07-09 19:53:00,636 - root - INFO - lr: 2.5154e-04 gnorm: 0.80 [ 6:18:02<15:45:10] +[titan] 2025-07-09 19:53:04,541 - root - INFO - step: 28575 loss: 16.6224 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5701 global_avg_mtp_loss: 14.0523 +[titan] 2025-07-09 19:53:04,541 - root - INFO - lr: 2.5153e-04 gnorm: 0.76 [ 6:18:06<15:45:06] +[titan] 2025-07-09 19:53:08,451 - root - INFO - step: 28580 loss: 16.8767 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 19:53:08,451 - root - INFO - lr: 2.5151e-04 gnorm: 0.79 [ 6:18:10<15:45:02] +[titan] 2025-07-09 19:53:12,394 - root - INFO - step: 28585 loss: 16.8731 memory: 44.58GiB(31.99%) tps: 83,108 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.6212 global_avg_mtp_loss: 14.2519 +[titan] 2025-07-09 19:53:12,395 - root - INFO - lr: 2.5149e-04 gnorm: 0.78 [ 6:18:14<15:44:58] +[titan] 2025-07-09 19:53:16,356 - root - INFO - step: 28590 loss: 16.8161 memory: 44.58GiB(31.99%) tps: 82,723 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.2138 +[titan] 2025-07-09 19:53:16,356 - root - INFO - lr: 2.5148e-04 gnorm: 0.79 [ 6:18:18<15:44:54] +[titan] 2025-07-09 19:53:20,254 - root - INFO - step: 28595 loss: 16.9450 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6404 global_avg_mtp_loss: 14.3046 +[titan] 2025-07-09 19:53:20,254 - root - INFO - lr: 2.5146e-04 gnorm: 0.73 [ 6:18:22<15:44:50] +[titan] 2025-07-09 19:53:23,372 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:53:24,167 - root - INFO - step: 28600 loss: 16.7220 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1298 +[titan] 2025-07-09 19:53:24,167 - root - INFO - lr: 2.5145e-04 gnorm: 0.82 [ 6:18:26<15:44:46] +[titan] 2025-07-09 19:53:28,118 - root - INFO - step: 28605 loss: 16.8006 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.94% global_avg_ntp_loss: 2.6095 global_avg_mtp_loss: 14.1911 +[titan] 2025-07-09 19:53:28,118 - root - INFO - lr: 2.5143e-04 gnorm: 0.80 [ 6:18:30<15:44:42] +[titan] 2025-07-09 19:53:32,058 - root - INFO - step: 28610 loss: 16.7530 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.1577 +[titan] 2025-07-09 19:53:32,058 - root - INFO - lr: 2.5141e-04 gnorm: 0.85 [ 6:18:34<15:44:38] +[titan] 2025-07-09 19:53:35,983 - root - INFO - step: 28615 loss: 16.7573 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.1746 +[titan] 2025-07-09 19:53:35,983 - root - INFO - lr: 2.5140e-04 gnorm: 0.75 [ 6:18:38<15:44:33] +[titan] 2025-07-09 19:53:39,934 - root - INFO - step: 28620 loss: 16.8558 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.6227 global_avg_mtp_loss: 14.2331 +[titan] 2025-07-09 19:53:39,934 - root - INFO - lr: 2.5138e-04 gnorm: 0.80 [ 6:18:41<15:44:29] +[titan] 2025-07-09 19:53:43,866 - root - INFO - step: 28625 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.8572 +[titan] 2025-07-09 19:53:43,867 - root - INFO - lr: 2.5136e-04 gnorm: 0.73 [ 6:18:45<15:44:25] +[titan] 2025-07-09 19:53:47,794 - root - INFO - step: 28630 loss: 16.8187 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.6104 global_avg_mtp_loss: 14.2083 +[titan] 2025-07-09 19:53:47,794 - root - INFO - lr: 2.5135e-04 gnorm: 0.83 [ 6:18:49<15:44:21] +[titan] 2025-07-09 19:53:51,718 - root - INFO - step: 28635 loss: 16.7886 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1919 +[titan] 2025-07-09 19:53:51,719 - root - INFO - lr: 2.5133e-04 gnorm: 0.78 [ 6:18:53<15:44:17] +[titan] 2025-07-09 19:53:55,666 - root - INFO - step: 28640 loss: 16.5930 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 19:53:55,666 - root - INFO - lr: 2.5131e-04 gnorm: 0.80 [ 6:18:57<15:44:13] +[titan] 2025-07-09 19:53:59,574 - root - INFO - step: 28645 loss: 16.9044 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6292 global_avg_mtp_loss: 14.2753 +[titan] 2025-07-09 19:53:59,575 - root - INFO - lr: 2.5130e-04 gnorm: 0.77 [ 6:19:01<15:44:09] +[titan] 2025-07-09 19:54:02,721 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:54:03,510 - root - INFO - step: 28650 loss: 16.7041 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5980 global_avg_mtp_loss: 14.1062 +[titan] 2025-07-09 19:54:03,510 - root - INFO - lr: 2.5128e-04 gnorm: 0.72 [ 6:19:05<15:44:05] +[titan] 2025-07-09 19:54:07,445 - root - INFO - step: 28655 loss: 16.8127 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6146 global_avg_mtp_loss: 14.1981 +[titan] 2025-07-09 19:54:07,445 - root - INFO - lr: 2.5126e-04 gnorm: 0.74 [ 6:19:09<15:44:01] +[titan] 2025-07-09 19:54:11,353 - root - INFO - step: 28660 loss: 16.8644 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6214 global_avg_mtp_loss: 14.2430 +[titan] 2025-07-09 19:54:11,353 - root - INFO - lr: 2.5125e-04 gnorm: 0.84 [ 6:19:13<15:43:57] +[titan] 2025-07-09 19:54:15,276 - root - INFO - step: 28665 loss: 16.7110 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1168 +[titan] 2025-07-09 19:54:15,277 - root - INFO - lr: 2.5123e-04 gnorm: 0.83 [ 6:19:17<15:43:53] +[titan] 2025-07-09 19:54:19,264 - root - INFO - step: 28670 loss: 16.9704 memory: 44.58GiB(31.99%) tps: 82,172 tflops: 283.59 mfu: 28.67% global_avg_ntp_loss: 2.6460 global_avg_mtp_loss: 14.3244 +[titan] 2025-07-09 19:54:19,265 - root - INFO - lr: 2.5121e-04 gnorm: 0.73 [ 6:19:21<15:43:49] +[titan] 2025-07-09 19:54:20,993 - root - INFO - Dumping profiler traces at step 28672 +[titan] 2025-07-09 19:54:21,025 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 19:54:23,393 - root - INFO - step: 28675 loss: 16.9537 memory: 44.58GiB(31.99%) tps: 79,373 tflops: 273.93 mfu: 27.70% global_avg_ntp_loss: 2.6267 global_avg_mtp_loss: 14.3270 +[titan] 2025-07-09 19:54:23,394 - root - INFO - lr: 2.5120e-04 gnorm: 0.73 [ 6:19:25<15:43:45] +[titan] 2025-07-09 19:54:27,323 - root - INFO - step: 28680 loss: 16.6538 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0774 +[titan] 2025-07-09 19:54:27,323 - root - INFO - lr: 2.5118e-04 gnorm: 0.73 [ 6:19:29<15:43:41] +[titan] 2025-07-09 19:54:31,259 - root - INFO - step: 28685 loss: 16.5953 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0460 +[titan] 2025-07-09 19:54:31,259 - root - INFO - lr: 2.5117e-04 gnorm: 0.78 [ 6:19:33<15:43:37] +[titan] 2025-07-09 19:54:35,210 - root - INFO - step: 28690 loss: 16.7141 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.1325 +[titan] 2025-07-09 19:54:35,210 - root - INFO - lr: 2.5115e-04 gnorm: 0.78 [ 6:19:37<15:43:33] +[titan] 2025-07-09 19:54:39,120 - root - INFO - step: 28695 loss: 16.7973 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.1860 +[titan] 2025-07-09 19:54:39,120 - root - INFO - lr: 2.5113e-04 gnorm: 0.88 [ 6:19:41<15:43:29] +[titan] 2025-07-09 19:54:42,249 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:54:43,032 - root - INFO - step: 28700 loss: 16.9924 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.3573 +[titan] 2025-07-09 19:54:43,033 - root - INFO - lr: 2.5112e-04 gnorm: 0.75 [ 6:19:45<15:43:25] +[titan] 2025-07-09 19:54:46,974 - root - INFO - step: 28705 loss: 16.7362 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.5943 global_avg_mtp_loss: 14.1419 +[titan] 2025-07-09 19:54:46,975 - root - INFO - lr: 2.5110e-04 gnorm: 0.84 [ 6:19:48<15:43:21] +[titan] 2025-07-09 19:54:50,868 - root - INFO - step: 28710 loss: 16.6090 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0431 +[titan] 2025-07-09 19:54:50,868 - root - INFO - lr: 2.5108e-04 gnorm: 0.80 [ 6:19:52<15:43:17] +[titan] 2025-07-09 19:54:54,815 - root - INFO - step: 28715 loss: 16.7630 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.1643 +[titan] 2025-07-09 19:54:54,815 - root - INFO - lr: 2.5107e-04 gnorm: 0.85 [ 6:19:56<15:43:13] +[titan] 2025-07-09 19:54:58,749 - root - INFO - step: 28720 loss: 16.8024 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6052 global_avg_mtp_loss: 14.1972 +[titan] 2025-07-09 19:54:58,749 - root - INFO - lr: 2.5105e-04 gnorm: 0.79 [ 6:20:00<15:43:09] +[titan] 2025-07-09 19:55:02,677 - root - INFO - step: 28725 loss: 16.7582 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1667 +[titan] 2025-07-09 19:55:02,677 - root - INFO - lr: 2.5103e-04 gnorm: 0.78 [ 6:20:04<15:43:05] +[titan] 2025-07-09 19:55:06,618 - root - INFO - step: 28730 loss: 16.8144 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6180 global_avg_mtp_loss: 14.1964 +[titan] 2025-07-09 19:55:06,618 - root - INFO - lr: 2.5102e-04 gnorm: 0.82 [ 6:20:08<15:43:00] +[titan] 2025-07-09 19:55:10,508 - root - INFO - step: 28735 loss: 16.6789 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0997 +[titan] 2025-07-09 19:55:10,508 - root - INFO - lr: 2.5100e-04 gnorm: 0.79 [ 6:20:12<15:42:56] +[titan] 2025-07-09 19:55:14,451 - root - INFO - step: 28740 loss: 16.6247 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0555 +[titan] 2025-07-09 19:55:14,452 - root - INFO - lr: 2.5098e-04 gnorm: 0.74 [ 6:20:16<15:42:52] +[titan] 2025-07-09 19:55:18,363 - root - INFO - step: 28745 loss: 16.8040 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.2032 +[titan] 2025-07-09 19:55:18,363 - root - INFO - lr: 2.5097e-04 gnorm: 0.87 [ 6:20:20<15:42:48] +[titan] 2025-07-09 19:55:21,470 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:55:22,255 - root - INFO - step: 28750 loss: 17.0170 memory: 44.58GiB(31.99%) tps: 84,199 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.6423 global_avg_mtp_loss: 14.3747 +[titan] 2025-07-09 19:55:22,255 - root - INFO - lr: 2.5095e-04 gnorm: 0.75 [ 6:20:24<15:42:44] +[titan] 2025-07-09 19:55:26,174 - root - INFO - step: 28755 loss: 16.9396 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6235 global_avg_mtp_loss: 14.3161 +[titan] 2025-07-09 19:55:26,174 - root - INFO - lr: 2.5093e-04 gnorm: 0.73 [ 6:20:28<15:42:40] +[titan] 2025-07-09 19:55:30,081 - root - INFO - step: 28760 loss: 16.8611 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6095 global_avg_mtp_loss: 14.2517 +[titan] 2025-07-09 19:55:30,081 - root - INFO - lr: 2.5092e-04 gnorm: 0.84 [ 6:20:32<15:42:36] +[titan] 2025-07-09 19:55:33,977 - root - INFO - step: 28765 loss: 16.6756 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5792 global_avg_mtp_loss: 14.0964 +[titan] 2025-07-09 19:55:33,978 - root - INFO - lr: 2.5090e-04 gnorm: 0.81 [ 6:20:35<15:42:32] +[titan] 2025-07-09 19:55:37,878 - root - INFO - step: 28770 loss: 16.8892 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6147 global_avg_mtp_loss: 14.2745 +[titan] 2025-07-09 19:55:37,878 - root - INFO - lr: 2.5088e-04 gnorm: 0.79 [ 6:20:39<15:42:27] +[titan] 2025-07-09 19:55:41,826 - root - INFO - step: 28775 loss: 16.8338 memory: 44.58GiB(31.99%) tps: 83,017 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.6033 global_avg_mtp_loss: 14.2306 +[titan] 2025-07-09 19:55:41,826 - root - INFO - lr: 2.5087e-04 gnorm: 0.80 [ 6:20:43<15:42:23] +[titan] 2025-07-09 19:55:45,733 - root - INFO - step: 28780 loss: 17.1056 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6674 global_avg_mtp_loss: 14.4382 +[titan] 2025-07-09 19:55:45,733 - root - INFO - lr: 2.5085e-04 gnorm: 0.81 [ 6:20:47<15:42:19] +[titan] 2025-07-09 19:55:49,639 - root - INFO - step: 28785 loss: 17.0600 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6552 global_avg_mtp_loss: 14.4048 +[titan] 2025-07-09 19:55:49,640 - root - INFO - lr: 2.5084e-04 gnorm: 0.74 [ 6:20:51<15:42:15] +[titan] 2025-07-09 19:55:53,540 - root - INFO - step: 28790 loss: 16.7821 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.1631 +[titan] 2025-07-09 19:55:53,540 - root - INFO - lr: 2.5082e-04 gnorm: 0.76 [ 6:20:55<15:42:11] +[titan] 2025-07-09 19:55:57,499 - root - INFO - step: 28795 loss: 16.3030 memory: 44.58GiB(31.99%) tps: 82,771 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.7804 +[titan] 2025-07-09 19:55:57,500 - root - INFO - lr: 2.5080e-04 gnorm: 0.82 [ 6:20:59<15:42:07] +[titan] 2025-07-09 19:56:00,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:56:01,399 - root - INFO - step: 28800 loss: 17.0561 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.33% global_avg_ntp_loss: 2.6666 global_avg_mtp_loss: 14.3896 +[titan] 2025-07-09 19:56:01,399 - root - INFO - lr: 2.5079e-04 gnorm: 0.78 [ 6:21:03<15:42:03] +[titan] 2025-07-09 19:56:05,305 - root - INFO - step: 28805 loss: 16.7384 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1494 +[titan] 2025-07-09 19:56:05,305 - root - INFO - lr: 2.5077e-04 gnorm: 0.73 [ 6:21:07<15:41:59] +[titan] 2025-07-09 19:56:09,292 - root - INFO - step: 28810 loss: 16.7442 memory: 44.58GiB(31.99%) tps: 82,179 tflops: 283.61 mfu: 28.68% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.1553 +[titan] 2025-07-09 19:56:09,293 - root - INFO - lr: 2.5075e-04 gnorm: 0.95 [ 6:21:11<15:41:55] +[titan] 2025-07-09 19:56:13,200 - root - INFO - step: 28815 loss: 16.7215 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5997 global_avg_mtp_loss: 14.1218 +[titan] 2025-07-09 19:56:13,201 - root - INFO - lr: 2.5074e-04 gnorm: 0.75 [ 6:21:15<15:41:51] +[titan] 2025-07-09 19:56:17,097 - root - INFO - step: 28820 loss: 16.6604 memory: 44.58GiB(31.99%) tps: 84,109 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.0807 +[titan] 2025-07-09 19:56:17,097 - root - INFO - lr: 2.5072e-04 gnorm: 0.83 [ 6:21:19<15:41:47] +[titan] 2025-07-09 19:56:21,005 - root - INFO - step: 28825 loss: 16.7798 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.1774 +[titan] 2025-07-09 19:56:21,005 - root - INFO - lr: 2.5070e-04 gnorm: 0.75 [ 6:21:23<15:41:42] +[titan] 2025-07-09 19:56:24,904 - root - INFO - step: 28830 loss: 16.8553 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.2531 +[titan] 2025-07-09 19:56:24,904 - root - INFO - lr: 2.5069e-04 gnorm: 0.78 [ 6:21:26<15:41:38] +[titan] 2025-07-09 19:56:28,834 - root - INFO - step: 28835 loss: 16.8164 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6142 global_avg_mtp_loss: 14.2022 +[titan] 2025-07-09 19:56:28,835 - root - INFO - lr: 2.5067e-04 gnorm: 0.74 [ 6:21:30<15:41:34] +[titan] 2025-07-09 19:56:32,745 - root - INFO - step: 28840 loss: 16.7947 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.1881 +[titan] 2025-07-09 19:56:32,745 - root - INFO - lr: 2.5065e-04 gnorm: 0.74 [ 6:21:34<15:41:30] +[titan] 2025-07-09 19:56:36,652 - root - INFO - step: 28845 loss: 16.5614 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 13.9777 +[titan] 2025-07-09 19:56:36,652 - root - INFO - lr: 2.5064e-04 gnorm: 0.75 [ 6:21:38<15:41:26] +[titan] 2025-07-09 19:56:39,783 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:56:40,569 - root - INFO - step: 28850 loss: 16.6252 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0540 +[titan] 2025-07-09 19:56:40,570 - root - INFO - lr: 2.5062e-04 gnorm: 0.74 [ 6:21:42<15:41:22] +[titan] 2025-07-09 19:56:44,476 - root - INFO - step: 28855 loss: 17.1166 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6661 global_avg_mtp_loss: 14.4505 +[titan] 2025-07-09 19:56:44,476 - root - INFO - lr: 2.5060e-04 gnorm: 0.83 [ 6:21:46<15:41:18] +[titan] 2025-07-09 19:56:48,379 - root - INFO - step: 28860 loss: 16.6657 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.0921 +[titan] 2025-07-09 19:56:48,380 - root - INFO - lr: 2.5059e-04 gnorm: 0.76 [ 6:21:50<15:41:14] +[titan] 2025-07-09 19:56:52,285 - root - INFO - step: 28865 loss: 16.8224 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.2216 +[titan] 2025-07-09 19:56:52,285 - root - INFO - lr: 2.5057e-04 gnorm: 0.76 [ 6:21:54<15:41:10] +[titan] 2025-07-09 19:56:56,184 - root - INFO - step: 28870 loss: 16.8442 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.2321 +[titan] 2025-07-09 19:56:56,184 - root - INFO - lr: 2.5055e-04 gnorm: 0.82 [ 6:21:58<15:41:05] +[titan] 2025-07-09 19:57:00,112 - root - INFO - step: 28875 loss: 16.9340 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.3022 +[titan] 2025-07-09 19:57:00,112 - root - INFO - lr: 2.5054e-04 gnorm: 0.74 [ 6:22:02<15:41:01] +[titan] 2025-07-09 19:57:04,053 - root - INFO - step: 28880 loss: 16.9017 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2795 +[titan] 2025-07-09 19:57:04,053 - root - INFO - lr: 2.5052e-04 gnorm: 0.70 [ 6:22:06<15:40:57] +[titan] 2025-07-09 19:57:07,975 - root - INFO - step: 28885 loss: 16.9707 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6348 global_avg_mtp_loss: 14.3359 +[titan] 2025-07-09 19:57:07,975 - root - INFO - lr: 2.5050e-04 gnorm: 0.73 [ 6:22:09<15:40:53] +[titan] 2025-07-09 19:57:11,881 - root - INFO - step: 28890 loss: 16.6667 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.0818 +[titan] 2025-07-09 19:57:11,882 - root - INFO - lr: 2.5049e-04 gnorm: 0.83 [ 6:22:13<15:40:49] +[titan] 2025-07-09 19:57:15,795 - root - INFO - step: 28895 loss: 16.7313 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1395 +[titan] 2025-07-09 19:57:15,796 - root - INFO - lr: 2.5047e-04 gnorm: 0.84 [ 6:22:17<15:40:45] +[titan] 2025-07-09 19:57:18,919 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:57:19,708 - root - INFO - step: 28900 loss: 16.6555 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.0691 +[titan] 2025-07-09 19:57:19,708 - root - INFO - lr: 2.5045e-04 gnorm: 0.79 [ 6:22:21<15:40:41] +[titan] 2025-07-09 19:57:23,624 - root - INFO - step: 28905 loss: 16.8583 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6129 global_avg_mtp_loss: 14.2454 +[titan] 2025-07-09 19:57:23,624 - root - INFO - lr: 2.5044e-04 gnorm: 0.79 [ 6:22:25<15:40:37] +[titan] 2025-07-09 19:57:27,573 - root - INFO - step: 28910 loss: 16.6681 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.0879 +[titan] 2025-07-09 19:57:27,573 - root - INFO - lr: 2.5042e-04 gnorm: 0.77 [ 6:22:29<15:40:33] +[titan] 2025-07-09 19:57:31,523 - root - INFO - step: 28915 loss: 17.0057 memory: 44.58GiB(31.99%) tps: 82,952 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.6546 global_avg_mtp_loss: 14.3511 +[titan] 2025-07-09 19:57:31,524 - root - INFO - lr: 2.5040e-04 gnorm: 0.75 [ 6:22:33<15:40:29] +[titan] 2025-07-09 19:57:35,442 - root - INFO - step: 28920 loss: 16.9510 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.3234 +[titan] 2025-07-09 19:57:35,442 - root - INFO - lr: 2.5039e-04 gnorm: 0.75 [ 6:22:37<15:40:25] +[titan] 2025-07-09 19:57:39,360 - root - INFO - step: 28925 loss: 16.9372 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.6230 global_avg_mtp_loss: 14.3141 +[titan] 2025-07-09 19:57:39,361 - root - INFO - lr: 2.5037e-04 gnorm: 0.82 [ 6:22:41<15:40:21] +[titan] 2025-07-09 19:57:43,311 - root - INFO - step: 28930 loss: 17.0494 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.3980 +[titan] 2025-07-09 19:57:43,311 - root - INFO - lr: 2.5035e-04 gnorm: 0.78 [ 6:22:45<15:40:17] +[titan] 2025-07-09 19:57:47,235 - root - INFO - step: 28935 loss: 16.7931 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6089 global_avg_mtp_loss: 14.1842 +[titan] 2025-07-09 19:57:47,235 - root - INFO - lr: 2.5034e-04 gnorm: 0.83 [ 6:22:49<15:40:12] +[titan] 2025-07-09 19:57:51,136 - root - INFO - step: 28940 loss: 16.7308 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 14.1560 +[titan] 2025-07-09 19:57:51,137 - root - INFO - lr: 2.5032e-04 gnorm: 0.79 [ 6:22:53<15:40:08] +[titan] 2025-07-09 19:57:55,040 - root - INFO - step: 28945 loss: 16.9579 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6306 global_avg_mtp_loss: 14.3274 +[titan] 2025-07-09 19:57:55,040 - root - INFO - lr: 2.5030e-04 gnorm: 0.81 [ 6:22:57<15:40:04] +[titan] 2025-07-09 19:57:58,163 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:57:58,953 - root - INFO - step: 28950 loss: 16.8012 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.1987 +[titan] 2025-07-09 19:57:58,954 - root - INFO - lr: 2.5029e-04 gnorm: 0.79 [ 6:23:00<15:40:00] +[titan] 2025-07-09 19:58:02,863 - root - INFO - step: 28955 loss: 16.9263 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.2892 +[titan] 2025-07-09 19:58:02,863 - root - INFO - lr: 2.5027e-04 gnorm: 0.76 [ 6:23:04<15:39:56] +[titan] 2025-07-09 19:58:06,767 - root - INFO - step: 28960 loss: 16.7857 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.1841 +[titan] 2025-07-09 19:58:06,767 - root - INFO - lr: 2.5026e-04 gnorm: 0.75 [ 6:23:08<15:39:52] +[titan] 2025-07-09 19:58:10,702 - root - INFO - step: 28965 loss: 16.6173 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0495 +[titan] 2025-07-09 19:58:10,702 - root - INFO - lr: 2.5024e-04 gnorm: 0.76 [ 6:23:12<15:39:48] +[titan] 2025-07-09 19:58:14,597 - root - INFO - step: 28970 loss: 17.0084 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.6463 global_avg_mtp_loss: 14.3621 +[titan] 2025-07-09 19:58:14,597 - root - INFO - lr: 2.5022e-04 gnorm: 0.73 [ 6:23:16<15:39:44] +[titan] 2025-07-09 19:58:18,533 - root - INFO - step: 28975 loss: 16.7922 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.1945 +[titan] 2025-07-09 19:58:18,533 - root - INFO - lr: 2.5021e-04 gnorm: 0.80 [ 6:23:20<15:39:40] +[titan] 2025-07-09 19:58:22,452 - root - INFO - step: 28980 loss: 16.8037 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6100 global_avg_mtp_loss: 14.1938 +[titan] 2025-07-09 19:58:22,452 - root - INFO - lr: 2.5019e-04 gnorm: 0.79 [ 6:23:24<15:39:35] +[titan] 2025-07-09 19:58:26,378 - root - INFO - step: 28985 loss: 16.8205 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6256 global_avg_mtp_loss: 14.1950 +[titan] 2025-07-09 19:58:26,378 - root - INFO - lr: 2.5017e-04 gnorm: 0.76 [ 6:23:28<15:39:31] +[titan] 2025-07-09 19:58:30,293 - root - INFO - step: 28990 loss: 16.8465 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2206 +[titan] 2025-07-09 19:58:30,294 - root - INFO - lr: 2.5016e-04 gnorm: 0.75 [ 6:23:32<15:39:27] +[titan] 2025-07-09 19:58:34,234 - root - INFO - step: 28995 loss: 16.6941 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.0997 +[titan] 2025-07-09 19:58:34,234 - root - INFO - lr: 2.5014e-04 gnorm: 0.86 [ 6:23:36<15:39:23] +[titan] 2025-07-09 19:58:37,354 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:58:38,141 - root - INFO - step: 29000 loss: 16.6762 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5877 global_avg_mtp_loss: 14.0885 +[titan] 2025-07-09 19:58:38,141 - root - INFO - lr: 2.5012e-04 gnorm: 0.84 [ 6:23:40<15:39:19] +[titan] 2025-07-09 19:58:42,049 - root - INFO - step: 29005 loss: 16.6992 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1143 +[titan] 2025-07-09 19:58:42,049 - root - INFO - lr: 2.5011e-04 gnorm: 0.79 [ 6:23:44<15:39:15] +[titan] 2025-07-09 19:58:46,013 - root - INFO - step: 29010 loss: 16.7888 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.6104 global_avg_mtp_loss: 14.1784 +[titan] 2025-07-09 19:58:46,013 - root - INFO - lr: 2.5009e-04 gnorm: 0.74 [ 6:23:48<15:39:11] +[titan] 2025-07-09 19:58:49,969 - root - INFO - step: 29015 loss: 16.7185 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1290 +[titan] 2025-07-09 19:58:49,969 - root - INFO - lr: 2.5007e-04 gnorm: 0.69 [ 6:23:51<15:39:07] +[titan] 2025-07-09 19:58:53,908 - root - INFO - step: 29020 loss: 16.8527 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.6077 global_avg_mtp_loss: 14.2451 +[titan] 2025-07-09 19:58:53,908 - root - INFO - lr: 2.5006e-04 gnorm: 0.76 [ 6:23:55<15:39:03] +[titan] 2025-07-09 19:58:57,814 - root - INFO - step: 29025 loss: 16.8854 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2631 +[titan] 2025-07-09 19:58:57,814 - root - INFO - lr: 2.5004e-04 gnorm: 0.75 [ 6:23:59<15:38:59] +[titan] 2025-07-09 19:59:01,716 - root - INFO - step: 29030 loss: 16.6732 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0959 +[titan] 2025-07-09 19:59:01,716 - root - INFO - lr: 2.5002e-04 gnorm: 0.78 [ 6:24:03<15:38:55] +[titan] 2025-07-09 19:59:05,651 - root - INFO - step: 29035 loss: 16.7164 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.6117 global_avg_mtp_loss: 14.1047 +[titan] 2025-07-09 19:59:05,651 - root - INFO - lr: 2.5001e-04 gnorm: 0.76 [ 6:24:07<15:38:51] +[titan] 2025-07-09 19:59:09,586 - root - INFO - step: 29040 loss: 16.8181 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.2187 +[titan] 2025-07-09 19:59:09,586 - root - INFO - lr: 2.4999e-04 gnorm: 0.74 [ 6:24:11<15:38:47] +[titan] 2025-07-09 19:59:13,499 - root - INFO - step: 29045 loss: 16.7841 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1951 +[titan] 2025-07-09 19:59:13,500 - root - INFO - lr: 2.4997e-04 gnorm: 0.81 [ 6:24:15<15:38:43] +[titan] 2025-07-09 19:59:16,625 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:59:17,412 - root - INFO - step: 29050 loss: 16.7660 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1647 +[titan] 2025-07-09 19:59:17,412 - root - INFO - lr: 2.4996e-04 gnorm: 0.82 [ 6:24:19<15:38:38] +[titan] 2025-07-09 19:59:21,356 - root - INFO - step: 29055 loss: 16.8494 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.6120 global_avg_mtp_loss: 14.2374 +[titan] 2025-07-09 19:59:21,356 - root - INFO - lr: 2.4994e-04 gnorm: 0.76 [ 6:24:23<15:38:34] +[titan] 2025-07-09 19:59:25,281 - root - INFO - step: 29060 loss: 16.6836 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.0838 +[titan] 2025-07-09 19:59:25,281 - root - INFO - lr: 2.4992e-04 gnorm: 0.99 [ 6:24:27<15:38:30] +[titan] 2025-07-09 19:59:29,214 - root - INFO - step: 29065 loss: 16.8722 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2585 +[titan] 2025-07-09 19:59:29,214 - root - INFO - lr: 2.4991e-04 gnorm: 0.74 [ 6:24:31<15:38:26] +[titan] 2025-07-09 19:59:33,175 - root - INFO - step: 29070 loss: 16.9034 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6303 global_avg_mtp_loss: 14.2731 +[titan] 2025-07-09 19:59:33,175 - root - INFO - lr: 2.4989e-04 gnorm: 0.76 [ 6:24:35<15:38:22] +[titan] 2025-07-09 19:59:37,116 - root - INFO - step: 29075 loss: 16.8311 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.6145 global_avg_mtp_loss: 14.2165 +[titan] 2025-07-09 19:59:37,116 - root - INFO - lr: 2.4987e-04 gnorm: 0.77 [ 6:24:39<15:38:18] +[titan] 2025-07-09 19:59:41,030 - root - INFO - step: 29080 loss: 16.7387 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1484 +[titan] 2025-07-09 19:59:41,031 - root - INFO - lr: 2.4986e-04 gnorm: 0.77 [ 6:24:43<15:38:14] +[titan] 2025-07-09 19:59:44,959 - root - INFO - step: 29085 loss: 16.5874 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5627 global_avg_mtp_loss: 14.0247 +[titan] 2025-07-09 19:59:44,959 - root - INFO - lr: 2.4984e-04 gnorm: 1.61 [ 6:24:46<15:38:10] +[titan] 2025-07-09 19:59:48,878 - root - INFO - step: 29090 loss: 16.7489 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.1656 +[titan] 2025-07-09 19:59:48,878 - root - INFO - lr: 2.4982e-04 gnorm: 0.77 [ 6:24:50<15:38:06] +[titan] 2025-07-09 19:59:52,783 - root - INFO - step: 29095 loss: 17.0569 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6519 global_avg_mtp_loss: 14.4050 +[titan] 2025-07-09 19:59:52,784 - root - INFO - lr: 2.4981e-04 gnorm: 0.83 [ 6:24:54<15:38:02] +[titan] 2025-07-09 19:59:55,943 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 19:59:56,740 - root - INFO - step: 29100 loss: 17.0392 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.6458 global_avg_mtp_loss: 14.3934 +[titan] 2025-07-09 19:59:56,740 - root - INFO - lr: 2.4979e-04 gnorm: 0.78 [ 6:24:58<15:37:58] +[titan] 2025-07-09 20:00:00,672 - root - INFO - step: 29105 loss: 16.8600 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.2394 +[titan] 2025-07-09 20:00:00,672 - root - INFO - lr: 2.4977e-04 gnorm: 0.74 [ 6:25:02<15:37:54] +[titan] 2025-07-09 20:00:04,571 - root - INFO - step: 29110 loss: 16.8713 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6037 global_avg_mtp_loss: 14.2676 +[titan] 2025-07-09 20:00:04,572 - root - INFO - lr: 2.4976e-04 gnorm: 0.79 [ 6:25:06<15:37:50] +[titan] 2025-07-09 20:00:08,490 - root - INFO - step: 29115 loss: 16.7930 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1964 +[titan] 2025-07-09 20:00:08,491 - root - INFO - lr: 2.4974e-04 gnorm: 0.71 [ 6:25:10<15:37:46] +[titan] 2025-07-09 20:00:12,411 - root - INFO - step: 29120 loss: 16.6497 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0856 +[titan] 2025-07-09 20:00:12,411 - root - INFO - lr: 2.4972e-04 gnorm: 0.74 [ 6:25:14<15:37:41] +[titan] 2025-07-09 20:00:16,309 - root - INFO - step: 29125 loss: 16.7339 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.1316 +[titan] 2025-07-09 20:00:16,310 - root - INFO - lr: 2.4971e-04 gnorm: 0.76 [ 6:25:18<15:37:37] +[titan] 2025-07-09 20:00:20,229 - root - INFO - step: 29130 loss: 16.6058 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.0328 +[titan] 2025-07-09 20:00:20,229 - root - INFO - lr: 2.4969e-04 gnorm: 0.72 [ 6:25:22<15:37:33] +[titan] 2025-07-09 20:00:24,158 - root - INFO - step: 29135 loss: 16.7732 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1812 +[titan] 2025-07-09 20:00:24,158 - root - INFO - lr: 2.4967e-04 gnorm: 0.78 [ 6:25:26<15:37:29] +[titan] 2025-07-09 20:00:28,051 - root - INFO - step: 29140 loss: 16.7031 memory: 44.58GiB(31.99%) tps: 84,182 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1223 +[titan] 2025-07-09 20:00:28,051 - root - INFO - lr: 2.4966e-04 gnorm: 0.74 [ 6:25:30<15:37:25] +[titan] 2025-07-09 20:00:31,979 - root - INFO - step: 29145 loss: 16.8415 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.2255 +[titan] 2025-07-09 20:00:31,980 - root - INFO - lr: 2.4964e-04 gnorm: 0.71 [ 6:25:33<15:37:21] +[titan] 2025-07-09 20:00:35,115 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:00:35,916 - root - INFO - step: 29150 loss: 16.7089 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.1193 +[titan] 2025-07-09 20:00:35,916 - root - INFO - lr: 2.4962e-04 gnorm: 0.78 [ 6:25:37<15:37:17] +[titan] 2025-07-09 20:00:39,862 - root - INFO - step: 29155 loss: 16.7341 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 20:00:39,862 - root - INFO - lr: 2.4961e-04 gnorm: 0.73 [ 6:25:41<15:37:13] +[titan] 2025-07-09 20:00:43,772 - root - INFO - step: 29160 loss: 16.8686 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6095 global_avg_mtp_loss: 14.2591 +[titan] 2025-07-09 20:00:43,772 - root - INFO - lr: 2.4959e-04 gnorm: 0.75 [ 6:25:45<15:37:09] +[titan] 2025-07-09 20:00:47,684 - root - INFO - step: 29165 loss: 16.9084 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.2774 +[titan] 2025-07-09 20:00:47,685 - root - INFO - lr: 2.4957e-04 gnorm: 0.79 [ 6:25:49<15:37:05] +[titan] 2025-07-09 20:00:51,616 - root - INFO - step: 29170 loss: 16.8139 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2056 +[titan] 2025-07-09 20:00:51,616 - root - INFO - lr: 2.4956e-04 gnorm: 0.79 [ 6:25:53<15:37:01] +[titan] 2025-07-09 20:00:55,551 - root - INFO - step: 29175 loss: 16.7338 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1454 +[titan] 2025-07-09 20:00:55,551 - root - INFO - lr: 2.4954e-04 gnorm: 0.77 [ 6:25:57<15:36:57] +[titan] 2025-07-09 20:00:59,459 - root - INFO - step: 29180 loss: 16.9197 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.2950 +[titan] 2025-07-09 20:00:59,459 - root - INFO - lr: 2.4952e-04 gnorm: 0.76 [ 6:26:01<15:36:52] +[titan] 2025-07-09 20:01:02,816 - root - INFO - Dumping profiler traces at step 29184 +[titan] 2025-07-09 20:01:02,848 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:01:03,654 - root - INFO - step: 29185 loss: 16.7638 memory: 44.58GiB(31.99%) tps: 78,118 tflops: 269.60 mfu: 27.26% global_avg_ntp_loss: 2.5943 global_avg_mtp_loss: 14.1695 +[titan] 2025-07-09 20:01:03,654 - root - INFO - lr: 2.4951e-04 gnorm: 0.82 [ 6:26:05<15:36:49] +[titan] 2025-07-09 20:01:07,562 - root - INFO - step: 29190 loss: 16.7707 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6049 global_avg_mtp_loss: 14.1659 +[titan] 2025-07-09 20:01:07,562 - root - INFO - lr: 2.4949e-04 gnorm: 0.82 [ 6:26:09<15:36:45] +[titan] 2025-07-09 20:01:11,521 - root - INFO - step: 29195 loss: 16.9111 memory: 44.58GiB(31.99%) tps: 82,767 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.6201 global_avg_mtp_loss: 14.2909 +[titan] 2025-07-09 20:01:11,521 - root - INFO - lr: 2.4947e-04 gnorm: 0.80 [ 6:26:13<15:36:41] +[titan] 2025-07-09 20:01:14,674 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:01:15,460 - root - INFO - step: 29200 loss: 16.9804 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.3472 +[titan] 2025-07-09 20:01:15,461 - root - INFO - lr: 2.4946e-04 gnorm: 0.78 [ 6:26:17<15:36:37] +[titan] 2025-07-09 20:01:19,365 - root - INFO - step: 29205 loss: 16.5701 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 14.0138 +[titan] 2025-07-09 20:01:19,365 - root - INFO - lr: 2.4944e-04 gnorm: 0.73 [ 6:26:21<15:36:33] +[titan] 2025-07-09 20:01:23,288 - root - INFO - step: 29210 loss: 16.7546 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1654 +[titan] 2025-07-09 20:01:23,288 - root - INFO - lr: 2.4942e-04 gnorm: 0.74 [ 6:26:25<15:36:29] +[titan] 2025-07-09 20:01:27,189 - root - INFO - step: 29215 loss: 16.5782 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0039 +[titan] 2025-07-09 20:01:27,189 - root - INFO - lr: 2.4941e-04 gnorm: 0.77 [ 6:26:29<15:36:25] +[titan] 2025-07-09 20:01:31,090 - root - INFO - step: 29220 loss: 16.6714 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.0917 +[titan] 2025-07-09 20:01:31,091 - root - INFO - lr: 2.4939e-04 gnorm: 0.84 [ 6:26:33<15:36:20] +[titan] 2025-07-09 20:01:35,037 - root - INFO - step: 29225 loss: 16.8904 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.6122 global_avg_mtp_loss: 14.2782 +[titan] 2025-07-09 20:01:35,037 - root - INFO - lr: 2.4937e-04 gnorm: 0.76 [ 6:26:37<15:36:16] +[titan] 2025-07-09 20:01:38,958 - root - INFO - step: 29230 loss: 16.9625 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.3275 +[titan] 2025-07-09 20:01:38,958 - root - INFO - lr: 2.4936e-04 gnorm: 0.76 [ 6:26:40<15:36:12] +[titan] 2025-07-09 20:01:42,875 - root - INFO - step: 29235 loss: 17.1202 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6880 global_avg_mtp_loss: 14.4322 +[titan] 2025-07-09 20:01:42,875 - root - INFO - lr: 2.4934e-04 gnorm: 0.77 [ 6:26:44<15:36:08] +[titan] 2025-07-09 20:01:46,820 - root - INFO - step: 29240 loss: 16.9670 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.6341 global_avg_mtp_loss: 14.3329 +[titan] 2025-07-09 20:01:46,820 - root - INFO - lr: 2.4932e-04 gnorm: 0.79 [ 6:26:48<15:36:04] +[titan] 2025-07-09 20:01:50,741 - root - INFO - step: 29245 loss: 16.7559 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.1455 +[titan] 2025-07-09 20:01:50,741 - root - INFO - lr: 2.4931e-04 gnorm: 0.81 [ 6:26:52<15:36:00] +[titan] 2025-07-09 20:01:53,878 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:01:54,674 - root - INFO - step: 29250 loss: 16.9194 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.3041 +[titan] 2025-07-09 20:01:54,674 - root - INFO - lr: 2.4929e-04 gnorm: 0.79 [ 6:26:56<15:35:56] +[titan] 2025-07-09 20:01:58,631 - root - INFO - step: 29255 loss: 16.9726 memory: 44.58GiB(31.99%) tps: 82,817 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.6374 global_avg_mtp_loss: 14.3352 +[titan] 2025-07-09 20:01:58,631 - root - INFO - lr: 2.4927e-04 gnorm: 0.73 [ 6:27:00<15:35:52] +[titan] 2025-07-09 20:02:02,650 - root - INFO - step: 29260 loss: 16.7888 memory: 44.58GiB(31.99%) tps: 81,531 tflops: 281.38 mfu: 28.45% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.1842 +[titan] 2025-07-09 20:02:02,651 - root - INFO - lr: 2.4925e-04 gnorm: 0.76 [ 6:27:04<15:35:48] +[titan] 2025-07-09 20:02:06,576 - root - INFO - step: 29265 loss: 16.7240 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.1399 +[titan] 2025-07-09 20:02:06,576 - root - INFO - lr: 2.4924e-04 gnorm: 0.73 [ 6:27:08<15:35:44] +[titan] 2025-07-09 20:02:10,474 - root - INFO - step: 29270 loss: 16.7401 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.1453 +[titan] 2025-07-09 20:02:10,474 - root - INFO - lr: 2.4922e-04 gnorm: 0.78 [ 6:27:12<15:35:40] +[titan] 2025-07-09 20:02:14,371 - root - INFO - step: 29275 loss: 17.0122 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6438 global_avg_mtp_loss: 14.3685 +[titan] 2025-07-09 20:02:14,372 - root - INFO - lr: 2.4920e-04 gnorm: 0.82 [ 6:27:16<15:35:36] +[titan] 2025-07-09 20:02:18,283 - root - INFO - step: 29280 loss: 16.9090 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6336 global_avg_mtp_loss: 14.2754 +[titan] 2025-07-09 20:02:18,284 - root - INFO - lr: 2.4919e-04 gnorm: 0.83 [ 6:27:20<15:35:32] +[titan] 2025-07-09 20:02:22,185 - root - INFO - step: 29285 loss: 16.7263 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1483 +[titan] 2025-07-09 20:02:22,185 - root - INFO - lr: 2.4917e-04 gnorm: 0.78 [ 6:27:24<15:35:28] +[titan] 2025-07-09 20:02:26,101 - root - INFO - step: 29290 loss: 16.9716 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6297 global_avg_mtp_loss: 14.3418 +[titan] 2025-07-09 20:02:26,101 - root - INFO - lr: 2.4915e-04 gnorm: 0.82 [ 6:27:28<15:35:23] +[titan] 2025-07-09 20:02:30,036 - root - INFO - step: 29295 loss: 16.9999 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.3596 +[titan] 2025-07-09 20:02:30,036 - root - INFO - lr: 2.4914e-04 gnorm: 0.77 [ 6:27:32<15:35:19] +[titan] 2025-07-09 20:02:33,151 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:02:33,942 - root - INFO - step: 29300 loss: 16.8863 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.2654 +[titan] 2025-07-09 20:02:33,942 - root - INFO - lr: 2.4912e-04 gnorm: 0.78 [ 6:27:35<15:35:15] +[titan] 2025-07-09 20:02:37,862 - root - INFO - step: 29305 loss: 16.7904 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.6085 global_avg_mtp_loss: 14.1819 +[titan] 2025-07-09 20:02:37,862 - root - INFO - lr: 2.4910e-04 gnorm: 0.83 [ 6:27:39<15:35:11] +[titan] 2025-07-09 20:02:41,788 - root - INFO - step: 29310 loss: 16.8244 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.2237 +[titan] 2025-07-09 20:02:41,789 - root - INFO - lr: 2.4909e-04 gnorm: 0.74 [ 6:27:43<15:35:07] +[titan] 2025-07-09 20:02:45,743 - root - INFO - step: 29315 loss: 16.8103 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.2050 +[titan] 2025-07-09 20:02:45,744 - root - INFO - lr: 2.4907e-04 gnorm: 0.70 [ 6:27:47<15:35:03] +[titan] 2025-07-09 20:02:49,644 - root - INFO - step: 29320 loss: 16.6873 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-09 20:02:49,644 - root - INFO - lr: 2.4905e-04 gnorm: 0.81 [ 6:27:51<15:34:59] +[titan] 2025-07-09 20:02:53,536 - root - INFO - step: 29325 loss: 16.7898 memory: 44.58GiB(31.99%) tps: 84,203 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.1830 +[titan] 2025-07-09 20:02:53,536 - root - INFO - lr: 2.4904e-04 gnorm: 0.77 [ 6:27:55<15:34:55] +[titan] 2025-07-09 20:02:57,477 - root - INFO - step: 29330 loss: 16.8789 memory: 44.58GiB(31.99%) tps: 83,147 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.2676 +[titan] 2025-07-09 20:02:57,477 - root - INFO - lr: 2.4902e-04 gnorm: 0.74 [ 6:27:59<15:34:51] +[titan] 2025-07-09 20:03:01,397 - root - INFO - step: 29335 loss: 16.6757 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.1014 +[titan] 2025-07-09 20:03:01,397 - root - INFO - lr: 2.4900e-04 gnorm: 0.76 [ 6:28:03<15:34:47] +[titan] 2025-07-09 20:03:05,332 - root - INFO - step: 29340 loss: 16.7540 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.1523 +[titan] 2025-07-09 20:03:05,332 - root - INFO - lr: 2.4899e-04 gnorm: 0.72 [ 6:28:07<15:34:43] +[titan] 2025-07-09 20:03:09,260 - root - INFO - step: 29345 loss: 17.0015 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6434 global_avg_mtp_loss: 14.3582 +[titan] 2025-07-09 20:03:09,260 - root - INFO - lr: 2.4897e-04 gnorm: 0.85 [ 6:28:11<15:34:39] +[titan] 2025-07-09 20:03:12,384 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:03:13,172 - root - INFO - step: 29350 loss: 16.8112 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6021 global_avg_mtp_loss: 14.2091 +[titan] 2025-07-09 20:03:13,172 - root - INFO - lr: 2.4895e-04 gnorm: 0.84 [ 6:28:15<15:34:35] +[titan] 2025-07-09 20:03:17,110 - root - INFO - step: 29355 loss: 16.8971 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6228 global_avg_mtp_loss: 14.2743 +[titan] 2025-07-09 20:03:17,110 - root - INFO - lr: 2.4894e-04 gnorm: 0.84 [ 6:28:19<15:34:30] +[titan] 2025-07-09 20:03:21,044 - root - INFO - step: 29360 loss: 16.8440 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.6112 global_avg_mtp_loss: 14.2328 +[titan] 2025-07-09 20:03:21,044 - root - INFO - lr: 2.4892e-04 gnorm: 0.82 [ 6:28:23<15:34:26] +[titan] 2025-07-09 20:03:24,977 - root - INFO - step: 29365 loss: 16.8880 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.6294 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 20:03:24,977 - root - INFO - lr: 2.4890e-04 gnorm: 0.76 [ 6:28:26<15:34:22] +[titan] 2025-07-09 20:03:28,937 - root - INFO - step: 29370 loss: 16.8594 memory: 44.58GiB(31.99%) tps: 82,746 tflops: 285.57 mfu: 28.87% global_avg_ntp_loss: 2.6214 global_avg_mtp_loss: 14.2380 +[titan] 2025-07-09 20:03:28,938 - root - INFO - lr: 2.4889e-04 gnorm: 0.80 [ 6:28:30<15:34:18] +[titan] 2025-07-09 20:03:32,856 - root - INFO - step: 29375 loss: 16.8409 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.2299 +[titan] 2025-07-09 20:03:32,856 - root - INFO - lr: 2.4887e-04 gnorm: 0.78 [ 6:28:34<15:34:14] +[titan] 2025-07-09 20:03:36,812 - root - INFO - step: 29380 loss: 16.6976 memory: 44.58GiB(31.99%) tps: 82,840 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1071 +[titan] 2025-07-09 20:03:36,812 - root - INFO - lr: 2.4885e-04 gnorm: 0.72 [ 6:28:38<15:34:10] +[titan] 2025-07-09 20:03:40,736 - root - INFO - step: 29385 loss: 16.8195 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2128 +[titan] 2025-07-09 20:03:40,736 - root - INFO - lr: 2.4884e-04 gnorm: 0.76 [ 6:28:42<15:34:06] +[titan] 2025-07-09 20:03:44,688 - root - INFO - step: 29390 loss: 16.7407 memory: 44.58GiB(31.99%) tps: 82,923 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.5878 global_avg_mtp_loss: 14.1530 +[titan] 2025-07-09 20:03:44,688 - root - INFO - lr: 2.4882e-04 gnorm: 0.81 [ 6:28:46<15:34:02] +[titan] 2025-07-09 20:03:48,632 - root - INFO - step: 29395 loss: 16.7899 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.1839 +[titan] 2025-07-09 20:03:48,632 - root - INFO - lr: 2.4880e-04 gnorm: 0.78 [ 6:28:50<15:33:58] +[titan] 2025-07-09 20:03:51,768 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:03:52,570 - root - INFO - step: 29400 loss: 16.8711 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.2578 +[titan] 2025-07-09 20:03:52,570 - root - INFO - lr: 2.4879e-04 gnorm: 0.74 [ 6:28:54<15:33:54] +[titan] 2025-07-09 20:03:56,490 - root - INFO - step: 29405 loss: 16.6217 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5759 global_avg_mtp_loss: 14.0458 +[titan] 2025-07-09 20:03:56,490 - root - INFO - lr: 2.4877e-04 gnorm: 0.84 [ 6:28:58<15:33:50] +[titan] 2025-07-09 20:04:00,428 - root - INFO - step: 29410 loss: 16.9601 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.3322 +[titan] 2025-07-09 20:04:00,429 - root - INFO - lr: 2.4875e-04 gnorm: 0.75 [ 6:29:02<15:33:46] +[titan] 2025-07-09 20:04:04,374 - root - INFO - step: 29415 loss: 17.2096 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.6854 global_avg_mtp_loss: 14.5242 +[titan] 2025-07-09 20:04:04,374 - root - INFO - lr: 2.4873e-04 gnorm: 0.81 [ 6:29:06<15:33:42] +[titan] 2025-07-09 20:04:08,281 - root - INFO - step: 29420 loss: 16.8426 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.2232 +[titan] 2025-07-09 20:04:08,282 - root - INFO - lr: 2.4872e-04 gnorm: 0.79 [ 6:29:10<15:33:38] +[titan] 2025-07-09 20:04:12,192 - root - INFO - step: 29425 loss: 16.7904 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.1863 +[titan] 2025-07-09 20:04:12,192 - root - INFO - lr: 2.4870e-04 gnorm: 0.77 [ 6:29:14<15:33:34] +[titan] 2025-07-09 20:04:16,119 - root - INFO - step: 29430 loss: 16.6585 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-09 20:04:16,119 - root - INFO - lr: 2.4868e-04 gnorm: 0.70 [ 6:29:18<15:33:30] +[titan] 2025-07-09 20:04:20,049 - root - INFO - step: 29435 loss: 16.8256 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.2130 +[titan] 2025-07-09 20:04:20,049 - root - INFO - lr: 2.4867e-04 gnorm: 0.78 [ 6:29:22<15:33:26] +[titan] 2025-07-09 20:04:23,949 - root - INFO - step: 29440 loss: 17.1464 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6751 global_avg_mtp_loss: 14.4712 +[titan] 2025-07-09 20:04:23,949 - root - INFO - lr: 2.4865e-04 gnorm: 0.80 [ 6:29:25<15:33:21] +[titan] 2025-07-09 20:04:27,899 - root - INFO - step: 29445 loss: 16.6581 memory: 44.58GiB(31.99%) tps: 82,966 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5806 global_avg_mtp_loss: 14.0775 +[titan] 2025-07-09 20:04:27,899 - root - INFO - lr: 2.4863e-04 gnorm: 0.78 [ 6:29:29<15:33:17] +[titan] 2025-07-09 20:04:31,046 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:04:31,856 - root - INFO - step: 29450 loss: 16.8158 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.6051 global_avg_mtp_loss: 14.2106 +[titan] 2025-07-09 20:04:31,856 - root - INFO - lr: 2.4862e-04 gnorm: 0.74 [ 6:29:33<15:33:13] +[titan] 2025-07-09 20:04:35,778 - root - INFO - step: 29455 loss: 16.5626 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-09 20:04:35,778 - root - INFO - lr: 2.4860e-04 gnorm: 0.77 [ 6:29:37<15:33:09] +[titan] 2025-07-09 20:04:39,684 - root - INFO - step: 29460 loss: 16.7317 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5877 global_avg_mtp_loss: 14.1440 +[titan] 2025-07-09 20:04:39,684 - root - INFO - lr: 2.4858e-04 gnorm: 0.76 [ 6:29:41<15:33:05] +[titan] 2025-07-09 20:04:43,642 - root - INFO - step: 29465 loss: 16.8321 memory: 44.58GiB(31.99%) tps: 82,807 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.2282 +[titan] 2025-07-09 20:04:43,642 - root - INFO - lr: 2.4857e-04 gnorm: 0.77 [ 6:29:45<15:33:01] +[titan] 2025-07-09 20:04:47,540 - root - INFO - step: 29470 loss: 16.6898 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.0956 +[titan] 2025-07-09 20:04:47,540 - root - INFO - lr: 2.4855e-04 gnorm: 0.81 [ 6:29:49<15:32:57] +[titan] 2025-07-09 20:04:51,440 - root - INFO - step: 29475 loss: 16.7961 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1947 +[titan] 2025-07-09 20:04:51,440 - root - INFO - lr: 2.4853e-04 gnorm: 0.81 [ 6:29:53<15:32:53] +[titan] 2025-07-09 20:04:55,354 - root - INFO - step: 29480 loss: 16.8246 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.2044 +[titan] 2025-07-09 20:04:55,354 - root - INFO - lr: 2.4852e-04 gnorm: 0.75 [ 6:29:57<15:32:49] +[titan] 2025-07-09 20:04:59,275 - root - INFO - step: 29485 loss: 16.6935 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1092 +[titan] 2025-07-09 20:04:59,276 - root - INFO - lr: 2.4850e-04 gnorm: 0.73 [ 6:30:01<15:32:45] +[titan] 2025-07-09 20:05:03,173 - root - INFO - step: 29490 loss: 16.7940 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1956 +[titan] 2025-07-09 20:05:03,173 - root - INFO - lr: 2.4848e-04 gnorm: 0.73 [ 6:30:05<15:32:41] +[titan] 2025-07-09 20:05:07,063 - root - INFO - step: 29495 loss: 16.6490 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.0779 +[titan] 2025-07-09 20:05:07,063 - root - INFO - lr: 2.4847e-04 gnorm: 0.77 [ 6:30:09<15:32:37] +[titan] 2025-07-09 20:05:10,205 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:05:10,992 - root - INFO - step: 29500 loss: 16.7871 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5980 global_avg_mtp_loss: 14.1891 +[titan] 2025-07-09 20:05:10,992 - root - INFO - lr: 2.4845e-04 gnorm: 0.74 [ 6:30:12<15:32:32] +[titan] 2025-07-09 20:05:14,926 - root - INFO - step: 29505 loss: 16.9293 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.3007 +[titan] 2025-07-09 20:05:14,926 - root - INFO - lr: 2.4843e-04 gnorm: 0.75 [ 6:30:16<15:32:28] +[titan] 2025-07-09 20:05:18,839 - root - INFO - step: 29510 loss: 16.9251 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6443 global_avg_mtp_loss: 14.2808 +[titan] 2025-07-09 20:05:18,839 - root - INFO - lr: 2.4842e-04 gnorm: 0.91 [ 6:30:20<15:32:24] +[titan] 2025-07-09 20:05:22,799 - root - INFO - step: 29515 loss: 16.8152 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.2103 +[titan] 2025-07-09 20:05:22,799 - root - INFO - lr: 2.4840e-04 gnorm: 0.73 [ 6:30:24<15:32:20] +[titan] 2025-07-09 20:05:26,804 - root - INFO - step: 29520 loss: 16.8313 memory: 44.58GiB(31.99%) tps: 81,821 tflops: 282.38 mfu: 28.55% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.2222 +[titan] 2025-07-09 20:05:26,805 - root - INFO - lr: 2.4838e-04 gnorm: 0.68 [ 6:30:28<15:32:16] +[titan] 2025-07-09 20:05:30,730 - root - INFO - step: 29525 loss: 16.6465 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0685 +[titan] 2025-07-09 20:05:30,731 - root - INFO - lr: 2.4836e-04 gnorm: 0.81 [ 6:30:32<15:32:12] +[titan] 2025-07-09 20:05:34,628 - root - INFO - step: 29530 loss: 16.8783 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.2478 +[titan] 2025-07-09 20:05:34,628 - root - INFO - lr: 2.4835e-04 gnorm: 0.73 [ 6:30:36<15:32:08] +[titan] 2025-07-09 20:05:38,530 - root - INFO - step: 29535 loss: 16.9702 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.6318 global_avg_mtp_loss: 14.3384 +[titan] 2025-07-09 20:05:38,530 - root - INFO - lr: 2.4833e-04 gnorm: 0.85 [ 6:30:40<15:32:04] +[titan] 2025-07-09 20:05:42,436 - root - INFO - step: 29540 loss: 16.8049 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.1943 +[titan] 2025-07-09 20:05:42,436 - root - INFO - lr: 2.4831e-04 gnorm: 0.77 [ 6:30:44<15:32:00] +[titan] 2025-07-09 20:05:46,347 - root - INFO - step: 29545 loss: 16.9083 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.2929 +[titan] 2025-07-09 20:05:46,347 - root - INFO - lr: 2.4830e-04 gnorm: 0.85 [ 6:30:48<15:31:56] +[titan] 2025-07-09 20:05:49,489 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:05:50,276 - root - INFO - step: 29550 loss: 16.8533 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6146 global_avg_mtp_loss: 14.2388 +[titan] 2025-07-09 20:05:50,276 - root - INFO - lr: 2.4828e-04 gnorm: 0.84 [ 6:30:52<15:31:52] +[titan] 2025-07-09 20:05:54,174 - root - INFO - step: 29555 loss: 16.7051 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1146 +[titan] 2025-07-09 20:05:54,174 - root - INFO - lr: 2.4826e-04 gnorm: 0.77 [ 6:30:56<15:31:48] +[titan] 2025-07-09 20:05:58,065 - root - INFO - step: 29560 loss: 16.8127 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.6201 global_avg_mtp_loss: 14.1925 +[titan] 2025-07-09 20:05:58,066 - root - INFO - lr: 2.4825e-04 gnorm: 0.79 [ 6:31:00<15:31:44] +[titan] 2025-07-09 20:06:01,970 - root - INFO - step: 29565 loss: 16.9094 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.2831 +[titan] 2025-07-09 20:06:01,971 - root - INFO - lr: 2.4823e-04 gnorm: 0.80 [ 6:31:03<15:31:39] +[titan] 2025-07-09 20:06:05,876 - root - INFO - step: 29570 loss: 16.9975 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.3610 +[titan] 2025-07-09 20:06:05,877 - root - INFO - lr: 2.4821e-04 gnorm: 0.80 [ 6:31:07<15:31:35] +[titan] 2025-07-09 20:06:09,773 - root - INFO - step: 29575 loss: 16.7805 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.1845 +[titan] 2025-07-09 20:06:09,773 - root - INFO - lr: 2.4820e-04 gnorm: 0.79 [ 6:31:11<15:31:31] +[titan] 2025-07-09 20:06:13,682 - root - INFO - step: 29580 loss: 16.7649 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.1739 +[titan] 2025-07-09 20:06:13,682 - root - INFO - lr: 2.4818e-04 gnorm: 0.74 [ 6:31:15<15:31:27] +[titan] 2025-07-09 20:06:17,591 - root - INFO - step: 29585 loss: 16.9878 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6321 global_avg_mtp_loss: 14.3556 +[titan] 2025-07-09 20:06:17,592 - root - INFO - lr: 2.4816e-04 gnorm: 0.73 [ 6:31:19<15:31:23] +[titan] 2025-07-09 20:06:21,491 - root - INFO - step: 29590 loss: 16.8584 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.2425 +[titan] 2025-07-09 20:06:21,491 - root - INFO - lr: 2.4815e-04 gnorm: 0.78 [ 6:31:23<15:31:19] +[titan] 2025-07-09 20:06:25,410 - root - INFO - step: 29595 loss: 17.0542 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6463 global_avg_mtp_loss: 14.4079 +[titan] 2025-07-09 20:06:25,410 - root - INFO - lr: 2.4813e-04 gnorm: 0.77 [ 6:31:27<15:31:15] +[titan] 2025-07-09 20:06:28,534 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:06:29,321 - root - INFO - step: 29600 loss: 16.8239 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6270 global_avg_mtp_loss: 14.1968 +[titan] 2025-07-09 20:06:29,321 - root - INFO - lr: 2.4811e-04 gnorm: 0.82 [ 6:31:31<15:31:11] +[titan] 2025-07-09 20:06:33,255 - root - INFO - step: 29605 loss: 16.6297 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.0395 +[titan] 2025-07-09 20:06:33,256 - root - INFO - lr: 2.4810e-04 gnorm: 0.80 [ 6:31:35<15:31:07] +[titan] 2025-07-09 20:06:37,194 - root - INFO - step: 29610 loss: 16.6963 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1116 +[titan] 2025-07-09 20:06:37,194 - root - INFO - lr: 2.4808e-04 gnorm: 0.78 [ 6:31:39<15:31:03] +[titan] 2025-07-09 20:06:41,128 - root - INFO - step: 29615 loss: 16.5358 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 13.9812 +[titan] 2025-07-09 20:06:41,128 - root - INFO - lr: 2.4806e-04 gnorm: 0.74 [ 6:31:43<15:30:58] +[titan] 2025-07-09 20:06:45,055 - root - INFO - step: 29620 loss: 16.8389 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.2283 +[titan] 2025-07-09 20:06:45,055 - root - INFO - lr: 2.4804e-04 gnorm: 0.74 [ 6:31:46<15:30:54] +[titan] 2025-07-09 20:06:48,962 - root - INFO - step: 29625 loss: 16.7177 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1256 +[titan] 2025-07-09 20:06:48,962 - root - INFO - lr: 2.4803e-04 gnorm: 0.76 [ 6:31:50<15:30:50] +[titan] 2025-07-09 20:06:52,893 - root - INFO - step: 29630 loss: 16.9453 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3059 +[titan] 2025-07-09 20:06:52,893 - root - INFO - lr: 2.4801e-04 gnorm: 0.73 [ 6:31:54<15:30:46] +[titan] 2025-07-09 20:06:56,847 - root - INFO - step: 29635 loss: 16.8473 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.2054 +[titan] 2025-07-09 20:06:56,847 - root - INFO - lr: 2.4799e-04 gnorm: 0.77 [ 6:31:58<15:30:42] +[titan] 2025-07-09 20:07:00,835 - root - INFO - step: 29640 loss: 16.5432 memory: 44.58GiB(31.99%) tps: 82,175 tflops: 283.60 mfu: 28.68% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 13.9976 +[titan] 2025-07-09 20:07:00,835 - root - INFO - lr: 2.4798e-04 gnorm: 0.79 [ 6:32:02<15:30:38] +[titan] 2025-07-09 20:07:04,766 - root - INFO - step: 29645 loss: 16.7848 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.6011 global_avg_mtp_loss: 14.1836 +[titan] 2025-07-09 20:07:04,767 - root - INFO - lr: 2.4796e-04 gnorm: 0.79 [ 6:32:06<15:30:34] +[titan] 2025-07-09 20:07:07,887 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:07:08,678 - root - INFO - step: 29650 loss: 16.8515 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6282 global_avg_mtp_loss: 14.2233 +[titan] 2025-07-09 20:07:08,678 - root - INFO - lr: 2.4794e-04 gnorm: 0.76 [ 6:32:10<15:30:30] +[titan] 2025-07-09 20:07:12,622 - root - INFO - step: 29655 loss: 16.8769 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6144 global_avg_mtp_loss: 14.2625 +[titan] 2025-07-09 20:07:12,622 - root - INFO - lr: 2.4793e-04 gnorm: 0.75 [ 6:32:14<15:30:26] +[titan] 2025-07-09 20:07:16,589 - root - INFO - step: 29660 loss: 17.0078 memory: 44.58GiB(31.99%) tps: 82,598 tflops: 285.06 mfu: 28.82% global_avg_ntp_loss: 2.6324 global_avg_mtp_loss: 14.3754 +[titan] 2025-07-09 20:07:16,590 - root - INFO - lr: 2.4791e-04 gnorm: 0.76 [ 6:32:18<15:30:22] +[titan] 2025-07-09 20:07:20,509 - root - INFO - step: 29665 loss: 16.9119 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.2909 +[titan] 2025-07-09 20:07:20,509 - root - INFO - lr: 2.4789e-04 gnorm: 0.79 [ 6:32:22<15:30:18] +[titan] 2025-07-09 20:07:24,439 - root - INFO - step: 29670 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-09 20:07:24,440 - root - INFO - lr: 2.4788e-04 gnorm: 0.76 [ 6:32:26<15:30:14] +[titan] 2025-07-09 20:07:28,413 - root - INFO - step: 29675 loss: 16.7303 memory: 44.58GiB(31.99%) tps: 82,461 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.1422 +[titan] 2025-07-09 20:07:28,414 - root - INFO - lr: 2.4786e-04 gnorm: 0.72 [ 6:32:30<15:30:10] +[titan] 2025-07-09 20:07:32,323 - root - INFO - step: 29680 loss: 16.5995 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 20:07:32,323 - root - INFO - lr: 2.4784e-04 gnorm: 0.76 [ 6:32:34<15:30:06] +[titan] 2025-07-09 20:07:36,248 - root - INFO - step: 29685 loss: 16.7558 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5951 global_avg_mtp_loss: 14.1608 +[titan] 2025-07-09 20:07:36,249 - root - INFO - lr: 2.4782e-04 gnorm: 0.74 [ 6:32:38<15:30:02] +[titan] 2025-07-09 20:07:40,198 - root - INFO - step: 29690 loss: 16.6478 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.0736 +[titan] 2025-07-09 20:07:40,199 - root - INFO - lr: 2.4781e-04 gnorm: 0.75 [ 6:32:42<15:29:58] +[titan] 2025-07-09 20:07:44,183 - root - INFO - step: 29695 loss: 16.8947 memory: 44.58GiB(31.99%) tps: 82,255 tflops: 283.88 mfu: 28.70% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2748 +[titan] 2025-07-09 20:07:44,183 - root - INFO - lr: 2.4779e-04 gnorm: 0.82 [ 6:32:46<15:29:54] +[titan] 2025-07-09 20:07:45,112 - root - INFO - Dumping profiler traces at step 29696 +[titan] 2025-07-09 20:07:45,144 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:07:47,512 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:07:48,339 - root - INFO - step: 29700 loss: 16.8926 memory: 44.58GiB(31.99%) tps: 78,836 tflops: 272.08 mfu: 27.51% global_avg_ntp_loss: 2.6273 global_avg_mtp_loss: 14.2653 +[titan] 2025-07-09 20:07:48,340 - root - INFO - lr: 2.4777e-04 gnorm: 0.71 [ 6:32:50<15:29:50] +[titan] 2025-07-09 20:07:52,247 - root - INFO - step: 29705 loss: 16.8975 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6361 global_avg_mtp_loss: 14.2614 +[titan] 2025-07-09 20:07:52,248 - root - INFO - lr: 2.4776e-04 gnorm: 0.82 [ 6:32:54<15:29:46] +[titan] 2025-07-09 20:07:56,154 - root - INFO - step: 29710 loss: 16.7281 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5865 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 20:07:56,155 - root - INFO - lr: 2.4774e-04 gnorm: 0.74 [ 6:32:58<15:29:42] +[titan] 2025-07-09 20:08:00,073 - root - INFO - step: 29715 loss: 16.8533 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6193 global_avg_mtp_loss: 14.2340 +[titan] 2025-07-09 20:08:00,073 - root - INFO - lr: 2.4772e-04 gnorm: 0.78 [ 6:33:02<15:29:38] +[titan] 2025-07-09 20:08:03,987 - root - INFO - step: 29720 loss: 16.8056 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.1875 +[titan] 2025-07-09 20:08:03,987 - root - INFO - lr: 2.4771e-04 gnorm: 0.82 [ 6:33:05<15:29:34] +[titan] 2025-07-09 20:08:07,935 - root - INFO - step: 29725 loss: 16.8090 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.2015 +[titan] 2025-07-09 20:08:07,935 - root - INFO - lr: 2.4769e-04 gnorm: 0.76 [ 6:33:09<15:29:30] +[titan] 2025-07-09 20:08:11,844 - root - INFO - step: 29730 loss: 16.8050 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.2006 +[titan] 2025-07-09 20:08:11,844 - root - INFO - lr: 2.4767e-04 gnorm: 0.74 [ 6:33:13<15:29:26] +[titan] 2025-07-09 20:08:15,756 - root - INFO - step: 29735 loss: 16.6773 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1058 +[titan] 2025-07-09 20:08:15,756 - root - INFO - lr: 2.4766e-04 gnorm: 0.80 [ 6:33:17<15:29:22] +[titan] 2025-07-09 20:08:19,663 - root - INFO - step: 29740 loss: 16.8493 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6058 global_avg_mtp_loss: 14.2435 +[titan] 2025-07-09 20:08:19,663 - root - INFO - lr: 2.4764e-04 gnorm: 0.84 [ 6:33:21<15:29:18] +[titan] 2025-07-09 20:08:23,580 - root - INFO - step: 29745 loss: 16.7594 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1679 +[titan] 2025-07-09 20:08:23,580 - root - INFO - lr: 2.4762e-04 gnorm: 0.75 [ 6:33:25<15:29:14] +[titan] 2025-07-09 20:08:26,715 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:08:27,509 - root - INFO - step: 29750 loss: 16.6360 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.0546 +[titan] 2025-07-09 20:08:27,509 - root - INFO - lr: 2.4760e-04 gnorm: 0.78 [ 6:33:29<15:29:10] +[titan] 2025-07-09 20:08:31,449 - root - INFO - step: 29755 loss: 16.7492 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.1602 +[titan] 2025-07-09 20:08:31,449 - root - INFO - lr: 2.4759e-04 gnorm: 0.77 [ 6:33:33<15:29:05] +[titan] 2025-07-09 20:08:35,348 - root - INFO - step: 29760 loss: 16.6265 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-09 20:08:35,348 - root - INFO - lr: 2.4757e-04 gnorm: 0.74 [ 6:33:37<15:29:01] +[titan] 2025-07-09 20:08:39,245 - root - INFO - step: 29765 loss: 16.8203 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.2196 +[titan] 2025-07-09 20:08:39,246 - root - INFO - lr: 2.4755e-04 gnorm: 0.74 [ 6:33:41<15:28:57] +[titan] 2025-07-09 20:08:43,154 - root - INFO - step: 29770 loss: 17.1390 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.6709 global_avg_mtp_loss: 14.4680 +[titan] 2025-07-09 20:08:43,155 - root - INFO - lr: 2.4754e-04 gnorm: 0.81 [ 6:33:45<15:28:53] +[titan] 2025-07-09 20:08:47,075 - root - INFO - step: 29775 loss: 16.5657 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.0019 +[titan] 2025-07-09 20:08:47,075 - root - INFO - lr: 2.4752e-04 gnorm: 0.75 [ 6:33:49<15:28:49] +[titan] 2025-07-09 20:08:50,998 - root - INFO - step: 29780 loss: 16.6898 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.1186 +[titan] 2025-07-09 20:08:50,998 - root - INFO - lr: 2.4750e-04 gnorm: 0.83 [ 6:33:52<15:28:45] +[titan] 2025-07-09 20:08:55,333 - root - INFO - step: 29785 loss: 16.6455 memory: 44.58GiB(31.99%) tps: 75,598 tflops: 260.90 mfu: 26.38% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.0717 +[titan] 2025-07-09 20:08:55,333 - root - INFO - lr: 2.4749e-04 gnorm: 0.78 [ 6:33:57<15:28:42] +[titan] 2025-07-09 20:08:59,263 - root - INFO - step: 29790 loss: 16.6626 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.0831 +[titan] 2025-07-09 20:08:59,264 - root - INFO - lr: 2.4747e-04 gnorm: 0.77 [ 6:34:01<15:28:38] +[titan] 2025-07-09 20:09:03,179 - root - INFO - step: 29795 loss: 16.3616 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8419 +[titan] 2025-07-09 20:09:03,179 - root - INFO - lr: 2.4745e-04 gnorm: 0.84 [ 6:34:05<15:28:34] +[titan] 2025-07-09 20:09:06,299 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:09:07,095 - root - INFO - step: 29800 loss: 16.8296 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6186 global_avg_mtp_loss: 14.2110 +[titan] 2025-07-09 20:09:07,095 - root - INFO - lr: 2.4744e-04 gnorm: 0.82 [ 6:34:09<15:28:30] +[titan] 2025-07-09 20:09:11,022 - root - INFO - step: 29805 loss: 16.8775 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6193 global_avg_mtp_loss: 14.2582 +[titan] 2025-07-09 20:09:11,022 - root - INFO - lr: 2.4742e-04 gnorm: 0.77 [ 6:34:12<15:28:26] +[titan] 2025-07-09 20:09:14,964 - root - INFO - step: 29810 loss: 16.7709 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1866 +[titan] 2025-07-09 20:09:14,964 - root - INFO - lr: 2.4740e-04 gnorm: 0.72 [ 6:34:16<15:28:22] +[titan] 2025-07-09 20:09:18,925 - root - INFO - step: 29815 loss: 16.9312 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6296 global_avg_mtp_loss: 14.3015 +[titan] 2025-07-09 20:09:18,925 - root - INFO - lr: 2.4738e-04 gnorm: 0.78 [ 6:34:20<15:28:18] +[titan] 2025-07-09 20:09:22,828 - root - INFO - step: 29820 loss: 16.7796 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6090 global_avg_mtp_loss: 14.1705 +[titan] 2025-07-09 20:09:22,829 - root - INFO - lr: 2.4737e-04 gnorm: 0.70 [ 6:34:24<15:28:13] +[titan] 2025-07-09 20:09:26,779 - root - INFO - step: 29825 loss: 16.8195 memory: 44.58GiB(31.99%) tps: 82,957 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.6073 global_avg_mtp_loss: 14.2122 +[titan] 2025-07-09 20:09:26,779 - root - INFO - lr: 2.4735e-04 gnorm: 0.70 [ 6:34:28<15:28:09] +[titan] 2025-07-09 20:09:30,689 - root - INFO - step: 29830 loss: 16.8704 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2517 +[titan] 2025-07-09 20:09:30,689 - root - INFO - lr: 2.4733e-04 gnorm: 0.84 [ 6:34:32<15:28:05] +[titan] 2025-07-09 20:09:34,593 - root - INFO - step: 29835 loss: 16.9371 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.3064 +[titan] 2025-07-09 20:09:34,593 - root - INFO - lr: 2.4732e-04 gnorm: 0.80 [ 6:34:36<15:28:01] +[titan] 2025-07-09 20:09:38,483 - root - INFO - step: 29840 loss: 16.7261 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1354 +[titan] 2025-07-09 20:09:38,483 - root - INFO - lr: 2.4730e-04 gnorm: 0.77 [ 6:34:40<15:27:57] +[titan] 2025-07-09 20:09:42,399 - root - INFO - step: 29845 loss: 16.9080 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.6392 global_avg_mtp_loss: 14.2688 +[titan] 2025-07-09 20:09:42,399 - root - INFO - lr: 2.4728e-04 gnorm: 0.79 [ 6:34:44<15:27:53] +[titan] 2025-07-09 20:09:45,528 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:09:46,317 - root - INFO - step: 29850 loss: 16.8897 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6191 global_avg_mtp_loss: 14.2706 +[titan] 2025-07-09 20:09:46,317 - root - INFO - lr: 2.4727e-04 gnorm: 0.73 [ 6:34:48<15:27:49] +[titan] 2025-07-09 20:09:50,214 - root - INFO - step: 29855 loss: 16.5343 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 13.9804 +[titan] 2025-07-09 20:09:50,215 - root - INFO - lr: 2.4725e-04 gnorm: 0.80 [ 6:34:52<15:27:45] +[titan] 2025-07-09 20:09:54,155 - root - INFO - step: 29860 loss: 16.6711 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.0937 +[titan] 2025-07-09 20:09:54,155 - root - INFO - lr: 2.4723e-04 gnorm: 0.77 [ 6:34:56<15:27:41] +[titan] 2025-07-09 20:09:58,093 - root - INFO - step: 29865 loss: 16.5948 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.0196 +[titan] 2025-07-09 20:09:58,093 - root - INFO - lr: 2.4721e-04 gnorm: 0.81 [ 6:35:00<15:27:37] +[titan] 2025-07-09 20:10:02,031 - root - INFO - step: 29870 loss: 16.6862 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.0977 +[titan] 2025-07-09 20:10:02,031 - root - INFO - lr: 2.4720e-04 gnorm: 0.77 [ 6:35:03<15:27:33] +[titan] 2025-07-09 20:10:05,958 - root - INFO - step: 29875 loss: 16.7386 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 20:10:05,958 - root - INFO - lr: 2.4718e-04 gnorm: 0.87 [ 6:35:07<15:27:29] +[titan] 2025-07-09 20:10:09,880 - root - INFO - step: 29880 loss: 16.8998 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.2844 +[titan] 2025-07-09 20:10:09,880 - root - INFO - lr: 2.4716e-04 gnorm: 0.79 [ 6:35:11<15:27:24] +[titan] 2025-07-09 20:10:13,826 - root - INFO - step: 29885 loss: 16.9376 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.3204 +[titan] 2025-07-09 20:10:13,827 - root - INFO - lr: 2.4715e-04 gnorm: 0.76 [ 6:35:15<15:27:20] +[titan] 2025-07-09 20:10:17,742 - root - INFO - step: 29890 loss: 16.8090 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.1883 +[titan] 2025-07-09 20:10:17,742 - root - INFO - lr: 2.4713e-04 gnorm: 0.75 [ 6:35:19<15:27:16] +[titan] 2025-07-09 20:10:21,649 - root - INFO - step: 29895 loss: 16.8207 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2020 +[titan] 2025-07-09 20:10:21,649 - root - INFO - lr: 2.4711e-04 gnorm: 0.74 [ 6:35:23<15:27:12] +[titan] 2025-07-09 20:10:24,798 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:10:25,595 - root - INFO - step: 29900 loss: 16.7117 memory: 44.58GiB(31.99%) tps: 83,045 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1242 +[titan] 2025-07-09 20:10:25,595 - root - INFO - lr: 2.4710e-04 gnorm: 0.75 [ 6:35:27<15:27:08] +[titan] 2025-07-09 20:10:29,516 - root - INFO - step: 29905 loss: 16.8549 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2365 +[titan] 2025-07-09 20:10:29,517 - root - INFO - lr: 2.4708e-04 gnorm: 0.71 [ 6:35:31<15:27:04] +[titan] 2025-07-09 20:10:33,439 - root - INFO - step: 29910 loss: 16.8930 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.2742 +[titan] 2025-07-09 20:10:33,439 - root - INFO - lr: 2.4706e-04 gnorm: 0.75 [ 6:35:35<15:27:00] +[titan] 2025-07-09 20:10:37,400 - root - INFO - step: 29915 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.8629 +[titan] 2025-07-09 20:10:37,401 - root - INFO - lr: 2.4704e-04 gnorm: 0.75 [ 6:35:39<15:26:56] +[titan] 2025-07-09 20:10:41,327 - root - INFO - step: 29920 loss: 16.7318 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1362 +[titan] 2025-07-09 20:10:41,328 - root - INFO - lr: 2.4703e-04 gnorm: 0.77 [ 6:35:43<15:26:52] +[titan] 2025-07-09 20:10:45,250 - root - INFO - step: 29925 loss: 16.8524 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6176 global_avg_mtp_loss: 14.2347 +[titan] 2025-07-09 20:10:45,251 - root - INFO - lr: 2.4701e-04 gnorm: 0.75 [ 6:35:47<15:26:48] +[titan] 2025-07-09 20:10:49,166 - root - INFO - step: 29930 loss: 16.6881 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.0977 +[titan] 2025-07-09 20:10:49,166 - root - INFO - lr: 2.4699e-04 gnorm: 0.84 [ 6:35:51<15:26:44] +[titan] 2025-07-09 20:10:53,293 - root - INFO - step: 29935 loss: 17.1366 memory: 44.58GiB(31.99%) tps: 79,417 tflops: 274.08 mfu: 27.71% global_avg_ntp_loss: 2.6849 global_avg_mtp_loss: 14.4517 +[titan] 2025-07-09 20:10:53,293 - root - INFO - lr: 2.4698e-04 gnorm: 0.79 [ 6:35:55<15:26:40] +[titan] 2025-07-09 20:10:57,227 - root - INFO - step: 29940 loss: 16.7472 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1551 +[titan] 2025-07-09 20:10:57,227 - root - INFO - lr: 2.4696e-04 gnorm: 0.78 [ 6:35:59<15:26:36] +[titan] 2025-07-09 20:11:01,145 - root - INFO - step: 29945 loss: 16.5996 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0326 +[titan] 2025-07-09 20:11:01,145 - root - INFO - lr: 2.4694e-04 gnorm: 0.77 [ 6:36:03<15:26:32] +[titan] 2025-07-09 20:11:04,277 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:11:05,069 - root - INFO - step: 29950 loss: 16.7736 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5971 global_avg_mtp_loss: 14.1765 +[titan] 2025-07-09 20:11:05,069 - root - INFO - lr: 2.4693e-04 gnorm: 0.76 [ 6:36:06<15:26:28] +[titan] 2025-07-09 20:11:09,020 - root - INFO - step: 29955 loss: 16.5131 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9624 +[titan] 2025-07-09 20:11:09,020 - root - INFO - lr: 2.4691e-04 gnorm: 0.75 [ 6:36:10<15:26:24] +[titan] 2025-07-09 20:11:12,919 - root - INFO - step: 29960 loss: 16.8951 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.2791 +[titan] 2025-07-09 20:11:12,919 - root - INFO - lr: 2.4689e-04 gnorm: 0.78 [ 6:36:14<15:26:20] +[titan] 2025-07-09 20:11:16,878 - root - INFO - step: 29965 loss: 16.3078 memory: 44.58GiB(31.99%) tps: 82,772 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.7965 +[titan] 2025-07-09 20:11:16,878 - root - INFO - lr: 2.4687e-04 gnorm: 0.72 [ 6:36:18<15:26:16] +[titan] 2025-07-09 20:11:20,784 - root - INFO - step: 29970 loss: 16.8829 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.2644 +[titan] 2025-07-09 20:11:20,784 - root - INFO - lr: 2.4686e-04 gnorm: 0.76 [ 6:36:22<15:26:12] +[titan] 2025-07-09 20:11:24,718 - root - INFO - step: 29975 loss: 16.8833 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.2725 +[titan] 2025-07-09 20:11:24,719 - root - INFO - lr: 2.4684e-04 gnorm: 0.79 [ 6:36:26<15:26:08] +[titan] 2025-07-09 20:11:28,665 - root - INFO - step: 29980 loss: 16.9107 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.6230 global_avg_mtp_loss: 14.2878 +[titan] 2025-07-09 20:11:28,665 - root - INFO - lr: 2.4682e-04 gnorm: 0.75 [ 6:36:30<15:26:04] +[titan] 2025-07-09 20:11:32,573 - root - INFO - step: 29985 loss: 16.9525 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6344 global_avg_mtp_loss: 14.3181 +[titan] 2025-07-09 20:11:32,573 - root - INFO - lr: 2.4681e-04 gnorm: 0.76 [ 6:36:34<15:26:00] +[titan] 2025-07-09 20:11:36,492 - root - INFO - step: 29990 loss: 17.0776 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6617 global_avg_mtp_loss: 14.4159 +[titan] 2025-07-09 20:11:36,492 - root - INFO - lr: 2.4679e-04 gnorm: 0.76 [ 6:36:38<15:25:56] +[titan] 2025-07-09 20:11:40,483 - root - INFO - step: 29995 loss: 16.7379 memory: 44.58GiB(31.99%) tps: 82,111 tflops: 283.38 mfu: 28.65% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1490 +[titan] 2025-07-09 20:11:40,483 - root - INFO - lr: 2.4677e-04 gnorm: 0.76 [ 6:36:42<15:25:52] +[titan] 2025-07-09 20:11:43,622 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:11:44,432 - root - INFO - step: 30000 loss: 16.7576 memory: 44.58GiB(31.99%) tps: 82,976 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.1628 +[titan] 2025-07-09 20:11:44,433 - root - INFO - lr: 2.4675e-04 gnorm: 0.80 [ 6:36:46<15:25:48] +[titan] 2025-07-09 20:11:44,433 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-09 20:11:46,935 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-09 20:11:46,935 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.50 seconds. +[titan] 2025-07-09 20:12:43,241 - root - INFO - step: 30005 loss: 16.8372 memory: 44.58GiB(31.99%) tps: 5,572 tflops: 19.23 mfu: 1.94% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2244 +[titan] 2025-07-09 20:12:43,241 - root - INFO - lr: 2.4674e-04 gnorm: 0.81 [ 6:37:45<15:27:52] +[titan] 2025-07-09 20:12:47,177 - root - INFO - step: 30010 loss: 16.8713 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.2605 +[titan] 2025-07-09 20:12:47,177 - root - INFO - lr: 2.4672e-04 gnorm: 0.82 [ 6:37:49<15:27:48] +[titan] 2025-07-09 20:12:51,113 - root - INFO - step: 30015 loss: 16.7307 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.1364 +[titan] 2025-07-09 20:12:51,113 - root - INFO - lr: 2.4670e-04 gnorm: 0.77 [ 6:37:53<15:27:43] +[titan] 2025-07-09 20:12:55,059 - root - INFO - step: 30020 loss: 17.0102 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.6512 global_avg_mtp_loss: 14.3590 +[titan] 2025-07-09 20:12:55,059 - root - INFO - lr: 2.4669e-04 gnorm: 0.77 [ 6:37:56<15:27:39] +[titan] 2025-07-09 20:12:58,967 - root - INFO - step: 30025 loss: 16.5377 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-09 20:12:58,967 - root - INFO - lr: 2.4667e-04 gnorm: 0.84 [ 6:38:00<15:27:35] +[titan] 2025-07-09 20:13:02,885 - root - INFO - step: 30030 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0091 +[titan] 2025-07-09 20:13:02,885 - root - INFO - lr: 2.4665e-04 gnorm: 0.85 [ 6:38:04<15:27:31] +[titan] 2025-07-09 20:13:06,780 - root - INFO - step: 30035 loss: 16.8720 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6175 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 20:13:06,780 - root - INFO - lr: 2.4664e-04 gnorm: 0.74 [ 6:38:08<15:27:27] +[titan] 2025-07-09 20:13:10,725 - root - INFO - step: 30040 loss: 16.7497 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.1599 +[titan] 2025-07-09 20:13:10,726 - root - INFO - lr: 2.4662e-04 gnorm: 0.78 [ 6:38:12<15:27:23] +[titan] 2025-07-09 20:13:14,643 - root - INFO - step: 30045 loss: 16.9622 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6283 global_avg_mtp_loss: 14.3339 +[titan] 2025-07-09 20:13:14,643 - root - INFO - lr: 2.4660e-04 gnorm: 0.72 [ 6:38:16<15:27:19] +[titan] 2025-07-09 20:13:17,758 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:13:18,549 - root - INFO - step: 30050 loss: 16.7271 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.1255 +[titan] 2025-07-09 20:13:18,550 - root - INFO - lr: 2.4658e-04 gnorm: 0.74 [ 6:38:20<15:27:15] +[titan] 2025-07-09 20:13:22,489 - root - INFO - step: 30055 loss: 16.8503 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.6218 global_avg_mtp_loss: 14.2285 +[titan] 2025-07-09 20:13:22,490 - root - INFO - lr: 2.4657e-04 gnorm: 0.78 [ 6:38:24<15:27:11] +[titan] 2025-07-09 20:13:26,401 - root - INFO - step: 30060 loss: 16.9819 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6346 global_avg_mtp_loss: 14.3473 +[titan] 2025-07-09 20:13:26,401 - root - INFO - lr: 2.4655e-04 gnorm: 0.72 [ 6:38:28<15:27:06] +[titan] 2025-07-09 20:13:30,331 - root - INFO - step: 30065 loss: 16.9252 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.3015 +[titan] 2025-07-09 20:13:30,331 - root - INFO - lr: 2.4653e-04 gnorm: 0.77 [ 6:38:32<15:27:02] +[titan] 2025-07-09 20:13:34,263 - root - INFO - step: 30070 loss: 16.9742 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.6598 global_avg_mtp_loss: 14.3144 +[titan] 2025-07-09 20:13:34,263 - root - INFO - lr: 2.4652e-04 gnorm: 0.81 [ 6:38:36<15:26:58] +[titan] 2025-07-09 20:13:38,215 - root - INFO - step: 30075 loss: 16.8246 memory: 44.58GiB(31.99%) tps: 82,928 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.2224 +[titan] 2025-07-09 20:13:38,215 - root - INFO - lr: 2.4650e-04 gnorm: 0.78 [ 6:38:40<15:26:54] +[titan] 2025-07-09 20:13:42,161 - root - INFO - step: 30080 loss: 16.7626 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5958 global_avg_mtp_loss: 14.1668 +[titan] 2025-07-09 20:13:42,161 - root - INFO - lr: 2.4648e-04 gnorm: 0.77 [ 6:38:44<15:26:50] +[titan] 2025-07-09 20:13:46,133 - root - INFO - step: 30085 loss: 16.7553 memory: 44.58GiB(31.99%) tps: 82,503 tflops: 284.73 mfu: 28.79% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1669 +[titan] 2025-07-09 20:13:46,133 - root - INFO - lr: 2.4646e-04 gnorm: 0.73 [ 6:38:48<15:26:46] +[titan] 2025-07-09 20:13:50,049 - root - INFO - step: 30090 loss: 16.7682 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5930 global_avg_mtp_loss: 14.1753 +[titan] 2025-07-09 20:13:50,050 - root - INFO - lr: 2.4645e-04 gnorm: 0.75 [ 6:38:51<15:26:42] +[titan] 2025-07-09 20:13:53,981 - root - INFO - step: 30095 loss: 16.6532 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0775 +[titan] 2025-07-09 20:13:53,981 - root - INFO - lr: 2.4643e-04 gnorm: 0.78 [ 6:38:55<15:26:38] +[titan] 2025-07-09 20:13:57,094 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:13:57,893 - root - INFO - step: 30100 loss: 16.7389 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5962 global_avg_mtp_loss: 14.1427 +[titan] 2025-07-09 20:13:57,893 - root - INFO - lr: 2.4641e-04 gnorm: 0.79 [ 6:38:59<15:26:34] +[titan] 2025-07-09 20:14:01,820 - root - INFO - step: 30105 loss: 16.5315 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5855 global_avg_mtp_loss: 13.9460 +[titan] 2025-07-09 20:14:01,821 - root - INFO - lr: 2.4640e-04 gnorm: 0.94 [ 6:39:03<15:26:30] +[titan] 2025-07-09 20:14:05,736 - root - INFO - step: 30110 loss: 16.6169 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.0483 +[titan] 2025-07-09 20:14:05,736 - root - INFO - lr: 2.4638e-04 gnorm: 0.90 [ 6:39:07<15:26:26] +[titan] 2025-07-09 20:14:09,673 - root - INFO - step: 30115 loss: 16.6702 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0929 +[titan] 2025-07-09 20:14:09,673 - root - INFO - lr: 2.4636e-04 gnorm: 0.80 [ 6:39:11<15:26:22] +[titan] 2025-07-09 20:14:13,620 - root - INFO - step: 30120 loss: 16.7195 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6045 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-09 20:14:13,620 - root - INFO - lr: 2.4635e-04 gnorm: 0.79 [ 6:39:15<15:26:18] +[titan] 2025-07-09 20:14:17,528 - root - INFO - step: 30125 loss: 16.8131 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.2090 +[titan] 2025-07-09 20:14:17,528 - root - INFO - lr: 2.4633e-04 gnorm: 0.74 [ 6:39:19<15:26:13] +[titan] 2025-07-09 20:14:21,493 - root - INFO - step: 30130 loss: 16.8001 memory: 44.58GiB(31.99%) tps: 82,643 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.1979 +[titan] 2025-07-09 20:14:21,494 - root - INFO - lr: 2.4631e-04 gnorm: 0.71 [ 6:39:23<15:26:09] +[titan] 2025-07-09 20:14:25,440 - root - INFO - step: 30135 loss: 16.8249 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2057 +[titan] 2025-07-09 20:14:25,440 - root - INFO - lr: 2.4629e-04 gnorm: 0.81 [ 6:39:27<15:26:05] +[titan] 2025-07-09 20:14:29,369 - root - INFO - step: 30140 loss: 16.6647 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.0804 +[titan] 2025-07-09 20:14:29,369 - root - INFO - lr: 2.4628e-04 gnorm: 0.75 [ 6:39:31<15:26:01] +[titan] 2025-07-09 20:14:33,311 - root - INFO - step: 30145 loss: 16.8763 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.6208 global_avg_mtp_loss: 14.2554 +[titan] 2025-07-09 20:14:33,311 - root - INFO - lr: 2.4626e-04 gnorm: 0.79 [ 6:39:35<15:25:57] +[titan] 2025-07-09 20:14:36,434 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:14:37,227 - root - INFO - step: 30150 loss: 16.6516 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.0701 +[titan] 2025-07-09 20:14:37,227 - root - INFO - lr: 2.4624e-04 gnorm: 0.82 [ 6:39:39<15:25:53] +[titan] 2025-07-09 20:14:41,149 - root - INFO - step: 30155 loss: 16.7008 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1169 +[titan] 2025-07-09 20:14:41,149 - root - INFO - lr: 2.4623e-04 gnorm: 0.75 [ 6:39:43<15:25:49] +[titan] 2025-07-09 20:14:45,079 - root - INFO - step: 30160 loss: 17.1207 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.6820 global_avg_mtp_loss: 14.4387 +[titan] 2025-07-09 20:14:45,079 - root - INFO - lr: 2.4621e-04 gnorm: 0.76 [ 6:39:46<15:25:45] +[titan] 2025-07-09 20:14:48,980 - root - INFO - step: 30165 loss: 16.7253 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5943 global_avg_mtp_loss: 14.1311 +[titan] 2025-07-09 20:14:48,980 - root - INFO - lr: 2.4619e-04 gnorm: 0.83 [ 6:39:50<15:25:41] +[titan] 2025-07-09 20:14:52,914 - root - INFO - step: 30170 loss: 16.7963 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.1913 +[titan] 2025-07-09 20:14:52,914 - root - INFO - lr: 2.4617e-04 gnorm: 0.78 [ 6:39:54<15:25:37] +[titan] 2025-07-09 20:14:56,832 - root - INFO - step: 30175 loss: 16.8792 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2552 +[titan] 2025-07-09 20:14:56,832 - root - INFO - lr: 2.4616e-04 gnorm: 0.80 [ 6:39:58<15:25:33] +[titan] 2025-07-09 20:15:00,775 - root - INFO - step: 30180 loss: 16.8736 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6091 global_avg_mtp_loss: 14.2646 +[titan] 2025-07-09 20:15:00,775 - root - INFO - lr: 2.4614e-04 gnorm: 0.92 [ 6:40:02<15:25:29] +[titan] 2025-07-09 20:15:04,694 - root - INFO - step: 30185 loss: 16.4564 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 13.8979 +[titan] 2025-07-09 20:15:04,695 - root - INFO - lr: 2.4612e-04 gnorm: 0.90 [ 6:40:06<15:25:24] +[titan] 2025-07-09 20:15:08,638 - root - INFO - step: 30190 loss: 16.7002 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5933 global_avg_mtp_loss: 14.1069 +[titan] 2025-07-09 20:15:08,638 - root - INFO - lr: 2.4611e-04 gnorm: 0.78 [ 6:40:10<15:25:20] +[titan] 2025-07-09 20:15:12,565 - root - INFO - step: 30195 loss: 16.5852 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.0141 +[titan] 2025-07-09 20:15:12,565 - root - INFO - lr: 2.4609e-04 gnorm: 0.73 [ 6:40:14<15:25:16] +[titan] 2025-07-09 20:15:15,844 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:15:16,635 - root - INFO - step: 30200 loss: 16.7318 memory: 44.58GiB(31.99%) tps: 80,509 tflops: 277.85 mfu: 28.09% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1373 +[titan] 2025-07-09 20:15:16,636 - root - INFO - lr: 2.4607e-04 gnorm: 0.75 [ 6:40:18<15:25:13] +[titan] 2025-07-09 20:15:20,689 - root - INFO - step: 30205 loss: 16.8609 memory: 44.58GiB(31.99%) tps: 80,855 tflops: 279.04 mfu: 28.21% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.2486 +[titan] 2025-07-09 20:15:20,689 - root - INFO - lr: 2.4605e-04 gnorm: 0.78 [ 6:40:22<15:25:09] +[titan] 2025-07-09 20:15:23,187 - root - INFO - Dumping profiler traces at step 30208 +[titan] 2025-07-09 20:15:23,219 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:15:24,799 - root - INFO - step: 30210 loss: 16.8483 memory: 44.58GiB(31.99%) tps: 79,724 tflops: 275.14 mfu: 27.82% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2284 +[titan] 2025-07-09 20:15:24,799 - root - INFO - lr: 2.4604e-04 gnorm: 0.77 [ 6:40:26<15:25:05] +[titan] 2025-07-09 20:15:28,726 - root - INFO - step: 30215 loss: 16.7382 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.6079 global_avg_mtp_loss: 14.1303 +[titan] 2025-07-09 20:15:28,726 - root - INFO - lr: 2.4602e-04 gnorm: 0.79 [ 6:40:30<15:25:01] +[titan] 2025-07-09 20:15:32,635 - root - INFO - step: 30220 loss: 16.4575 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9250 +[titan] 2025-07-09 20:15:32,636 - root - INFO - lr: 2.4600e-04 gnorm: 0.74 [ 6:40:34<15:24:57] +[titan] 2025-07-09 20:15:36,566 - root - INFO - step: 30225 loss: 16.7481 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.1542 +[titan] 2025-07-09 20:15:36,566 - root - INFO - lr: 2.4599e-04 gnorm: 0.73 [ 6:40:38<15:24:53] +[titan] 2025-07-09 20:15:40,465 - root - INFO - step: 30230 loss: 16.6263 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0615 +[titan] 2025-07-09 20:15:40,465 - root - INFO - lr: 2.4597e-04 gnorm: 0.71 [ 6:40:42<15:24:49] +[titan] 2025-07-09 20:15:44,402 - root - INFO - step: 30235 loss: 16.8116 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.2150 +[titan] 2025-07-09 20:15:44,402 - root - INFO - lr: 2.4595e-04 gnorm: 0.73 [ 6:40:46<15:24:45] +[titan] 2025-07-09 20:15:48,334 - root - INFO - step: 30240 loss: 16.5754 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0061 +[titan] 2025-07-09 20:15:48,335 - root - INFO - lr: 2.4593e-04 gnorm: 0.74 [ 6:40:50<15:24:40] +[titan] 2025-07-09 20:15:52,248 - root - INFO - step: 30245 loss: 16.8612 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.2410 +[titan] 2025-07-09 20:15:52,249 - root - INFO - lr: 2.4592e-04 gnorm: 0.77 [ 6:40:54<15:24:36] +[titan] 2025-07-09 20:15:55,406 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:15:56,207 - root - INFO - step: 30250 loss: 16.5955 memory: 44.58GiB(31.99%) tps: 82,793 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.0167 +[titan] 2025-07-09 20:15:56,207 - root - INFO - lr: 2.4590e-04 gnorm: 0.80 [ 6:40:58<15:24:32] +[titan] 2025-07-09 20:16:00,106 - root - INFO - step: 30255 loss: 16.6213 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5653 global_avg_mtp_loss: 14.0560 +[titan] 2025-07-09 20:16:00,106 - root - INFO - lr: 2.4588e-04 gnorm: 0.75 [ 6:41:01<15:24:28] +[titan] 2025-07-09 20:16:04,026 - root - INFO - step: 30260 loss: 16.9304 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6282 global_avg_mtp_loss: 14.3023 +[titan] 2025-07-09 20:16:04,027 - root - INFO - lr: 2.4587e-04 gnorm: 0.76 [ 6:41:05<15:24:24] +[titan] 2025-07-09 20:16:07,932 - root - INFO - step: 30265 loss: 16.5477 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 13.9778 +[titan] 2025-07-09 20:16:07,932 - root - INFO - lr: 2.4585e-04 gnorm: 0.76 [ 6:41:09<15:24:20] +[titan] 2025-07-09 20:16:11,831 - root - INFO - step: 30270 loss: 16.8149 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6058 global_avg_mtp_loss: 14.2091 +[titan] 2025-07-09 20:16:11,832 - root - INFO - lr: 2.4583e-04 gnorm: 0.76 [ 6:41:13<15:24:16] +[titan] 2025-07-09 20:16:15,765 - root - INFO - step: 30275 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 83,321 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.0288 +[titan] 2025-07-09 20:16:15,765 - root - INFO - lr: 2.4581e-04 gnorm: 0.73 [ 6:41:17<15:24:12] +[titan] 2025-07-09 20:16:19,685 - root - INFO - step: 30280 loss: 16.6706 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.0834 +[titan] 2025-07-09 20:16:19,685 - root - INFO - lr: 2.4580e-04 gnorm: 0.77 [ 6:41:21<15:24:08] +[titan] 2025-07-09 20:16:23,598 - root - INFO - step: 30285 loss: 16.8860 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.2688 +[titan] 2025-07-09 20:16:23,599 - root - INFO - lr: 2.4578e-04 gnorm: 0.76 [ 6:41:25<15:24:03] +[titan] 2025-07-09 20:16:27,521 - root - INFO - step: 30290 loss: 16.6676 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5868 global_avg_mtp_loss: 14.0807 +[titan] 2025-07-09 20:16:27,522 - root - INFO - lr: 2.4576e-04 gnorm: 0.81 [ 6:41:29<15:23:59] +[titan] 2025-07-09 20:16:31,466 - root - INFO - step: 30295 loss: 16.7460 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1568 +[titan] 2025-07-09 20:16:31,466 - root - INFO - lr: 2.4575e-04 gnorm: 0.79 [ 6:41:33<15:23:55] +[titan] 2025-07-09 20:16:34,588 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:16:35,375 - root - INFO - step: 30300 loss: 16.8957 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2765 +[titan] 2025-07-09 20:16:35,375 - root - INFO - lr: 2.4573e-04 gnorm: 0.87 [ 6:41:37<15:23:51] +[titan] 2025-07-09 20:16:39,266 - root - INFO - step: 30305 loss: 16.8731 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.6101 global_avg_mtp_loss: 14.2630 +[titan] 2025-07-09 20:16:39,266 - root - INFO - lr: 2.4571e-04 gnorm: 0.79 [ 6:41:41<15:23:47] +[titan] 2025-07-09 20:16:43,188 - root - INFO - step: 30310 loss: 17.0795 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6559 global_avg_mtp_loss: 14.4235 +[titan] 2025-07-09 20:16:43,188 - root - INFO - lr: 2.4569e-04 gnorm: 0.77 [ 6:41:45<15:23:43] +[titan] 2025-07-09 20:16:47,107 - root - INFO - step: 30315 loss: 16.7146 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6054 global_avg_mtp_loss: 14.1092 +[titan] 2025-07-09 20:16:47,107 - root - INFO - lr: 2.4568e-04 gnorm: 0.83 [ 6:41:48<15:23:39] +[titan] 2025-07-09 20:16:51,035 - root - INFO - step: 30320 loss: 16.7013 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1149 +[titan] 2025-07-09 20:16:51,035 - root - INFO - lr: 2.4566e-04 gnorm: 0.80 [ 6:41:52<15:23:35] +[titan] 2025-07-09 20:16:54,931 - root - INFO - step: 30325 loss: 16.6766 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-09 20:16:54,931 - root - INFO - lr: 2.4564e-04 gnorm: 0.76 [ 6:41:56<15:23:30] +[titan] 2025-07-09 20:16:58,900 - root - INFO - step: 30330 loss: 16.6019 memory: 44.58GiB(31.99%) tps: 82,572 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.0215 +[titan] 2025-07-09 20:16:58,900 - root - INFO - lr: 2.4563e-04 gnorm: 0.81 [ 6:42:00<15:23:26] +[titan] 2025-07-09 20:17:02,813 - root - INFO - step: 30335 loss: 16.3698 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.8434 +[titan] 2025-07-09 20:17:02,814 - root - INFO - lr: 2.4561e-04 gnorm: 0.80 [ 6:42:04<15:23:22] +[titan] 2025-07-09 20:17:06,724 - root - INFO - step: 30340 loss: 16.4972 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 13.9437 +[titan] 2025-07-09 20:17:06,724 - root - INFO - lr: 2.4559e-04 gnorm: 0.75 [ 6:42:08<15:23:18] +[titan] 2025-07-09 20:17:10,628 - root - INFO - step: 30345 loss: 16.8845 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6245 global_avg_mtp_loss: 14.2600 +[titan] 2025-07-09 20:17:10,629 - root - INFO - lr: 2.4557e-04 gnorm: 0.77 [ 6:42:12<15:23:14] +[titan] 2025-07-09 20:17:13,771 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:17:14,561 - root - INFO - step: 30350 loss: 16.5976 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 14.0347 +[titan] 2025-07-09 20:17:14,561 - root - INFO - lr: 2.4556e-04 gnorm: 0.79 [ 6:42:16<15:23:10] +[titan] 2025-07-09 20:17:18,466 - root - INFO - step: 30355 loss: 16.8652 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6021 global_avg_mtp_loss: 14.2631 +[titan] 2025-07-09 20:17:18,466 - root - INFO - lr: 2.4554e-04 gnorm: 0.71 [ 6:42:20<15:23:06] +[titan] 2025-07-09 20:17:22,387 - root - INFO - step: 30360 loss: 16.6815 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1034 +[titan] 2025-07-09 20:17:22,387 - root - INFO - lr: 2.4552e-04 gnorm: 0.73 [ 6:42:24<15:23:02] +[titan] 2025-07-09 20:17:26,289 - root - INFO - step: 30365 loss: 16.6936 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.1138 +[titan] 2025-07-09 20:17:26,289 - root - INFO - lr: 2.4551e-04 gnorm: 0.71 [ 6:42:28<15:22:58] +[titan] 2025-07-09 20:17:30,224 - root - INFO - step: 30370 loss: 16.7534 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1644 +[titan] 2025-07-09 20:17:30,224 - root - INFO - lr: 2.4549e-04 gnorm: 0.73 [ 6:42:32<15:22:54] +[titan] 2025-07-09 20:17:34,128 - root - INFO - step: 30375 loss: 16.9578 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6239 global_avg_mtp_loss: 14.3339 +[titan] 2025-07-09 20:17:34,128 - root - INFO - lr: 2.4547e-04 gnorm: 0.74 [ 6:42:35<15:22:49] +[titan] 2025-07-09 20:17:38,034 - root - INFO - step: 30380 loss: 16.8193 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.6058 global_avg_mtp_loss: 14.2134 +[titan] 2025-07-09 20:17:38,034 - root - INFO - lr: 2.4545e-04 gnorm: 0.79 [ 6:42:39<15:22:45] +[titan] 2025-07-09 20:17:41,957 - root - INFO - step: 30385 loss: 16.8817 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2548 +[titan] 2025-07-09 20:17:41,958 - root - INFO - lr: 2.4544e-04 gnorm: 0.77 [ 6:42:43<15:22:41] +[titan] 2025-07-09 20:17:45,879 - root - INFO - step: 30390 loss: 16.6346 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5633 global_avg_mtp_loss: 14.0713 +[titan] 2025-07-09 20:17:45,879 - root - INFO - lr: 2.4542e-04 gnorm: 0.78 [ 6:42:47<15:22:37] +[titan] 2025-07-09 20:17:49,796 - root - INFO - step: 30395 loss: 16.8994 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6283 global_avg_mtp_loss: 14.2711 +[titan] 2025-07-09 20:17:49,797 - root - INFO - lr: 2.4540e-04 gnorm: 0.79 [ 6:42:51<15:22:33] +[titan] 2025-07-09 20:17:52,931 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:17:53,718 - root - INFO - step: 30400 loss: 16.9426 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.6401 global_avg_mtp_loss: 14.3026 +[titan] 2025-07-09 20:17:53,719 - root - INFO - lr: 2.4538e-04 gnorm: 0.73 [ 6:42:55<15:22:29] +[titan] 2025-07-09 20:17:57,716 - root - INFO - step: 30405 loss: 16.6039 memory: 44.58GiB(31.99%) tps: 81,965 tflops: 282.88 mfu: 28.60% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0295 +[titan] 2025-07-09 20:17:57,717 - root - INFO - lr: 2.4537e-04 gnorm: 0.80 [ 6:42:59<15:22:25] +[titan] 2025-07-09 20:18:01,646 - root - INFO - step: 30410 loss: 16.6842 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.1047 +[titan] 2025-07-09 20:18:01,646 - root - INFO - lr: 2.4535e-04 gnorm: 0.77 [ 6:43:03<15:22:21] +[titan] 2025-07-09 20:18:05,570 - root - INFO - step: 30415 loss: 16.9602 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6477 global_avg_mtp_loss: 14.3125 +[titan] 2025-07-09 20:18:05,570 - root - INFO - lr: 2.4533e-04 gnorm: 0.79 [ 6:43:07<15:22:17] +[titan] 2025-07-09 20:18:09,550 - root - INFO - step: 30420 loss: 16.6074 memory: 44.58GiB(31.99%) tps: 82,349 tflops: 284.20 mfu: 28.74% global_avg_ntp_loss: 2.5661 global_avg_mtp_loss: 14.0413 +[titan] 2025-07-09 20:18:09,550 - root - INFO - lr: 2.4532e-04 gnorm: 0.78 [ 6:43:11<15:22:13] +[titan] 2025-07-09 20:18:13,523 - root - INFO - step: 30425 loss: 16.7745 memory: 44.58GiB(31.99%) tps: 82,490 tflops: 284.69 mfu: 28.79% global_avg_ntp_loss: 2.5841 global_avg_mtp_loss: 14.1904 +[titan] 2025-07-09 20:18:13,523 - root - INFO - lr: 2.4530e-04 gnorm: 0.81 [ 6:43:15<15:22:09] +[titan] 2025-07-09 20:18:17,499 - root - INFO - step: 30430 loss: 16.7220 memory: 44.58GiB(31.99%) tps: 82,415 tflops: 284.43 mfu: 28.76% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1290 +[titan] 2025-07-09 20:18:17,499 - root - INFO - lr: 2.4528e-04 gnorm: 0.72 [ 6:43:19<15:22:05] +[titan] 2025-07-09 20:18:21,417 - root - INFO - step: 30435 loss: 16.7256 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5949 global_avg_mtp_loss: 14.1307 +[titan] 2025-07-09 20:18:21,417 - root - INFO - lr: 2.4526e-04 gnorm: 0.74 [ 6:43:23<15:22:01] +[titan] 2025-07-09 20:18:25,354 - root - INFO - step: 30440 loss: 16.5571 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 14.0072 +[titan] 2025-07-09 20:18:25,355 - root - INFO - lr: 2.4525e-04 gnorm: 0.80 [ 6:43:27<15:21:57] +[titan] 2025-07-09 20:18:29,301 - root - INFO - step: 30445 loss: 16.9851 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.3500 +[titan] 2025-07-09 20:18:29,301 - root - INFO - lr: 2.4523e-04 gnorm: 0.77 [ 6:43:31<15:21:53] +[titan] 2025-07-09 20:18:32,428 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:18:33,225 - root - INFO - step: 30450 loss: 16.9517 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.3115 +[titan] 2025-07-09 20:18:33,225 - root - INFO - lr: 2.4521e-04 gnorm: 0.81 [ 6:43:35<15:21:49] +[titan] 2025-07-09 20:18:37,122 - root - INFO - step: 30455 loss: 16.8832 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6337 global_avg_mtp_loss: 14.2495 +[titan] 2025-07-09 20:18:37,122 - root - INFO - lr: 2.4520e-04 gnorm: 0.78 [ 6:43:38<15:21:44] +[titan] 2025-07-09 20:18:41,094 - root - INFO - step: 30460 loss: 16.9560 memory: 44.58GiB(31.99%) tps: 82,493 tflops: 284.70 mfu: 28.79% global_avg_ntp_loss: 2.6254 global_avg_mtp_loss: 14.3306 +[titan] 2025-07-09 20:18:41,095 - root - INFO - lr: 2.4518e-04 gnorm: 0.77 [ 6:43:42<15:21:40] +[titan] 2025-07-09 20:18:45,001 - root - INFO - step: 30465 loss: 16.8865 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2700 +[titan] 2025-07-09 20:18:45,001 - root - INFO - lr: 2.4516e-04 gnorm: 0.81 [ 6:43:46<15:21:36] +[titan] 2025-07-09 20:18:48,927 - root - INFO - step: 30470 loss: 16.5963 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5626 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 20:18:48,927 - root - INFO - lr: 2.4514e-04 gnorm: 0.74 [ 6:43:50<15:21:32] +[titan] 2025-07-09 20:18:52,874 - root - INFO - step: 30475 loss: 16.5676 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0143 +[titan] 2025-07-09 20:18:52,874 - root - INFO - lr: 2.4513e-04 gnorm: 1.16 [ 6:43:54<15:21:28] +[titan] 2025-07-09 20:18:56,795 - root - INFO - step: 30480 loss: 16.6855 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-09 20:18:56,795 - root - INFO - lr: 2.4511e-04 gnorm: 0.78 [ 6:43:58<15:21:24] +[titan] 2025-07-09 20:19:00,734 - root - INFO - step: 30485 loss: 16.7141 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1247 +[titan] 2025-07-09 20:19:00,734 - root - INFO - lr: 2.4509e-04 gnorm: 0.75 [ 6:44:02<15:21:20] +[titan] 2025-07-09 20:19:04,646 - root - INFO - step: 30490 loss: 16.7303 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1459 +[titan] 2025-07-09 20:19:04,647 - root - INFO - lr: 2.4507e-04 gnorm: 0.72 [ 6:44:06<15:21:16] +[titan] 2025-07-09 20:19:08,571 - root - INFO - step: 30495 loss: 16.9434 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.3191 +[titan] 2025-07-09 20:19:08,572 - root - INFO - lr: 2.4506e-04 gnorm: 0.77 [ 6:44:10<15:21:12] +[titan] 2025-07-09 20:19:11,690 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:19:12,509 - root - INFO - step: 30500 loss: 17.0980 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6479 global_avg_mtp_loss: 14.4501 +[titan] 2025-07-09 20:19:12,509 - root - INFO - lr: 2.4504e-04 gnorm: 0.79 [ 6:44:14<15:21:08] +[titan] 2025-07-09 20:19:16,444 - root - INFO - step: 30505 loss: 16.7645 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5964 global_avg_mtp_loss: 14.1681 +[titan] 2025-07-09 20:19:16,444 - root - INFO - lr: 2.4502e-04 gnorm: 0.78 [ 6:44:18<15:21:04] +[titan] 2025-07-09 20:19:20,346 - root - INFO - step: 30510 loss: 16.7208 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.1295 +[titan] 2025-07-09 20:19:20,346 - root - INFO - lr: 2.4501e-04 gnorm: 0.77 [ 6:44:22<15:20:59] +[titan] 2025-07-09 20:19:24,254 - root - INFO - step: 30515 loss: 16.8231 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2198 +[titan] 2025-07-09 20:19:24,254 - root - INFO - lr: 2.4499e-04 gnorm: 0.76 [ 6:44:26<15:20:55] +[titan] 2025-07-09 20:19:28,158 - root - INFO - step: 30520 loss: 16.7877 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6006 global_avg_mtp_loss: 14.1871 +[titan] 2025-07-09 20:19:28,158 - root - INFO - lr: 2.4497e-04 gnorm: 0.75 [ 6:44:30<15:20:51] +[titan] 2025-07-09 20:19:32,071 - root - INFO - step: 30525 loss: 16.7921 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1862 +[titan] 2025-07-09 20:19:32,071 - root - INFO - lr: 2.4495e-04 gnorm: 0.78 [ 6:44:33<15:20:47] +[titan] 2025-07-09 20:19:35,976 - root - INFO - step: 30530 loss: 16.9145 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6347 global_avg_mtp_loss: 14.2798 +[titan] 2025-07-09 20:19:35,976 - root - INFO - lr: 2.4494e-04 gnorm: 0.77 [ 6:44:37<15:20:43] +[titan] 2025-07-09 20:19:39,874 - root - INFO - step: 30535 loss: 16.7736 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5955 global_avg_mtp_loss: 14.1782 +[titan] 2025-07-09 20:19:39,875 - root - INFO - lr: 2.4492e-04 gnorm: 0.73 [ 6:44:41<15:20:39] +[titan] 2025-07-09 20:19:43,783 - root - INFO - step: 30540 loss: 16.9039 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.2780 +[titan] 2025-07-09 20:19:43,783 - root - INFO - lr: 2.4490e-04 gnorm: 0.74 [ 6:44:45<15:20:35] +[titan] 2025-07-09 20:19:47,734 - root - INFO - step: 30545 loss: 16.6878 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.1062 +[titan] 2025-07-09 20:19:47,734 - root - INFO - lr: 2.4488e-04 gnorm: 0.76 [ 6:44:49<15:20:31] +[titan] 2025-07-09 20:19:50,847 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:19:51,633 - root - INFO - step: 30550 loss: 16.6583 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.0865 +[titan] 2025-07-09 20:19:51,634 - root - INFO - lr: 2.4487e-04 gnorm: 0.77 [ 6:44:53<15:20:26] +[titan] 2025-07-09 20:19:55,532 - root - INFO - step: 30555 loss: 16.9437 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.3161 +[titan] 2025-07-09 20:19:55,533 - root - INFO - lr: 2.4485e-04 gnorm: 0.77 [ 6:44:57<15:20:22] +[titan] 2025-07-09 20:19:59,434 - root - INFO - step: 30560 loss: 16.5964 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5590 global_avg_mtp_loss: 14.0374 +[titan] 2025-07-09 20:19:59,435 - root - INFO - lr: 2.4483e-04 gnorm: 0.79 [ 6:45:01<15:20:18] +[titan] 2025-07-09 20:20:03,390 - root - INFO - step: 30565 loss: 16.8220 memory: 44.58GiB(31.99%) tps: 82,855 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.6077 global_avg_mtp_loss: 14.2144 +[titan] 2025-07-09 20:20:03,390 - root - INFO - lr: 2.4482e-04 gnorm: 0.79 [ 6:45:05<15:20:14] +[titan] 2025-07-09 20:20:07,344 - root - INFO - step: 30570 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.8876 +[titan] 2025-07-09 20:20:07,345 - root - INFO - lr: 2.4480e-04 gnorm: 0.77 [ 6:45:09<15:20:10] +[titan] 2025-07-09 20:20:11,256 - root - INFO - step: 30575 loss: 17.0012 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6479 global_avg_mtp_loss: 14.3533 +[titan] 2025-07-09 20:20:11,256 - root - INFO - lr: 2.4478e-04 gnorm: 0.82 [ 6:45:13<15:20:06] +[titan] 2025-07-09 20:20:15,163 - root - INFO - step: 30580 loss: 16.4507 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5531 global_avg_mtp_loss: 13.8976 +[titan] 2025-07-09 20:20:15,163 - root - INFO - lr: 2.4476e-04 gnorm: 0.79 [ 6:45:17<15:20:02] +[titan] 2025-07-09 20:20:19,067 - root - INFO - step: 30585 loss: 16.7175 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.1290 +[titan] 2025-07-09 20:20:19,067 - root - INFO - lr: 2.4475e-04 gnorm: 0.74 [ 6:45:20<15:19:58] +[titan] 2025-07-09 20:20:22,995 - root - INFO - step: 30590 loss: 16.8901 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.2781 +[titan] 2025-07-09 20:20:22,996 - root - INFO - lr: 2.4473e-04 gnorm: 0.73 [ 6:45:24<15:19:54] +[titan] 2025-07-09 20:20:26,919 - root - INFO - step: 30595 loss: 16.6335 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-09 20:20:26,919 - root - INFO - lr: 2.4471e-04 gnorm: 0.73 [ 6:45:28<15:19:50] +[titan] 2025-07-09 20:20:30,094 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:20:30,881 - root - INFO - step: 30600 loss: 16.6779 memory: 44.58GiB(31.99%) tps: 82,705 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.5903 global_avg_mtp_loss: 14.0876 +[titan] 2025-07-09 20:20:30,881 - root - INFO - lr: 2.4469e-04 gnorm: 0.78 [ 6:45:32<15:19:45] +[titan] 2025-07-09 20:20:34,788 - root - INFO - step: 30605 loss: 16.7729 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6027 global_avg_mtp_loss: 14.1702 +[titan] 2025-07-09 20:20:34,789 - root - INFO - lr: 2.4468e-04 gnorm: 0.84 [ 6:45:36<15:19:41] +[titan] 2025-07-09 20:20:38,714 - root - INFO - step: 30610 loss: 16.7970 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.6144 global_avg_mtp_loss: 14.1826 +[titan] 2025-07-09 20:20:38,714 - root - INFO - lr: 2.4466e-04 gnorm: 0.77 [ 6:45:40<15:19:37] +[titan] 2025-07-09 20:20:42,637 - root - INFO - step: 30615 loss: 16.5561 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0027 +[titan] 2025-07-09 20:20:42,638 - root - INFO - lr: 2.4464e-04 gnorm: 0.77 [ 6:45:44<15:19:33] +[titan] 2025-07-09 20:20:46,537 - root - INFO - step: 30620 loss: 16.5423 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-09 20:20:46,538 - root - INFO - lr: 2.4463e-04 gnorm: 0.77 [ 6:45:48<15:19:29] +[titan] 2025-07-09 20:20:50,457 - root - INFO - step: 30625 loss: 16.5912 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.0149 +[titan] 2025-07-09 20:20:50,457 - root - INFO - lr: 2.4461e-04 gnorm: 0.79 [ 6:45:52<15:19:25] +[titan] 2025-07-09 20:20:54,362 - root - INFO - step: 30630 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 13.9848 +[titan] 2025-07-09 20:20:54,362 - root - INFO - lr: 2.4459e-04 gnorm: 0.84 [ 6:45:56<15:19:21] +[titan] 2025-07-09 20:20:58,308 - root - INFO - step: 30635 loss: 16.8364 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.2276 +[titan] 2025-07-09 20:20:58,309 - root - INFO - lr: 2.4457e-04 gnorm: 0.77 [ 6:46:00<15:19:17] +[titan] 2025-07-09 20:21:02,228 - root - INFO - step: 30640 loss: 16.8094 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.2032 +[titan] 2025-07-09 20:21:02,228 - root - INFO - lr: 2.4456e-04 gnorm: 0.76 [ 6:46:04<15:19:13] +[titan] 2025-07-09 20:21:06,133 - root - INFO - step: 30645 loss: 16.8238 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2110 +[titan] 2025-07-09 20:21:06,133 - root - INFO - lr: 2.4454e-04 gnorm: 0.72 [ 6:46:07<15:19:08] +[titan] 2025-07-09 20:21:09,267 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:21:10,064 - root - INFO - step: 30650 loss: 16.6711 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.0961 +[titan] 2025-07-09 20:21:10,064 - root - INFO - lr: 2.4452e-04 gnorm: 0.84 [ 6:46:11<15:19:04] +[titan] 2025-07-09 20:21:13,993 - root - INFO - step: 30655 loss: 16.6496 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0703 +[titan] 2025-07-09 20:21:13,993 - root - INFO - lr: 2.4450e-04 gnorm: 0.80 [ 6:46:15<15:19:00] +[titan] 2025-07-09 20:21:17,896 - root - INFO - step: 30660 loss: 16.7480 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1588 +[titan] 2025-07-09 20:21:17,897 - root - INFO - lr: 2.4449e-04 gnorm: 0.72 [ 6:46:19<15:18:56] +[titan] 2025-07-09 20:21:21,811 - root - INFO - step: 30665 loss: 16.6507 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0867 +[titan] 2025-07-09 20:21:21,811 - root - INFO - lr: 2.4447e-04 gnorm: 0.75 [ 6:46:23<15:18:52] +[titan] 2025-07-09 20:21:25,756 - root - INFO - step: 30670 loss: 16.7246 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1354 +[titan] 2025-07-09 20:21:25,756 - root - INFO - lr: 2.4445e-04 gnorm: 0.78 [ 6:46:27<15:18:48] +[titan] 2025-07-09 20:21:29,657 - root - INFO - step: 30675 loss: 16.7023 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6129 global_avg_mtp_loss: 14.0895 +[titan] 2025-07-09 20:21:29,657 - root - INFO - lr: 2.4444e-04 gnorm: 0.76 [ 6:46:31<15:18:44] +[titan] 2025-07-09 20:21:33,571 - root - INFO - step: 30680 loss: 16.8401 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.2307 +[titan] 2025-07-09 20:21:33,571 - root - INFO - lr: 2.4442e-04 gnorm: 0.80 [ 6:46:35<15:18:40] +[titan] 2025-07-09 20:21:37,489 - root - INFO - step: 30685 loss: 16.9031 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6211 global_avg_mtp_loss: 14.2820 +[titan] 2025-07-09 20:21:37,489 - root - INFO - lr: 2.4440e-04 gnorm: 0.74 [ 6:46:39<15:18:36] +[titan] 2025-07-09 20:21:41,406 - root - INFO - step: 30690 loss: 17.2231 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6845 global_avg_mtp_loss: 14.5385 +[titan] 2025-07-09 20:21:41,406 - root - INFO - lr: 2.4438e-04 gnorm: 0.71 [ 6:46:43<15:18:32] +[titan] 2025-07-09 20:21:45,325 - root - INFO - step: 30695 loss: 16.9592 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6379 global_avg_mtp_loss: 14.3213 +[titan] 2025-07-09 20:21:45,325 - root - INFO - lr: 2.4437e-04 gnorm: 0.75 [ 6:46:47<15:18:27] +[titan] 2025-07-09 20:21:48,454 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:21:49,254 - root - INFO - step: 30700 loss: 16.6412 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0714 +[titan] 2025-07-09 20:21:49,255 - root - INFO - lr: 2.4435e-04 gnorm: 0.79 [ 6:46:51<15:18:23] +[titan] 2025-07-09 20:21:53,167 - root - INFO - step: 30705 loss: 16.8102 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5930 global_avg_mtp_loss: 14.2172 +[titan] 2025-07-09 20:21:53,168 - root - INFO - lr: 2.4433e-04 gnorm: 0.74 [ 6:46:55<15:18:19] +[titan] 2025-07-09 20:21:57,134 - root - INFO - step: 30710 loss: 16.8148 memory: 44.58GiB(31.99%) tps: 82,624 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.6270 global_avg_mtp_loss: 14.1878 +[titan] 2025-07-09 20:21:57,134 - root - INFO - lr: 2.4431e-04 gnorm: 0.78 [ 6:46:58<15:18:15] +[titan] 2025-07-09 20:22:01,040 - root - INFO - step: 30715 loss: 16.6328 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5786 global_avg_mtp_loss: 14.0542 +[titan] 2025-07-09 20:22:01,040 - root - INFO - lr: 2.4430e-04 gnorm: 0.79 [ 6:47:02<15:18:11] +[titan] 2025-07-09 20:22:05,028 - root - INFO - step: 30720 loss: 17.0229 memory: 44.58GiB(31.99%) tps: 82,172 tflops: 283.59 mfu: 28.67% global_avg_ntp_loss: 2.6614 global_avg_mtp_loss: 14.3615 +[titan] 2025-07-09 20:22:05,029 - root - INFO - lr: 2.4428e-04 gnorm: 0.84 [ 6:47:06<15:18:07] +[titan] 2025-07-09 20:22:05,183 - root - INFO - Dumping profiler traces at step 30720 +[titan] 2025-07-09 20:22:05,215 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:22:09,126 - root - INFO - step: 30725 loss: 16.8315 memory: 44.58GiB(31.99%) tps: 79,965 tflops: 275.97 mfu: 27.90% global_avg_ntp_loss: 2.6076 global_avg_mtp_loss: 14.2239 +[titan] 2025-07-09 20:22:09,127 - root - INFO - lr: 2.4426e-04 gnorm: 0.79 [ 6:47:10<15:18:03] +[titan] 2025-07-09 20:22:13,046 - root - INFO - step: 30730 loss: 16.7465 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.1448 +[titan] 2025-07-09 20:22:13,046 - root - INFO - lr: 2.4424e-04 gnorm: 0.74 [ 6:47:14<15:17:59] +[titan] 2025-07-09 20:22:16,958 - root - INFO - step: 30735 loss: 17.0380 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.3960 +[titan] 2025-07-09 20:22:16,958 - root - INFO - lr: 2.4423e-04 gnorm: 0.83 [ 6:47:18<15:17:55] +[titan] 2025-07-09 20:22:20,855 - root - INFO - step: 30740 loss: 16.8428 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2300 +[titan] 2025-07-09 20:22:20,855 - root - INFO - lr: 2.4421e-04 gnorm: 0.75 [ 6:47:22<15:17:51] +[titan] 2025-07-09 20:22:24,799 - root - INFO - step: 30745 loss: 16.8705 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.2623 +[titan] 2025-07-09 20:22:24,800 - root - INFO - lr: 2.4419e-04 gnorm: 0.76 [ 6:47:26<15:17:47] +[titan] 2025-07-09 20:22:28,093 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:22:28,891 - root - INFO - step: 30750 loss: 16.8014 memory: 44.58GiB(31.99%) tps: 80,100 tflops: 276.44 mfu: 27.95% global_avg_ntp_loss: 2.6014 global_avg_mtp_loss: 14.2001 +[titan] 2025-07-09 20:22:28,891 - root - INFO - lr: 2.4418e-04 gnorm: 0.73 [ 6:47:30<15:17:43] +[titan] 2025-07-09 20:22:32,818 - root - INFO - step: 30755 loss: 16.6179 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0592 +[titan] 2025-07-09 20:22:32,818 - root - INFO - lr: 2.4416e-04 gnorm: 0.79 [ 6:47:34<15:17:39] +[titan] 2025-07-09 20:22:36,756 - root - INFO - step: 30760 loss: 16.7796 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.1531 +[titan] 2025-07-09 20:22:36,756 - root - INFO - lr: 2.4414e-04 gnorm: 2.12 [ 6:47:38<15:17:35] +[titan] 2025-07-09 20:22:40,703 - root - INFO - step: 30765 loss: 16.8346 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.2302 +[titan] 2025-07-09 20:22:40,704 - root - INFO - lr: 2.4412e-04 gnorm: 0.83 [ 6:47:42<15:17:31] +[titan] 2025-07-09 20:22:44,649 - root - INFO - step: 30770 loss: 16.7396 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.1484 +[titan] 2025-07-09 20:22:44,650 - root - INFO - lr: 2.4411e-04 gnorm: 0.78 [ 6:47:46<15:17:27] +[titan] 2025-07-09 20:22:48,544 - root - INFO - step: 30775 loss: 16.8356 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6178 global_avg_mtp_loss: 14.2179 +[titan] 2025-07-09 20:22:48,544 - root - INFO - lr: 2.4409e-04 gnorm: 0.80 [ 6:47:50<15:17:23] +[titan] 2025-07-09 20:22:52,473 - root - INFO - step: 30780 loss: 16.4827 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9460 +[titan] 2025-07-09 20:22:52,473 - root - INFO - lr: 2.4407e-04 gnorm: 0.75 [ 6:47:54<15:17:19] +[titan] 2025-07-09 20:22:56,555 - root - INFO - step: 30785 loss: 16.4720 memory: 44.58GiB(31.99%) tps: 80,286 tflops: 277.08 mfu: 28.02% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9254 +[titan] 2025-07-09 20:22:56,555 - root - INFO - lr: 2.4405e-04 gnorm: 0.85 [ 6:47:58<15:17:15] +[titan] 2025-07-09 20:23:00,477 - root - INFO - step: 30790 loss: 16.5893 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5601 global_avg_mtp_loss: 14.0292 +[titan] 2025-07-09 20:23:00,477 - root - INFO - lr: 2.4404e-04 gnorm: 0.75 [ 6:48:02<15:17:11] +[titan] 2025-07-09 20:23:04,384 - root - INFO - step: 30795 loss: 16.9821 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.3417 +[titan] 2025-07-09 20:23:04,385 - root - INFO - lr: 2.4402e-04 gnorm: 0.89 [ 6:48:06<15:17:07] +[titan] 2025-07-09 20:23:07,530 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:23:08,332 - root - INFO - step: 30800 loss: 16.4981 memory: 44.58GiB(31.99%) tps: 83,010 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 13.9416 +[titan] 2025-07-09 20:23:08,333 - root - INFO - lr: 2.4400e-04 gnorm: 0.77 [ 6:48:10<15:17:03] +[titan] 2025-07-09 20:23:12,226 - root - INFO - step: 30805 loss: 16.5828 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0259 +[titan] 2025-07-09 20:23:12,227 - root - INFO - lr: 2.4398e-04 gnorm: 0.82 [ 6:48:14<15:16:59] +[titan] 2025-07-09 20:23:16,142 - root - INFO - step: 30810 loss: 16.8497 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.2159 +[titan] 2025-07-09 20:23:16,143 - root - INFO - lr: 2.4397e-04 gnorm: 0.82 [ 6:48:17<15:16:55] +[titan] 2025-07-09 20:23:20,050 - root - INFO - step: 30815 loss: 16.5739 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0097 +[titan] 2025-07-09 20:23:20,050 - root - INFO - lr: 2.4395e-04 gnorm: 0.80 [ 6:48:21<15:16:50] +[titan] 2025-07-09 20:23:23,946 - root - INFO - step: 30820 loss: 16.6513 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0756 +[titan] 2025-07-09 20:23:23,946 - root - INFO - lr: 2.4393e-04 gnorm: 0.73 [ 6:48:25<15:16:46] +[titan] 2025-07-09 20:23:27,871 - root - INFO - step: 30825 loss: 17.0021 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.3638 +[titan] 2025-07-09 20:23:27,871 - root - INFO - lr: 2.4391e-04 gnorm: 0.78 [ 6:48:29<15:16:42] +[titan] 2025-07-09 20:23:31,784 - root - INFO - step: 30830 loss: 17.0245 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6717 global_avg_mtp_loss: 14.3528 +[titan] 2025-07-09 20:23:31,785 - root - INFO - lr: 2.4390e-04 gnorm: 0.73 [ 6:48:33<15:16:38] +[titan] 2025-07-09 20:23:35,682 - root - INFO - step: 30835 loss: 16.7607 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.1609 +[titan] 2025-07-09 20:23:35,683 - root - INFO - lr: 2.4388e-04 gnorm: 0.77 [ 6:48:37<15:16:34] +[titan] 2025-07-09 20:23:39,603 - root - INFO - step: 30840 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.1324 +[titan] 2025-07-09 20:23:39,603 - root - INFO - lr: 2.4386e-04 gnorm: 0.73 [ 6:48:41<15:16:30] +[titan] 2025-07-09 20:23:43,548 - root - INFO - step: 30845 loss: 16.8942 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2804 +[titan] 2025-07-09 20:23:43,548 - root - INFO - lr: 2.4385e-04 gnorm: 0.79 [ 6:48:45<15:16:26] +[titan] 2025-07-09 20:23:46,720 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:23:47,506 - root - INFO - step: 30850 loss: 16.9884 memory: 44.58GiB(31.99%) tps: 82,806 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.6493 global_avg_mtp_loss: 14.3391 +[titan] 2025-07-09 20:23:47,506 - root - INFO - lr: 2.4383e-04 gnorm: 0.78 [ 6:48:49<15:16:22] +[titan] 2025-07-09 20:23:51,412 - root - INFO - step: 30855 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 13.9485 +[titan] 2025-07-09 20:23:51,412 - root - INFO - lr: 2.4381e-04 gnorm: 0.77 [ 6:48:53<15:16:18] +[titan] 2025-07-09 20:23:55,312 - root - INFO - step: 30860 loss: 16.8079 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6000 global_avg_mtp_loss: 14.2078 +[titan] 2025-07-09 20:23:55,312 - root - INFO - lr: 2.4379e-04 gnorm: 0.80 [ 6:48:57<15:16:13] +[titan] 2025-07-09 20:23:59,223 - root - INFO - step: 30865 loss: 16.7588 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1620 +[titan] 2025-07-09 20:23:59,224 - root - INFO - lr: 2.4378e-04 gnorm: 0.80 [ 6:49:01<15:16:09] +[titan] 2025-07-09 20:24:03,135 - root - INFO - step: 30870 loss: 16.9610 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.6349 global_avg_mtp_loss: 14.3262 +[titan] 2025-07-09 20:24:03,135 - root - INFO - lr: 2.4376e-04 gnorm: 0.77 [ 6:49:04<15:16:05] +[titan] 2025-07-09 20:24:07,052 - root - INFO - step: 30875 loss: 16.9315 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6290 global_avg_mtp_loss: 14.3024 +[titan] 2025-07-09 20:24:07,052 - root - INFO - lr: 2.4374e-04 gnorm: 0.79 [ 6:49:08<15:16:01] +[titan] 2025-07-09 20:24:11,035 - root - INFO - step: 30880 loss: 16.7316 memory: 44.58GiB(31.99%) tps: 82,269 tflops: 283.93 mfu: 28.71% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1528 +[titan] 2025-07-09 20:24:11,035 - root - INFO - lr: 2.4372e-04 gnorm: 0.78 [ 6:49:12<15:15:57] +[titan] 2025-07-09 20:24:14,939 - root - INFO - step: 30885 loss: 16.8910 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2737 +[titan] 2025-07-09 20:24:14,939 - root - INFO - lr: 2.4371e-04 gnorm: 0.80 [ 6:49:16<15:15:53] +[titan] 2025-07-09 20:24:18,833 - root - INFO - step: 30890 loss: 16.6417 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0776 +[titan] 2025-07-09 20:24:18,834 - root - INFO - lr: 2.4369e-04 gnorm: 0.85 [ 6:49:20<15:15:49] +[titan] 2025-07-09 20:24:22,756 - root - INFO - step: 30895 loss: 16.5719 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0010 +[titan] 2025-07-09 20:24:22,756 - root - INFO - lr: 2.4367e-04 gnorm: 0.77 [ 6:49:24<15:15:45] +[titan] 2025-07-09 20:24:25,861 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:24:26,654 - root - INFO - step: 30900 loss: 16.9636 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.6333 global_avg_mtp_loss: 14.3303 +[titan] 2025-07-09 20:24:26,654 - root - INFO - lr: 2.4365e-04 gnorm: 0.75 [ 6:49:28<15:15:41] +[titan] 2025-07-09 20:24:30,571 - root - INFO - step: 30905 loss: 16.8726 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.2516 +[titan] 2025-07-09 20:24:30,571 - root - INFO - lr: 2.4364e-04 gnorm: 0.76 [ 6:49:32<15:15:37] +[titan] 2025-07-09 20:24:34,485 - root - INFO - step: 30910 loss: 16.5026 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9578 +[titan] 2025-07-09 20:24:34,485 - root - INFO - lr: 2.4362e-04 gnorm: 0.77 [ 6:49:36<15:15:32] +[titan] 2025-07-09 20:24:38,411 - root - INFO - step: 30915 loss: 16.4779 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-09 20:24:38,411 - root - INFO - lr: 2.4360e-04 gnorm: 0.85 [ 6:49:40<15:15:28] +[titan] 2025-07-09 20:24:42,345 - root - INFO - step: 30920 loss: 16.7966 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.6130 global_avg_mtp_loss: 14.1836 +[titan] 2025-07-09 20:24:42,346 - root - INFO - lr: 2.4358e-04 gnorm: 0.84 [ 6:49:44<15:15:24] +[titan] 2025-07-09 20:24:46,279 - root - INFO - step: 30925 loss: 16.7938 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.1929 +[titan] 2025-07-09 20:24:46,279 - root - INFO - lr: 2.4357e-04 gnorm: 0.77 [ 6:49:48<15:15:20] +[titan] 2025-07-09 20:24:50,177 - root - INFO - step: 30930 loss: 16.8277 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.6019 global_avg_mtp_loss: 14.2258 +[titan] 2025-07-09 20:24:50,178 - root - INFO - lr: 2.4355e-04 gnorm: 0.86 [ 6:49:51<15:15:16] +[titan] 2025-07-09 20:24:54,101 - root - INFO - step: 30935 loss: 16.7778 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.1686 +[titan] 2025-07-09 20:24:54,101 - root - INFO - lr: 2.4353e-04 gnorm: 0.79 [ 6:49:55<15:15:12] +[titan] 2025-07-09 20:24:58,036 - root - INFO - step: 30940 loss: 16.8716 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.2405 +[titan] 2025-07-09 20:24:58,037 - root - INFO - lr: 2.4351e-04 gnorm: 0.79 [ 6:49:59<15:15:08] +[titan] 2025-07-09 20:25:01,980 - root - INFO - step: 30945 loss: 16.4687 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9113 +[titan] 2025-07-09 20:25:01,981 - root - INFO - lr: 2.4350e-04 gnorm: 0.77 [ 6:50:03<15:15:04] +[titan] 2025-07-09 20:25:05,107 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:25:05,900 - root - INFO - step: 30950 loss: 16.8077 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.2069 +[titan] 2025-07-09 20:25:05,901 - root - INFO - lr: 2.4348e-04 gnorm: 0.72 [ 6:50:07<15:15:00] +[titan] 2025-07-09 20:25:09,822 - root - INFO - step: 30955 loss: 16.5834 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0162 +[titan] 2025-07-09 20:25:09,823 - root - INFO - lr: 2.4346e-04 gnorm: 0.72 [ 6:50:11<15:14:56] +[titan] 2025-07-09 20:25:13,752 - root - INFO - step: 30960 loss: 16.5983 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-09 20:25:13,752 - root - INFO - lr: 2.4344e-04 gnorm: 0.76 [ 6:50:15<15:14:52] +[titan] 2025-07-09 20:25:17,662 - root - INFO - step: 30965 loss: 16.3653 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-09 20:25:17,662 - root - INFO - lr: 2.4343e-04 gnorm: 0.75 [ 6:50:19<15:14:47] +[titan] 2025-07-09 20:25:21,609 - root - INFO - step: 30970 loss: 16.5865 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0197 +[titan] 2025-07-09 20:25:21,609 - root - INFO - lr: 2.4341e-04 gnorm: 0.81 [ 6:50:23<15:14:43] +[titan] 2025-07-09 20:25:25,520 - root - INFO - step: 30975 loss: 16.6800 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.0967 +[titan] 2025-07-09 20:25:25,520 - root - INFO - lr: 2.4339e-04 gnorm: 0.81 [ 6:50:27<15:14:39] +[titan] 2025-07-09 20:25:29,435 - root - INFO - step: 30980 loss: 16.6546 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0792 +[titan] 2025-07-09 20:25:29,435 - root - INFO - lr: 2.4338e-04 gnorm: 0.81 [ 6:50:31<15:14:35] +[titan] 2025-07-09 20:25:33,366 - root - INFO - step: 30985 loss: 16.7479 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1532 +[titan] 2025-07-09 20:25:33,366 - root - INFO - lr: 2.4336e-04 gnorm: 0.83 [ 6:50:35<15:14:31] +[titan] 2025-07-09 20:25:37,252 - root - INFO - step: 30990 loss: 16.8626 memory: 44.58GiB(31.99%) tps: 84,320 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.6119 global_avg_mtp_loss: 14.2508 +[titan] 2025-07-09 20:25:37,253 - root - INFO - lr: 2.4334e-04 gnorm: 0.77 [ 6:50:39<15:14:27] +[titan] 2025-07-09 20:25:41,155 - root - INFO - step: 30995 loss: 16.8932 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6205 global_avg_mtp_loss: 14.2727 +[titan] 2025-07-09 20:25:41,155 - root - INFO - lr: 2.4332e-04 gnorm: 0.74 [ 6:50:42<15:14:23] +[titan] 2025-07-09 20:25:44,275 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:25:45,069 - root - INFO - step: 31000 loss: 16.6638 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0984 +[titan] 2025-07-09 20:25:45,069 - root - INFO - lr: 2.4331e-04 gnorm: 0.76 [ 6:50:46<15:14:19] +[titan] 2025-07-09 20:25:49,016 - root - INFO - step: 31005 loss: 16.7330 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.1469 +[titan] 2025-07-09 20:25:49,017 - root - INFO - lr: 2.4329e-04 gnorm: 0.75 [ 6:50:50<15:14:15] +[titan] 2025-07-09 20:25:52,927 - root - INFO - step: 31010 loss: 16.7623 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.1631 +[titan] 2025-07-09 20:25:52,927 - root - INFO - lr: 2.4327e-04 gnorm: 0.78 [ 6:50:54<15:14:11] +[titan] 2025-07-09 20:25:56,822 - root - INFO - step: 31015 loss: 16.5537 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 13.9997 +[titan] 2025-07-09 20:25:56,822 - root - INFO - lr: 2.4325e-04 gnorm: 0.77 [ 6:50:58<15:14:06] +[titan] 2025-07-09 20:26:00,738 - root - INFO - step: 31020 loss: 16.6130 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0436 +[titan] 2025-07-09 20:26:00,738 - root - INFO - lr: 2.4324e-04 gnorm: 0.73 [ 6:51:02<15:14:02] +[titan] 2025-07-09 20:26:04,638 - root - INFO - step: 31025 loss: 16.9416 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6321 global_avg_mtp_loss: 14.3095 +[titan] 2025-07-09 20:26:04,638 - root - INFO - lr: 2.4322e-04 gnorm: 0.72 [ 6:51:06<15:13:58] +[titan] 2025-07-09 20:26:08,545 - root - INFO - step: 31030 loss: 16.8740 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.2551 +[titan] 2025-07-09 20:26:08,545 - root - INFO - lr: 2.4320e-04 gnorm: 0.76 [ 6:51:10<15:13:54] +[titan] 2025-07-09 20:26:12,474 - root - INFO - step: 31035 loss: 16.7190 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1387 +[titan] 2025-07-09 20:26:12,474 - root - INFO - lr: 2.4318e-04 gnorm: 0.84 [ 6:51:14<15:13:50] +[titan] 2025-07-09 20:26:16,381 - root - INFO - step: 31040 loss: 16.8220 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6001 global_avg_mtp_loss: 14.2219 +[titan] 2025-07-09 20:26:16,381 - root - INFO - lr: 2.4317e-04 gnorm: 0.75 [ 6:51:18<15:13:46] +[titan] 2025-07-09 20:26:20,308 - root - INFO - step: 31045 loss: 16.7728 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5983 global_avg_mtp_loss: 14.1746 +[titan] 2025-07-09 20:26:20,308 - root - INFO - lr: 2.4315e-04 gnorm: 0.78 [ 6:51:22<15:13:42] +[titan] 2025-07-09 20:26:23,426 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:26:24,235 - root - INFO - step: 31050 loss: 16.7422 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.1444 +[titan] 2025-07-09 20:26:24,235 - root - INFO - lr: 2.4313e-04 gnorm: 0.75 [ 6:51:26<15:13:38] +[titan] 2025-07-09 20:26:28,150 - root - INFO - step: 31055 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5970 global_avg_mtp_loss: 14.1694 +[titan] 2025-07-09 20:26:28,151 - root - INFO - lr: 2.4311e-04 gnorm: 0.75 [ 6:51:29<15:13:34] +[titan] 2025-07-09 20:26:32,048 - root - INFO - step: 31060 loss: 17.1570 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.6770 global_avg_mtp_loss: 14.4800 +[titan] 2025-07-09 20:26:32,049 - root - INFO - lr: 2.4310e-04 gnorm: 0.76 [ 6:51:33<15:13:29] +[titan] 2025-07-09 20:26:35,935 - root - INFO - step: 31065 loss: 16.5844 memory: 44.58GiB(31.99%) tps: 84,311 tflops: 290.97 mfu: 29.42% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0123 +[titan] 2025-07-09 20:26:35,936 - root - INFO - lr: 2.4308e-04 gnorm: 0.78 [ 6:51:37<15:13:25] +[titan] 2025-07-09 20:26:39,826 - root - INFO - step: 31070 loss: 16.9194 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.6354 global_avg_mtp_loss: 14.2840 +[titan] 2025-07-09 20:26:39,827 - root - INFO - lr: 2.4306e-04 gnorm: 0.78 [ 6:51:41<15:13:21] +[titan] 2025-07-09 20:26:43,755 - root - INFO - step: 31075 loss: 16.9082 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6237 global_avg_mtp_loss: 14.2845 +[titan] 2025-07-09 20:26:43,755 - root - INFO - lr: 2.4304e-04 gnorm: 0.76 [ 6:51:45<15:13:17] +[titan] 2025-07-09 20:26:47,650 - root - INFO - step: 31080 loss: 16.8618 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6169 global_avg_mtp_loss: 14.2449 +[titan] 2025-07-09 20:26:47,651 - root - INFO - lr: 2.4303e-04 gnorm: 0.79 [ 6:51:49<15:13:13] +[titan] 2025-07-09 20:26:51,616 - root - INFO - step: 31085 loss: 16.9007 memory: 44.58GiB(31.99%) tps: 82,643 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.2867 +[titan] 2025-07-09 20:26:51,616 - root - INFO - lr: 2.4301e-04 gnorm: 0.77 [ 6:51:53<15:13:09] +[titan] 2025-07-09 20:26:55,544 - root - INFO - step: 31090 loss: 16.8449 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.2341 +[titan] 2025-07-09 20:26:55,544 - root - INFO - lr: 2.4299e-04 gnorm: 0.74 [ 6:51:57<15:13:05] +[titan] 2025-07-09 20:26:59,445 - root - INFO - step: 31095 loss: 16.3937 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 13.8104 +[titan] 2025-07-09 20:26:59,445 - root - INFO - lr: 2.4297e-04 gnorm: 0.81 [ 6:52:01<15:13:01] +[titan] 2025-07-09 20:27:02,583 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:27:03,371 - root - INFO - step: 31100 loss: 16.7628 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.1558 +[titan] 2025-07-09 20:27:03,371 - root - INFO - lr: 2.4296e-04 gnorm: 0.78 [ 6:52:05<15:12:56] +[titan] 2025-07-09 20:27:07,266 - root - INFO - step: 31105 loss: 16.9616 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6494 global_avg_mtp_loss: 14.3122 +[titan] 2025-07-09 20:27:07,267 - root - INFO - lr: 2.4294e-04 gnorm: 0.75 [ 6:52:09<15:12:52] +[titan] 2025-07-09 20:27:11,155 - root - INFO - step: 31110 loss: 17.0567 memory: 44.58GiB(31.99%) tps: 84,277 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.6662 global_avg_mtp_loss: 14.3905 +[titan] 2025-07-09 20:27:11,155 - root - INFO - lr: 2.4292e-04 gnorm: 0.78 [ 6:52:12<15:12:48] +[titan] 2025-07-09 20:27:15,054 - root - INFO - step: 31115 loss: 16.4650 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9235 +[titan] 2025-07-09 20:27:15,054 - root - INFO - lr: 2.4290e-04 gnorm: 0.77 [ 6:52:16<15:12:44] +[titan] 2025-07-09 20:27:19,013 - root - INFO - step: 31120 loss: 16.9982 memory: 44.58GiB(31.99%) tps: 82,782 tflops: 285.69 mfu: 28.89% global_avg_ntp_loss: 2.6425 global_avg_mtp_loss: 14.3556 +[titan] 2025-07-09 20:27:19,013 - root - INFO - lr: 2.4289e-04 gnorm: 0.78 [ 6:52:20<15:12:40] +[titan] 2025-07-09 20:27:22,921 - root - INFO - step: 31125 loss: 16.7414 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.1656 +[titan] 2025-07-09 20:27:22,921 - root - INFO - lr: 2.4287e-04 gnorm: 0.83 [ 6:52:24<15:12:36] +[titan] 2025-07-09 20:27:26,848 - root - INFO - step: 31130 loss: 16.8814 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2543 +[titan] 2025-07-09 20:27:26,848 - root - INFO - lr: 2.4285e-04 gnorm: 0.83 [ 6:52:28<15:12:32] +[titan] 2025-07-09 20:27:30,762 - root - INFO - step: 31135 loss: 16.7416 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5817 global_avg_mtp_loss: 14.1600 +[titan] 2025-07-09 20:27:30,763 - root - INFO - lr: 2.4283e-04 gnorm: 0.80 [ 6:52:32<15:12:28] +[titan] 2025-07-09 20:27:34,655 - root - INFO - step: 31140 loss: 16.6284 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.0534 +[titan] 2025-07-09 20:27:34,655 - root - INFO - lr: 2.4282e-04 gnorm: 0.77 [ 6:52:36<15:12:24] +[titan] 2025-07-09 20:27:38,563 - root - INFO - step: 31145 loss: 16.8146 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.1956 +[titan] 2025-07-09 20:27:38,563 - root - INFO - lr: 2.4280e-04 gnorm: 0.80 [ 6:52:40<15:12:19] +[titan] 2025-07-09 20:27:41,678 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:27:42,467 - root - INFO - step: 31150 loss: 16.8102 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.2113 +[titan] 2025-07-09 20:27:42,467 - root - INFO - lr: 2.4278e-04 gnorm: 0.73 [ 6:52:44<15:12:15] +[titan] 2025-07-09 20:27:46,366 - root - INFO - step: 31155 loss: 16.9698 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6332 global_avg_mtp_loss: 14.3366 +[titan] 2025-07-09 20:27:46,366 - root - INFO - lr: 2.4276e-04 gnorm: 0.75 [ 6:52:48<15:12:11] +[titan] 2025-07-09 20:27:50,273 - root - INFO - step: 31160 loss: 16.7468 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6003 global_avg_mtp_loss: 14.1465 +[titan] 2025-07-09 20:27:50,273 - root - INFO - lr: 2.4275e-04 gnorm: 0.77 [ 6:52:52<15:12:07] +[titan] 2025-07-09 20:27:54,200 - root - INFO - step: 31165 loss: 16.5571 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5808 global_avg_mtp_loss: 13.9763 +[titan] 2025-07-09 20:27:54,200 - root - INFO - lr: 2.4273e-04 gnorm: 0.80 [ 6:52:55<15:12:03] +[titan] 2025-07-09 20:27:58,121 - root - INFO - step: 31170 loss: 16.6725 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0982 +[titan] 2025-07-09 20:27:58,121 - root - INFO - lr: 2.4271e-04 gnorm: 0.75 [ 6:52:59<15:11:59] +[titan] 2025-07-09 20:28:02,007 - root - INFO - step: 31175 loss: 16.5492 memory: 44.58GiB(31.99%) tps: 84,335 tflops: 291.05 mfu: 29.43% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 13.9844 +[titan] 2025-07-09 20:28:02,007 - root - INFO - lr: 2.4269e-04 gnorm: 0.75 [ 6:53:03<15:11:55] +[titan] 2025-07-09 20:28:05,949 - root - INFO - step: 31180 loss: 16.9329 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.6451 global_avg_mtp_loss: 14.2879 +[titan] 2025-07-09 20:28:05,949 - root - INFO - lr: 2.4268e-04 gnorm: 0.82 [ 6:53:07<15:11:51] +[titan] 2025-07-09 20:28:09,868 - root - INFO - step: 31185 loss: 16.7195 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1271 +[titan] 2025-07-09 20:28:09,868 - root - INFO - lr: 2.4266e-04 gnorm: 0.79 [ 6:53:11<15:11:47] +[titan] 2025-07-09 20:28:13,819 - root - INFO - step: 31190 loss: 16.5530 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5568 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-09 20:28:13,819 - root - INFO - lr: 2.4264e-04 gnorm: 0.77 [ 6:53:15<15:11:42] +[titan] 2025-07-09 20:28:17,741 - root - INFO - step: 31195 loss: 16.8534 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6074 global_avg_mtp_loss: 14.2460 +[titan] 2025-07-09 20:28:17,741 - root - INFO - lr: 2.4262e-04 gnorm: 0.77 [ 6:53:19<15:11:38] +[titan] 2025-07-09 20:28:20,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:28:21,670 - root - INFO - step: 31200 loss: 16.8211 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.2287 +[titan] 2025-07-09 20:28:21,670 - root - INFO - lr: 2.4261e-04 gnorm: 0.85 [ 6:53:23<15:11:34] +[titan] 2025-07-09 20:28:25,579 - root - INFO - step: 31205 loss: 16.8003 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.1916 +[titan] 2025-07-09 20:28:25,579 - root - INFO - lr: 2.4259e-04 gnorm: 0.92 [ 6:53:27<15:11:30] +[titan] 2025-07-09 20:28:29,482 - root - INFO - step: 31210 loss: 16.7684 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.1706 +[titan] 2025-07-09 20:28:29,483 - root - INFO - lr: 2.4257e-04 gnorm: 0.75 [ 6:53:31<15:11:26] +[titan] 2025-07-09 20:28:33,410 - root - INFO - step: 31215 loss: 16.7437 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1603 +[titan] 2025-07-09 20:28:33,410 - root - INFO - lr: 2.4255e-04 gnorm: 0.81 [ 6:53:35<15:11:22] +[titan] 2025-07-09 20:28:37,340 - root - INFO - step: 31220 loss: 16.8788 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6408 global_avg_mtp_loss: 14.2380 +[titan] 2025-07-09 20:28:37,340 - root - INFO - lr: 2.4254e-04 gnorm: 0.76 [ 6:53:39<15:11:18] +[titan] 2025-07-09 20:28:41,237 - root - INFO - step: 31225 loss: 16.7195 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.1343 +[titan] 2025-07-09 20:28:41,237 - root - INFO - lr: 2.4252e-04 gnorm: 0.81 [ 6:53:43<15:11:14] +[titan] 2025-07-09 20:28:45,233 - root - INFO - step: 31230 loss: 16.5107 memory: 44.58GiB(31.99%) tps: 82,021 tflops: 283.07 mfu: 28.62% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9640 +[titan] 2025-07-09 20:28:45,233 - root - INFO - lr: 2.4250e-04 gnorm: 0.81 [ 6:53:47<15:11:10] +[titan] 2025-07-09 20:28:46,963 - root - INFO - Dumping profiler traces at step 31232 +[titan] 2025-07-09 20:28:46,994 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:28:49,350 - root - INFO - step: 31235 loss: 16.5581 memory: 44.58GiB(31.99%) tps: 79,590 tflops: 274.68 mfu: 27.77% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 14.0014 +[titan] 2025-07-09 20:28:49,350 - root - INFO - lr: 2.4248e-04 gnorm: 0.75 [ 6:53:51<15:11:06] +[titan] 2025-07-09 20:28:53,250 - root - INFO - step: 31240 loss: 16.7528 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5850 global_avg_mtp_loss: 14.1678 +[titan] 2025-07-09 20:28:53,250 - root - INFO - lr: 2.4247e-04 gnorm: 0.80 [ 6:53:55<15:11:02] +[titan] 2025-07-09 20:28:57,156 - root - INFO - step: 31245 loss: 16.7885 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1873 +[titan] 2025-07-09 20:28:57,156 - root - INFO - lr: 2.4245e-04 gnorm: 0.72 [ 6:53:58<15:10:58] +[titan] 2025-07-09 20:29:00,270 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:29:01,059 - root - INFO - step: 31250 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1170 +[titan] 2025-07-09 20:29:01,059 - root - INFO - lr: 2.4243e-04 gnorm: 0.81 [ 6:54:02<15:10:54] +[titan] 2025-07-09 20:29:04,961 - root - INFO - step: 31255 loss: 16.9607 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.6249 global_avg_mtp_loss: 14.3358 +[titan] 2025-07-09 20:29:04,962 - root - INFO - lr: 2.4241e-04 gnorm: 0.77 [ 6:54:06<15:10:50] +[titan] 2025-07-09 20:29:08,879 - root - INFO - step: 31260 loss: 16.5610 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 13.9991 +[titan] 2025-07-09 20:29:08,879 - root - INFO - lr: 2.4240e-04 gnorm: 0.80 [ 6:54:10<15:10:46] +[titan] 2025-07-09 20:29:12,855 - root - INFO - step: 31265 loss: 16.6941 memory: 44.58GiB(31.99%) tps: 82,417 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.1151 +[titan] 2025-07-09 20:29:12,855 - root - INFO - lr: 2.4238e-04 gnorm: 0.73 [ 6:54:14<15:10:42] +[titan] 2025-07-09 20:29:16,791 - root - INFO - step: 31270 loss: 16.8665 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2500 +[titan] 2025-07-09 20:29:16,791 - root - INFO - lr: 2.4236e-04 gnorm: 0.80 [ 6:54:18<15:10:37] +[titan] 2025-07-09 20:29:20,727 - root - INFO - step: 31275 loss: 16.6044 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0317 +[titan] 2025-07-09 20:29:20,727 - root - INFO - lr: 2.4234e-04 gnorm: 0.82 [ 6:54:22<15:10:33] +[titan] 2025-07-09 20:29:24,674 - root - INFO - step: 31280 loss: 16.7378 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1421 +[titan] 2025-07-09 20:29:24,674 - root - INFO - lr: 2.4233e-04 gnorm: 0.80 [ 6:54:26<15:10:29] +[titan] 2025-07-09 20:29:28,582 - root - INFO - step: 31285 loss: 16.9825 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6473 global_avg_mtp_loss: 14.3353 +[titan] 2025-07-09 20:29:28,582 - root - INFO - lr: 2.4231e-04 gnorm: 0.89 [ 6:54:30<15:10:25] +[titan] 2025-07-09 20:29:32,511 - root - INFO - step: 31290 loss: 16.7239 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1320 +[titan] 2025-07-09 20:29:32,512 - root - INFO - lr: 2.4229e-04 gnorm: 0.78 [ 6:54:34<15:10:21] +[titan] 2025-07-09 20:29:36,423 - root - INFO - step: 31295 loss: 16.5987 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 14.0379 +[titan] 2025-07-09 20:29:36,423 - root - INFO - lr: 2.4227e-04 gnorm: 0.74 [ 6:54:38<15:10:17] +[titan] 2025-07-09 20:29:39,545 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:29:40,332 - root - INFO - step: 31300 loss: 16.9249 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.6375 global_avg_mtp_loss: 14.2873 +[titan] 2025-07-09 20:29:40,333 - root - INFO - lr: 2.4225e-04 gnorm: 0.76 [ 6:54:42<15:10:13] +[titan] 2025-07-09 20:29:44,245 - root - INFO - step: 31305 loss: 16.8227 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.2152 +[titan] 2025-07-09 20:29:44,245 - root - INFO - lr: 2.4224e-04 gnorm: 0.70 [ 6:54:46<15:10:09] +[titan] 2025-07-09 20:29:48,187 - root - INFO - step: 31310 loss: 16.8096 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.6035 global_avg_mtp_loss: 14.2061 +[titan] 2025-07-09 20:29:48,188 - root - INFO - lr: 2.4222e-04 gnorm: 0.77 [ 6:54:49<15:10:05] +[titan] 2025-07-09 20:29:52,093 - root - INFO - step: 31315 loss: 16.6089 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.0336 +[titan] 2025-07-09 20:29:52,093 - root - INFO - lr: 2.4220e-04 gnorm: 0.87 [ 6:54:53<15:10:01] +[titan] 2025-07-09 20:29:55,997 - root - INFO - step: 31320 loss: 16.8106 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.1896 +[titan] 2025-07-09 20:29:55,997 - root - INFO - lr: 2.4218e-04 gnorm: 0.71 [ 6:54:57<15:09:57] +[titan] 2025-07-09 20:29:59,924 - root - INFO - step: 31325 loss: 16.9228 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.2857 +[titan] 2025-07-09 20:29:59,924 - root - INFO - lr: 2.4217e-04 gnorm: 0.78 [ 6:55:01<15:09:52] +[titan] 2025-07-09 20:30:03,818 - root - INFO - step: 31330 loss: 16.5871 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0278 +[titan] 2025-07-09 20:30:03,818 - root - INFO - lr: 2.4215e-04 gnorm: 0.77 [ 6:55:05<15:09:48] +[titan] 2025-07-09 20:30:07,749 - root - INFO - step: 31335 loss: 16.5837 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.0087 +[titan] 2025-07-09 20:30:07,749 - root - INFO - lr: 2.4213e-04 gnorm: 0.74 [ 6:55:09<15:09:44] +[titan] 2025-07-09 20:30:11,659 - root - INFO - step: 31340 loss: 16.8607 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.2366 +[titan] 2025-07-09 20:30:11,659 - root - INFO - lr: 2.4211e-04 gnorm: 0.75 [ 6:55:13<15:09:40] +[titan] 2025-07-09 20:30:15,554 - root - INFO - step: 31345 loss: 16.8916 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6287 global_avg_mtp_loss: 14.2628 +[titan] 2025-07-09 20:30:15,554 - root - INFO - lr: 2.4210e-04 gnorm: 0.75 [ 6:55:17<15:09:36] +[titan] 2025-07-09 20:30:18,672 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:30:19,473 - root - INFO - step: 31350 loss: 16.7958 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.1895 +[titan] 2025-07-09 20:30:19,473 - root - INFO - lr: 2.4208e-04 gnorm: 0.77 [ 6:55:21<15:09:32] +[titan] 2025-07-09 20:30:23,389 - root - INFO - step: 31355 loss: 16.6140 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.0436 +[titan] 2025-07-09 20:30:23,389 - root - INFO - lr: 2.4206e-04 gnorm: 0.76 [ 6:55:25<15:09:28] +[titan] 2025-07-09 20:30:27,325 - root - INFO - step: 31360 loss: 16.8922 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6167 global_avg_mtp_loss: 14.2755 +[titan] 2025-07-09 20:30:27,325 - root - INFO - lr: 2.4204e-04 gnorm: 0.82 [ 6:55:29<15:09:24] +[titan] 2025-07-09 20:30:31,244 - root - INFO - step: 31365 loss: 16.7368 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6073 global_avg_mtp_loss: 14.1294 +[titan] 2025-07-09 20:30:31,244 - root - INFO - lr: 2.4203e-04 gnorm: 0.75 [ 6:55:33<15:09:20] +[titan] 2025-07-09 20:30:35,170 - root - INFO - step: 31370 loss: 16.7006 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.1172 +[titan] 2025-07-09 20:30:35,170 - root - INFO - lr: 2.4201e-04 gnorm: 0.77 [ 6:55:36<15:09:16] +[titan] 2025-07-09 20:30:39,077 - root - INFO - step: 31375 loss: 16.8688 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2515 +[titan] 2025-07-09 20:30:39,077 - root - INFO - lr: 2.4199e-04 gnorm: 0.81 [ 6:55:40<15:09:11] +[titan] 2025-07-09 20:30:42,983 - root - INFO - step: 31380 loss: 16.4432 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9155 +[titan] 2025-07-09 20:30:42,984 - root - INFO - lr: 2.4197e-04 gnorm: 0.74 [ 6:55:44<15:09:07] +[titan] 2025-07-09 20:30:46,922 - root - INFO - step: 31385 loss: 16.4638 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9209 +[titan] 2025-07-09 20:30:46,922 - root - INFO - lr: 2.4196e-04 gnorm: 0.76 [ 6:55:48<15:09:03] +[titan] 2025-07-09 20:30:50,888 - root - INFO - step: 31390 loss: 16.6416 memory: 44.58GiB(31.99%) tps: 82,629 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0721 +[titan] 2025-07-09 20:30:50,888 - root - INFO - lr: 2.4194e-04 gnorm: 0.74 [ 6:55:52<15:08:59] +[titan] 2025-07-09 20:30:54,806 - root - INFO - step: 31395 loss: 16.9639 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.3275 +[titan] 2025-07-09 20:30:54,807 - root - INFO - lr: 2.4192e-04 gnorm: 0.84 [ 6:55:56<15:08:55] +[titan] 2025-07-09 20:30:57,928 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:30:58,716 - root - INFO - step: 31400 loss: 16.6737 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1033 +[titan] 2025-07-09 20:30:58,716 - root - INFO - lr: 2.4190e-04 gnorm: 0.79 [ 6:56:00<15:08:51] +[titan] 2025-07-09 20:31:02,623 - root - INFO - step: 31405 loss: 16.8974 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6326 global_avg_mtp_loss: 14.2648 +[titan] 2025-07-09 20:31:02,623 - root - INFO - lr: 2.4189e-04 gnorm: 0.74 [ 6:56:04<15:08:47] +[titan] 2025-07-09 20:31:06,525 - root - INFO - step: 31410 loss: 16.7781 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.1788 +[titan] 2025-07-09 20:31:06,526 - root - INFO - lr: 2.4187e-04 gnorm: 0.78 [ 6:56:08<15:08:43] +[titan] 2025-07-09 20:31:10,467 - root - INFO - step: 31415 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0060 +[titan] 2025-07-09 20:31:10,467 - root - INFO - lr: 2.4185e-04 gnorm: 0.75 [ 6:56:12<15:08:39] +[titan] 2025-07-09 20:31:14,396 - root - INFO - step: 31420 loss: 16.8990 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.6335 global_avg_mtp_loss: 14.2655 +[titan] 2025-07-09 20:31:14,397 - root - INFO - lr: 2.4183e-04 gnorm: 0.74 [ 6:56:16<15:08:35] +[titan] 2025-07-09 20:31:18,353 - root - INFO - step: 31425 loss: 16.6242 memory: 44.58GiB(31.99%) tps: 82,827 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.5877 global_avg_mtp_loss: 14.0365 +[titan] 2025-07-09 20:31:18,353 - root - INFO - lr: 2.4182e-04 gnorm: 0.76 [ 6:56:20<15:08:31] +[titan] 2025-07-09 20:31:22,282 - root - INFO - step: 31430 loss: 16.8348 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6196 global_avg_mtp_loss: 14.2151 +[titan] 2025-07-09 20:31:22,282 - root - INFO - lr: 2.4180e-04 gnorm: 0.74 [ 6:56:24<15:08:27] +[titan] 2025-07-09 20:31:26,218 - root - INFO - step: 31435 loss: 16.5647 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.06% global_avg_ntp_loss: 2.5618 global_avg_mtp_loss: 14.0029 +[titan] 2025-07-09 20:31:26,218 - root - INFO - lr: 2.4178e-04 gnorm: 0.76 [ 6:56:27<15:08:22] +[titan] 2025-07-09 20:31:30,154 - root - INFO - step: 31440 loss: 16.7094 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5985 global_avg_mtp_loss: 14.1109 +[titan] 2025-07-09 20:31:30,154 - root - INFO - lr: 2.4176e-04 gnorm: 0.76 [ 6:56:31<15:08:18] +[titan] 2025-07-09 20:31:34,074 - root - INFO - step: 31445 loss: 16.8441 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.2354 +[titan] 2025-07-09 20:31:34,074 - root - INFO - lr: 2.4174e-04 gnorm: 0.90 [ 6:56:35<15:08:14] +[titan] 2025-07-09 20:31:37,232 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:31:38,043 - root - INFO - step: 31450 loss: 16.9535 memory: 44.58GiB(31.99%) tps: 82,558 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.6423 global_avg_mtp_loss: 14.3112 +[titan] 2025-07-09 20:31:38,043 - root - INFO - lr: 2.4173e-04 gnorm: 0.75 [ 6:56:39<15:08:10] +[titan] 2025-07-09 20:31:41,973 - root - INFO - step: 31455 loss: 16.7932 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.1915 +[titan] 2025-07-09 20:31:41,973 - root - INFO - lr: 2.4171e-04 gnorm: 0.75 [ 6:56:43<15:08:06] +[titan] 2025-07-09 20:31:45,869 - root - INFO - step: 31460 loss: 16.6455 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.0735 +[titan] 2025-07-09 20:31:45,870 - root - INFO - lr: 2.4169e-04 gnorm: 0.77 [ 6:56:47<15:08:02] +[titan] 2025-07-09 20:31:49,782 - root - INFO - step: 31465 loss: 16.6676 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0953 +[titan] 2025-07-09 20:31:49,783 - root - INFO - lr: 2.4167e-04 gnorm: 0.77 [ 6:56:51<15:07:58] +[titan] 2025-07-09 20:31:53,688 - root - INFO - step: 31470 loss: 17.0689 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6948 global_avg_mtp_loss: 14.3741 +[titan] 2025-07-09 20:31:53,688 - root - INFO - lr: 2.4166e-04 gnorm: 0.94 [ 6:56:55<15:07:54] +[titan] 2025-07-09 20:31:57,620 - root - INFO - step: 31475 loss: 16.8907 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2685 +[titan] 2025-07-09 20:31:57,621 - root - INFO - lr: 2.4164e-04 gnorm: 0.88 [ 6:56:59<15:07:50] +[titan] 2025-07-09 20:32:01,525 - root - INFO - step: 31480 loss: 16.6480 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0889 +[titan] 2025-07-09 20:32:01,525 - root - INFO - lr: 2.4162e-04 gnorm: 0.81 [ 6:57:03<15:07:46] +[titan] 2025-07-09 20:32:05,482 - root - INFO - step: 31485 loss: 16.8238 memory: 44.58GiB(31.99%) tps: 82,825 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.2125 +[titan] 2025-07-09 20:32:05,482 - root - INFO - lr: 2.4160e-04 gnorm: 0.78 [ 6:57:07<15:07:42] +[titan] 2025-07-09 20:32:09,424 - root - INFO - step: 31490 loss: 16.6747 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.0996 +[titan] 2025-07-09 20:32:09,424 - root - INFO - lr: 2.4159e-04 gnorm: 0.87 [ 6:57:11<15:07:38] +[titan] 2025-07-09 20:32:13,352 - root - INFO - step: 31495 loss: 16.9147 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.2734 +[titan] 2025-07-09 20:32:13,352 - root - INFO - lr: 2.4157e-04 gnorm: 0.79 [ 6:57:15<15:07:34] +[titan] 2025-07-09 20:32:16,490 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:32:17,287 - root - INFO - step: 31500 loss: 16.6154 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0486 +[titan] 2025-07-09 20:32:17,288 - root - INFO - lr: 2.4155e-04 gnorm: 0.76 [ 6:57:19<15:07:30] +[titan] 2025-07-09 20:32:21,222 - root - INFO - step: 31505 loss: 16.4856 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-09 20:32:21,222 - root - INFO - lr: 2.4153e-04 gnorm: 0.71 [ 6:57:22<15:07:25] +[titan] 2025-07-09 20:32:25,167 - root - INFO - step: 31510 loss: 16.9163 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.6325 global_avg_mtp_loss: 14.2838 +[titan] 2025-07-09 20:32:25,167 - root - INFO - lr: 2.4152e-04 gnorm: 0.78 [ 6:57:26<15:07:21] +[titan] 2025-07-09 20:32:29,096 - root - INFO - step: 31515 loss: 16.4264 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 13.8728 +[titan] 2025-07-09 20:32:29,097 - root - INFO - lr: 2.4150e-04 gnorm: 0.78 [ 6:57:30<15:07:17] +[titan] 2025-07-09 20:32:33,032 - root - INFO - step: 31520 loss: 16.4716 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 13.9256 +[titan] 2025-07-09 20:32:33,032 - root - INFO - lr: 2.4148e-04 gnorm: 0.81 [ 6:57:34<15:07:13] +[titan] 2025-07-09 20:32:36,975 - root - INFO - step: 31525 loss: 16.8254 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.6099 global_avg_mtp_loss: 14.2155 +[titan] 2025-07-09 20:32:36,976 - root - INFO - lr: 2.4146e-04 gnorm: 0.76 [ 6:57:38<15:07:09] +[titan] 2025-07-09 20:32:40,881 - root - INFO - step: 31530 loss: 16.7709 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1857 +[titan] 2025-07-09 20:32:40,881 - root - INFO - lr: 2.4144e-04 gnorm: 0.77 [ 6:57:42<15:07:05] +[titan] 2025-07-09 20:32:44,786 - root - INFO - step: 31535 loss: 16.7362 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.1474 +[titan] 2025-07-09 20:32:44,786 - root - INFO - lr: 2.4143e-04 gnorm: 0.72 [ 6:57:46<15:07:01] +[titan] 2025-07-09 20:32:48,712 - root - INFO - step: 31540 loss: 16.7707 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1765 +[titan] 2025-07-09 20:32:48,712 - root - INFO - lr: 2.4141e-04 gnorm: 0.89 [ 6:57:50<15:06:57] +[titan] 2025-07-09 20:32:52,631 - root - INFO - step: 31545 loss: 16.9378 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.6364 global_avg_mtp_loss: 14.3014 +[titan] 2025-07-09 20:32:52,631 - root - INFO - lr: 2.4139e-04 gnorm: 0.76 [ 6:57:54<15:06:53] +[titan] 2025-07-09 20:32:55,739 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:32:56,534 - root - INFO - step: 31550 loss: 17.0333 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.6698 global_avg_mtp_loss: 14.3635 +[titan] 2025-07-09 20:32:56,535 - root - INFO - lr: 2.4137e-04 gnorm: 0.79 [ 6:57:58<15:06:49] +[titan] 2025-07-09 20:33:00,450 - root - INFO - step: 31555 loss: 16.7084 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1237 +[titan] 2025-07-09 20:33:00,450 - root - INFO - lr: 2.4136e-04 gnorm: 0.73 [ 6:58:02<15:06:45] +[titan] 2025-07-09 20:33:04,387 - root - INFO - step: 31560 loss: 16.6960 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1114 +[titan] 2025-07-09 20:33:04,388 - root - INFO - lr: 2.4134e-04 gnorm: 0.78 [ 6:58:06<15:06:41] +[titan] 2025-07-09 20:33:08,311 - root - INFO - step: 31565 loss: 16.9549 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6362 global_avg_mtp_loss: 14.3187 +[titan] 2025-07-09 20:33:08,312 - root - INFO - lr: 2.4132e-04 gnorm: 0.75 [ 6:58:10<15:06:36] +[titan] 2025-07-09 20:33:12,244 - root - INFO - step: 31570 loss: 16.6275 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0651 +[titan] 2025-07-09 20:33:12,244 - root - INFO - lr: 2.4130e-04 gnorm: 0.73 [ 6:58:14<15:06:32] +[titan] 2025-07-09 20:33:16,194 - root - INFO - step: 31575 loss: 16.8676 memory: 44.58GiB(31.99%) tps: 82,966 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.2429 +[titan] 2025-07-09 20:33:16,194 - root - INFO - lr: 2.4129e-04 gnorm: 0.77 [ 6:58:17<15:06:28] +[titan] 2025-07-09 20:33:20,110 - root - INFO - step: 31580 loss: 16.8109 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.1969 +[titan] 2025-07-09 20:33:20,111 - root - INFO - lr: 2.4127e-04 gnorm: 0.82 [ 6:58:21<15:06:24] +[titan] 2025-07-09 20:33:24,007 - root - INFO - step: 31585 loss: 16.7244 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5871 global_avg_mtp_loss: 14.1373 +[titan] 2025-07-09 20:33:24,007 - root - INFO - lr: 2.4125e-04 gnorm: 0.80 [ 6:58:25<15:06:20] +[titan] 2025-07-09 20:33:27,911 - root - INFO - step: 31590 loss: 16.7902 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5909 global_avg_mtp_loss: 14.1993 +[titan] 2025-07-09 20:33:27,911 - root - INFO - lr: 2.4123e-04 gnorm: 0.84 [ 6:58:29<15:06:16] +[titan] 2025-07-09 20:33:31,822 - root - INFO - step: 31595 loss: 16.6714 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0897 +[titan] 2025-07-09 20:33:31,822 - root - INFO - lr: 2.4122e-04 gnorm: 0.79 [ 6:58:33<15:06:12] +[titan] 2025-07-09 20:33:34,953 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:33:35,745 - root - INFO - step: 31600 loss: 16.6431 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.0515 +[titan] 2025-07-09 20:33:35,745 - root - INFO - lr: 2.4120e-04 gnorm: 0.72 [ 6:58:37<15:06:08] +[titan] 2025-07-09 20:33:39,666 - root - INFO - step: 31605 loss: 16.9324 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.3005 +[titan] 2025-07-09 20:33:39,666 - root - INFO - lr: 2.4118e-04 gnorm: 0.76 [ 6:58:41<15:06:04] +[titan] 2025-07-09 20:33:43,567 - root - INFO - step: 31610 loss: 16.7645 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5922 global_avg_mtp_loss: 14.1722 +[titan] 2025-07-09 20:33:43,568 - root - INFO - lr: 2.4116e-04 gnorm: 0.77 [ 6:58:45<15:06:00] +[titan] 2025-07-09 20:33:47,505 - root - INFO - step: 31615 loss: 16.9496 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6403 global_avg_mtp_loss: 14.3093 +[titan] 2025-07-09 20:33:47,505 - root - INFO - lr: 2.4114e-04 gnorm: 0.77 [ 6:58:49<15:05:55] +[titan] 2025-07-09 20:33:51,410 - root - INFO - step: 31620 loss: 16.6966 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.1189 +[titan] 2025-07-09 20:33:51,410 - root - INFO - lr: 2.4113e-04 gnorm: 0.75 [ 6:58:53<15:05:51] +[titan] 2025-07-09 20:33:55,372 - root - INFO - step: 31625 loss: 16.9136 memory: 44.58GiB(31.99%) tps: 82,708 tflops: 285.44 mfu: 28.86% global_avg_ntp_loss: 2.6498 global_avg_mtp_loss: 14.2639 +[titan] 2025-07-09 20:33:55,372 - root - INFO - lr: 2.4111e-04 gnorm: 0.79 [ 6:58:57<15:05:47] +[titan] 2025-07-09 20:33:59,302 - root - INFO - step: 31630 loss: 16.6874 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 20:33:59,303 - root - INFO - lr: 2.4109e-04 gnorm: 0.76 [ 6:59:01<15:05:43] +[titan] 2025-07-09 20:34:03,213 - root - INFO - step: 31635 loss: 16.6616 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.0847 +[titan] 2025-07-09 20:34:03,213 - root - INFO - lr: 2.4107e-04 gnorm: 0.81 [ 6:59:04<15:05:39] +[titan] 2025-07-09 20:34:07,162 - root - INFO - step: 31640 loss: 16.8229 memory: 44.58GiB(31.99%) tps: 82,986 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.6203 global_avg_mtp_loss: 14.2026 +[titan] 2025-07-09 20:34:07,162 - root - INFO - lr: 2.4106e-04 gnorm: 0.75 [ 6:59:08<15:05:35] +[titan] 2025-07-09 20:34:11,106 - root - INFO - step: 31645 loss: 16.7809 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.1832 +[titan] 2025-07-09 20:34:11,106 - root - INFO - lr: 2.4104e-04 gnorm: 0.76 [ 6:59:12<15:05:31] +[titan] 2025-07-09 20:34:14,230 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:34:15,025 - root - INFO - step: 31650 loss: 16.6766 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.0942 +[titan] 2025-07-09 20:34:15,026 - root - INFO - lr: 2.4102e-04 gnorm: 0.75 [ 6:59:16<15:05:27] +[titan] 2025-07-09 20:34:18,934 - root - INFO - step: 31655 loss: 16.6953 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1078 +[titan] 2025-07-09 20:34:18,935 - root - INFO - lr: 2.4100e-04 gnorm: 0.73 [ 6:59:20<15:05:23] +[titan] 2025-07-09 20:34:22,839 - root - INFO - step: 31660 loss: 16.7286 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5876 global_avg_mtp_loss: 14.1410 +[titan] 2025-07-09 20:34:22,839 - root - INFO - lr: 2.4099e-04 gnorm: 0.77 [ 6:59:24<15:05:19] +[titan] 2025-07-09 20:34:26,748 - root - INFO - step: 31665 loss: 16.8725 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.6263 global_avg_mtp_loss: 14.2462 +[titan] 2025-07-09 20:34:26,748 - root - INFO - lr: 2.4097e-04 gnorm: 0.81 [ 6:59:28<15:05:15] +[titan] 2025-07-09 20:34:30,691 - root - INFO - step: 31670 loss: 17.0567 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6665 global_avg_mtp_loss: 14.3902 +[titan] 2025-07-09 20:34:30,691 - root - INFO - lr: 2.4095e-04 gnorm: 0.76 [ 6:59:32<15:05:11] +[titan] 2025-07-09 20:34:34,594 - root - INFO - step: 31675 loss: 16.9220 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6300 global_avg_mtp_loss: 14.2920 +[titan] 2025-07-09 20:34:34,594 - root - INFO - lr: 2.4093e-04 gnorm: 0.79 [ 6:59:36<15:05:06] +[titan] 2025-07-09 20:34:38,507 - root - INFO - step: 31680 loss: 16.8794 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.2671 +[titan] 2025-07-09 20:34:38,507 - root - INFO - lr: 2.4091e-04 gnorm: 0.76 [ 6:59:40<15:05:02] +[titan] 2025-07-09 20:34:42,440 - root - INFO - step: 31685 loss: 16.7633 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1695 +[titan] 2025-07-09 20:34:42,440 - root - INFO - lr: 2.4090e-04 gnorm: 0.80 [ 6:59:44<15:04:58] +[titan] 2025-07-09 20:34:46,384 - root - INFO - step: 31690 loss: 16.8566 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6220 global_avg_mtp_loss: 14.2346 +[titan] 2025-07-09 20:34:46,384 - root - INFO - lr: 2.4088e-04 gnorm: 0.75 [ 6:59:48<15:04:54] +[titan] 2025-07-09 20:34:50,325 - root - INFO - step: 31695 loss: 16.7441 memory: 44.58GiB(31.99%) tps: 83,140 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5903 global_avg_mtp_loss: 14.1538 +[titan] 2025-07-09 20:34:50,326 - root - INFO - lr: 2.4086e-04 gnorm: 0.77 [ 6:59:52<15:04:50] +[titan] 2025-07-09 20:34:53,465 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:34:54,246 - root - INFO - step: 31700 loss: 16.7921 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5985 global_avg_mtp_loss: 14.1936 +[titan] 2025-07-09 20:34:54,246 - root - INFO - lr: 2.4084e-04 gnorm: 0.75 [ 6:59:55<15:04:46] +[titan] 2025-07-09 20:34:58,157 - root - INFO - step: 31705 loss: 16.7036 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5880 global_avg_mtp_loss: 14.1156 +[titan] 2025-07-09 20:34:58,158 - root - INFO - lr: 2.4083e-04 gnorm: 0.86 [ 6:59:59<15:04:42] +[titan] 2025-07-09 20:35:02,076 - root - INFO - step: 31710 loss: 16.4465 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 13.8885 +[titan] 2025-07-09 20:35:02,076 - root - INFO - lr: 2.4081e-04 gnorm: 0.82 [ 7:00:03<15:04:38] +[titan] 2025-07-09 20:35:05,973 - root - INFO - step: 31715 loss: 16.7692 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1770 +[titan] 2025-07-09 20:35:05,973 - root - INFO - lr: 2.4079e-04 gnorm: 0.74 [ 7:00:07<15:04:34] +[titan] 2025-07-09 20:35:09,875 - root - INFO - step: 31720 loss: 17.1421 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.7021 global_avg_mtp_loss: 14.4400 +[titan] 2025-07-09 20:35:09,875 - root - INFO - lr: 2.4077e-04 gnorm: 0.85 [ 7:00:11<15:04:30] +[titan] 2025-07-09 20:35:13,796 - root - INFO - step: 31725 loss: 16.6447 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-09 20:35:13,797 - root - INFO - lr: 2.4075e-04 gnorm: 0.79 [ 7:00:15<15:04:26] +[titan] 2025-07-09 20:35:17,732 - root - INFO - step: 31730 loss: 16.6123 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.0229 +[titan] 2025-07-09 20:35:17,732 - root - INFO - lr: 2.4074e-04 gnorm: 0.78 [ 7:00:19<15:04:22] +[titan] 2025-07-09 20:35:21,669 - root - INFO - step: 31735 loss: 16.9626 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.6316 global_avg_mtp_loss: 14.3310 +[titan] 2025-07-09 20:35:21,669 - root - INFO - lr: 2.4072e-04 gnorm: 0.88 [ 7:00:23<15:04:17] +[titan] 2025-07-09 20:35:25,578 - root - INFO - step: 31740 loss: 16.6073 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.0271 +[titan] 2025-07-09 20:35:25,578 - root - INFO - lr: 2.4070e-04 gnorm: 0.76 [ 7:00:27<15:04:13] +[titan] 2025-07-09 20:35:28,945 - root - INFO - Dumping profiler traces at step 31744 +[titan] 2025-07-09 20:35:28,977 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:35:29,772 - root - INFO - step: 31745 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 78,125 tflops: 269.62 mfu: 27.26% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 14.0670 +[titan] 2025-07-09 20:35:29,773 - root - INFO - lr: 2.4068e-04 gnorm: 0.84 [ 7:00:31<15:04:10] +[titan] 2025-07-09 20:35:32,885 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:35:33,706 - root - INFO - step: 31750 loss: 16.6638 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5865 global_avg_mtp_loss: 14.0773 +[titan] 2025-07-09 20:35:33,706 - root - INFO - lr: 2.4067e-04 gnorm: 0.83 [ 7:00:35<15:04:06] +[titan] 2025-07-09 20:35:37,612 - root - INFO - step: 31755 loss: 16.6526 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0746 +[titan] 2025-07-09 20:35:37,612 - root - INFO - lr: 2.4065e-04 gnorm: 0.80 [ 7:00:39<15:04:02] +[titan] 2025-07-09 20:35:41,528 - root - INFO - step: 31760 loss: 16.6281 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5678 global_avg_mtp_loss: 14.0604 +[titan] 2025-07-09 20:35:41,529 - root - INFO - lr: 2.4063e-04 gnorm: 0.77 [ 7:00:43<15:03:58] +[titan] 2025-07-09 20:35:45,433 - root - INFO - step: 31765 loss: 16.9991 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6566 global_avg_mtp_loss: 14.3426 +[titan] 2025-07-09 20:35:45,433 - root - INFO - lr: 2.4061e-04 gnorm: 0.76 [ 7:00:47<15:03:53] +[titan] 2025-07-09 20:35:49,388 - root - INFO - step: 31770 loss: 16.7293 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5803 global_avg_mtp_loss: 14.1491 +[titan] 2025-07-09 20:35:49,388 - root - INFO - lr: 2.4060e-04 gnorm: 0.77 [ 7:00:51<15:03:49] +[titan] 2025-07-09 20:35:53,296 - root - INFO - step: 31775 loss: 16.6578 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5706 global_avg_mtp_loss: 14.0873 +[titan] 2025-07-09 20:35:53,296 - root - INFO - lr: 2.4058e-04 gnorm: 0.77 [ 7:00:55<15:03:45] +[titan] 2025-07-09 20:35:57,213 - root - INFO - step: 31780 loss: 16.8215 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6029 global_avg_mtp_loss: 14.2185 +[titan] 2025-07-09 20:35:57,213 - root - INFO - lr: 2.4056e-04 gnorm: 1.00 [ 7:00:58<15:03:41] +[titan] 2025-07-09 20:36:01,170 - root - INFO - step: 31785 loss: 16.5685 memory: 44.58GiB(31.99%) tps: 82,828 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 14.0147 +[titan] 2025-07-09 20:36:01,170 - root - INFO - lr: 2.4054e-04 gnorm: 0.83 [ 7:01:02<15:03:37] +[titan] 2025-07-09 20:36:05,128 - root - INFO - step: 31790 loss: 16.5764 memory: 44.58GiB(31.99%) tps: 82,792 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-09 20:36:05,128 - root - INFO - lr: 2.4052e-04 gnorm: 0.82 [ 7:01:06<15:03:33] +[titan] 2025-07-09 20:36:09,059 - root - INFO - step: 31795 loss: 16.7465 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1576 +[titan] 2025-07-09 20:36:09,060 - root - INFO - lr: 2.4051e-04 gnorm: 0.75 [ 7:01:10<15:03:29] +[titan] 2025-07-09 20:36:12,194 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:36:12,988 - root - INFO - step: 31800 loss: 16.7983 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.1857 +[titan] 2025-07-09 20:36:12,988 - root - INFO - lr: 2.4049e-04 gnorm: 0.78 [ 7:01:14<15:03:25] +[titan] 2025-07-09 20:36:16,915 - root - INFO - step: 31805 loss: 16.7308 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1469 +[titan] 2025-07-09 20:36:16,916 - root - INFO - lr: 2.4047e-04 gnorm: 0.79 [ 7:01:18<15:03:21] +[titan] 2025-07-09 20:36:20,834 - root - INFO - step: 31810 loss: 16.6102 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.0403 +[titan] 2025-07-09 20:36:20,835 - root - INFO - lr: 2.4045e-04 gnorm: 0.83 [ 7:01:22<15:03:17] +[titan] 2025-07-09 20:36:24,724 - root - INFO - step: 31815 loss: 16.7111 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1277 +[titan] 2025-07-09 20:36:24,724 - root - INFO - lr: 2.4044e-04 gnorm: 0.78 [ 7:01:26<15:03:13] +[titan] 2025-07-09 20:36:28,620 - root - INFO - step: 31820 loss: 16.9157 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.6143 global_avg_mtp_loss: 14.3014 +[titan] 2025-07-09 20:36:28,621 - root - INFO - lr: 2.4042e-04 gnorm: 0.79 [ 7:01:30<15:03:09] +[titan] 2025-07-09 20:36:32,566 - root - INFO - step: 31825 loss: 16.6660 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0906 +[titan] 2025-07-09 20:36:32,566 - root - INFO - lr: 2.4040e-04 gnorm: 0.79 [ 7:01:34<15:03:05] +[titan] 2025-07-09 20:36:36,466 - root - INFO - step: 31830 loss: 16.7342 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1516 +[titan] 2025-07-09 20:36:36,466 - root - INFO - lr: 2.4038e-04 gnorm: 0.75 [ 7:01:38<15:03:00] +[titan] 2025-07-09 20:36:40,395 - root - INFO - step: 31835 loss: 16.8029 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6094 global_avg_mtp_loss: 14.1935 +[titan] 2025-07-09 20:36:40,395 - root - INFO - lr: 2.4036e-04 gnorm: 0.85 [ 7:01:42<15:02:56] +[titan] 2025-07-09 20:36:44,313 - root - INFO - step: 31840 loss: 16.4244 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-09 20:36:44,313 - root - INFO - lr: 2.4035e-04 gnorm: 0.81 [ 7:01:46<15:02:52] +[titan] 2025-07-09 20:36:48,235 - root - INFO - step: 31845 loss: 16.6420 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.0667 +[titan] 2025-07-09 20:36:48,236 - root - INFO - lr: 2.4033e-04 gnorm: 0.80 [ 7:01:49<15:02:48] +[titan] 2025-07-09 20:36:51,358 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:36:52,148 - root - INFO - step: 31850 loss: 16.6424 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5784 global_avg_mtp_loss: 14.0641 +[titan] 2025-07-09 20:36:52,149 - root - INFO - lr: 2.4031e-04 gnorm: 0.73 [ 7:01:53<15:02:44] +[titan] 2025-07-09 20:36:56,047 - root - INFO - step: 31855 loss: 16.9478 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6220 global_avg_mtp_loss: 14.3258 +[titan] 2025-07-09 20:36:56,047 - root - INFO - lr: 2.4029e-04 gnorm: 0.75 [ 7:01:57<15:02:40] +[titan] 2025-07-09 20:36:59,961 - root - INFO - step: 31860 loss: 17.0684 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6479 global_avg_mtp_loss: 14.4205 +[titan] 2025-07-09 20:36:59,961 - root - INFO - lr: 2.4028e-04 gnorm: 0.79 [ 7:02:01<15:02:36] +[titan] 2025-07-09 20:37:03,894 - root - INFO - step: 31865 loss: 16.3925 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.8487 +[titan] 2025-07-09 20:37:03,895 - root - INFO - lr: 2.4026e-04 gnorm: 0.82 [ 7:02:05<15:02:32] +[titan] 2025-07-09 20:37:07,810 - root - INFO - step: 31870 loss: 16.9508 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6339 global_avg_mtp_loss: 14.3169 +[titan] 2025-07-09 20:37:07,811 - root - INFO - lr: 2.4024e-04 gnorm: 0.74 [ 7:02:09<15:02:28] +[titan] 2025-07-09 20:37:11,716 - root - INFO - step: 31875 loss: 16.7449 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5976 global_avg_mtp_loss: 14.1473 +[titan] 2025-07-09 20:37:11,717 - root - INFO - lr: 2.4022e-04 gnorm: 0.78 [ 7:02:13<15:02:24] +[titan] 2025-07-09 20:37:15,632 - root - INFO - step: 31880 loss: 16.7313 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5951 global_avg_mtp_loss: 14.1362 +[titan] 2025-07-09 20:37:15,633 - root - INFO - lr: 2.4020e-04 gnorm: 0.75 [ 7:02:17<15:02:19] +[titan] 2025-07-09 20:37:19,561 - root - INFO - step: 31885 loss: 16.8060 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1945 +[titan] 2025-07-09 20:37:19,561 - root - INFO - lr: 2.4019e-04 gnorm: 0.79 [ 7:02:21<15:02:15] +[titan] 2025-07-09 20:37:23,476 - root - INFO - step: 31890 loss: 16.6070 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-09 20:37:23,476 - root - INFO - lr: 2.4017e-04 gnorm: 0.80 [ 7:02:25<15:02:11] +[titan] 2025-07-09 20:37:27,405 - root - INFO - step: 31895 loss: 17.2046 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6895 global_avg_mtp_loss: 14.5151 +[titan] 2025-07-09 20:37:27,405 - root - INFO - lr: 2.4015e-04 gnorm: 0.76 [ 7:02:29<15:02:07] +[titan] 2025-07-09 20:37:30,524 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:37:31,330 - root - INFO - step: 31900 loss: 16.8807 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6280 global_avg_mtp_loss: 14.2527 +[titan] 2025-07-09 20:37:31,330 - root - INFO - lr: 2.4013e-04 gnorm: 0.73 [ 7:02:33<15:02:03] +[titan] 2025-07-09 20:37:35,231 - root - INFO - step: 31905 loss: 16.7820 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1901 +[titan] 2025-07-09 20:37:35,232 - root - INFO - lr: 2.4012e-04 gnorm: 0.76 [ 7:02:36<15:01:59] +[titan] 2025-07-09 20:37:39,187 - root - INFO - step: 31910 loss: 16.6436 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.0584 +[titan] 2025-07-09 20:37:39,187 - root - INFO - lr: 2.4010e-04 gnorm: 0.80 [ 7:02:40<15:01:55] +[titan] 2025-07-09 20:37:43,095 - root - INFO - step: 31915 loss: 16.8503 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.2400 +[titan] 2025-07-09 20:37:43,096 - root - INFO - lr: 2.4008e-04 gnorm: 0.77 [ 7:02:44<15:01:51] +[titan] 2025-07-09 20:37:47,011 - root - INFO - step: 31920 loss: 16.7906 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5990 global_avg_mtp_loss: 14.1916 +[titan] 2025-07-09 20:37:47,011 - root - INFO - lr: 2.4006e-04 gnorm: 0.78 [ 7:02:48<15:01:47] +[titan] 2025-07-09 20:37:50,923 - root - INFO - step: 31925 loss: 16.5634 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 13.9979 +[titan] 2025-07-09 20:37:50,923 - root - INFO - lr: 2.4004e-04 gnorm: 0.77 [ 7:02:52<15:01:43] +[titan] 2025-07-09 20:37:54,850 - root - INFO - step: 31930 loss: 16.6436 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.0638 +[titan] 2025-07-09 20:37:54,850 - root - INFO - lr: 2.4003e-04 gnorm: 0.80 [ 7:02:56<15:01:39] +[titan] 2025-07-09 20:37:58,777 - root - INFO - step: 31935 loss: 16.5267 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 13.9737 +[titan] 2025-07-09 20:37:58,777 - root - INFO - lr: 2.4001e-04 gnorm: 0.76 [ 7:03:00<15:01:35] +[titan] 2025-07-09 20:38:02,675 - root - INFO - step: 31940 loss: 16.7034 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.1223 +[titan] 2025-07-09 20:38:02,675 - root - INFO - lr: 2.3999e-04 gnorm: 0.83 [ 7:03:04<15:01:30] +[titan] 2025-07-09 20:38:06,576 - root - INFO - step: 31945 loss: 16.8092 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6021 global_avg_mtp_loss: 14.2071 +[titan] 2025-07-09 20:38:06,576 - root - INFO - lr: 2.3997e-04 gnorm: 0.81 [ 7:03:08<15:01:26] +[titan] 2025-07-09 20:38:09,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:38:10,510 - root - INFO - step: 31950 loss: 16.7670 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1861 +[titan] 2025-07-09 20:38:10,511 - root - INFO - lr: 2.3996e-04 gnorm: 0.81 [ 7:03:12<15:01:22] +[titan] 2025-07-09 20:38:14,467 - root - INFO - step: 31955 loss: 16.8037 memory: 44.58GiB(31.99%) tps: 82,821 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.1906 +[titan] 2025-07-09 20:38:14,468 - root - INFO - lr: 2.3994e-04 gnorm: 0.81 [ 7:03:16<15:01:18] +[titan] 2025-07-09 20:38:18,431 - root - INFO - step: 31960 loss: 16.5984 memory: 44.58GiB(31.99%) tps: 82,682 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0227 +[titan] 2025-07-09 20:38:18,431 - root - INFO - lr: 2.3992e-04 gnorm: 0.77 [ 7:03:20<15:01:14] +[titan] 2025-07-09 20:38:22,373 - root - INFO - step: 31965 loss: 16.5504 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 13.9911 +[titan] 2025-07-09 20:38:22,373 - root - INFO - lr: 2.3990e-04 gnorm: 0.76 [ 7:03:24<15:01:10] +[titan] 2025-07-09 20:38:26,267 - root - INFO - step: 31970 loss: 16.8031 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.2038 +[titan] 2025-07-09 20:38:26,268 - root - INFO - lr: 2.3988e-04 gnorm: 0.79 [ 7:03:27<15:01:06] +[titan] 2025-07-09 20:38:30,182 - root - INFO - step: 31975 loss: 16.9052 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6132 global_avg_mtp_loss: 14.2920 +[titan] 2025-07-09 20:38:30,183 - root - INFO - lr: 2.3987e-04 gnorm: 0.72 [ 7:03:31<15:01:02] +[titan] 2025-07-09 20:38:34,100 - root - INFO - step: 31980 loss: 16.8330 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6363 global_avg_mtp_loss: 14.1968 +[titan] 2025-07-09 20:38:34,101 - root - INFO - lr: 2.3985e-04 gnorm: 0.73 [ 7:03:35<15:00:58] +[titan] 2025-07-09 20:38:37,998 - root - INFO - step: 31985 loss: 16.8111 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.2133 +[titan] 2025-07-09 20:38:37,998 - root - INFO - lr: 2.3983e-04 gnorm: 0.76 [ 7:03:39<15:00:54] +[titan] 2025-07-09 20:38:41,902 - root - INFO - step: 31990 loss: 16.7481 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1578 +[titan] 2025-07-09 20:38:41,902 - root - INFO - lr: 2.3981e-04 gnorm: 0.73 [ 7:03:43<15:00:50] +[titan] 2025-07-09 20:38:45,819 - root - INFO - step: 31995 loss: 16.7593 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5970 global_avg_mtp_loss: 14.1623 +[titan] 2025-07-09 20:38:45,819 - root - INFO - lr: 2.3979e-04 gnorm: 0.80 [ 7:03:47<15:00:45] +[titan] 2025-07-09 20:38:48,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:38:49,741 - root - INFO - step: 32000 loss: 16.7252 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5807 global_avg_mtp_loss: 14.1445 +[titan] 2025-07-09 20:38:49,741 - root - INFO - lr: 2.3978e-04 gnorm: 0.76 [ 7:03:51<15:00:41] +[titan] 2025-07-09 20:38:53,635 - root - INFO - step: 32005 loss: 16.7804 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1931 +[titan] 2025-07-09 20:38:53,635 - root - INFO - lr: 2.3976e-04 gnorm: 0.73 [ 7:03:55<15:00:37] +[titan] 2025-07-09 20:38:57,549 - root - INFO - step: 32010 loss: 16.7363 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.1429 +[titan] 2025-07-09 20:38:57,549 - root - INFO - lr: 2.3974e-04 gnorm: 0.77 [ 7:03:59<15:00:33] +[titan] 2025-07-09 20:39:01,450 - root - INFO - step: 32015 loss: 16.7944 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1960 +[titan] 2025-07-09 20:39:01,450 - root - INFO - lr: 2.3972e-04 gnorm: 0.80 [ 7:04:03<15:00:29] +[titan] 2025-07-09 20:39:05,434 - root - INFO - step: 32020 loss: 16.4517 memory: 44.58GiB(31.99%) tps: 82,254 tflops: 283.87 mfu: 28.70% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.9231 +[titan] 2025-07-09 20:39:05,434 - root - INFO - lr: 2.3971e-04 gnorm: 0.81 [ 7:04:07<15:00:25] +[titan] 2025-07-09 20:39:09,343 - root - INFO - step: 32025 loss: 16.6291 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.0591 +[titan] 2025-07-09 20:39:09,343 - root - INFO - lr: 2.3969e-04 gnorm: 0.75 [ 7:04:11<15:00:21] +[titan] 2025-07-09 20:39:13,273 - root - INFO - step: 32030 loss: 16.6435 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.0673 +[titan] 2025-07-09 20:39:13,273 - root - INFO - lr: 2.3967e-04 gnorm: 0.80 [ 7:04:14<15:00:17] +[titan] 2025-07-09 20:39:17,173 - root - INFO - step: 32035 loss: 16.9605 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.3225 +[titan] 2025-07-09 20:39:17,173 - root - INFO - lr: 2.3965e-04 gnorm: 0.83 [ 7:04:18<15:00:13] +[titan] 2025-07-09 20:39:21,069 - root - INFO - step: 32040 loss: 16.4184 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.8902 +[titan] 2025-07-09 20:39:21,069 - root - INFO - lr: 2.3963e-04 gnorm: 0.70 [ 7:04:22<15:00:09] +[titan] 2025-07-09 20:39:25,011 - root - INFO - step: 32045 loss: 16.4354 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.8944 +[titan] 2025-07-09 20:39:25,011 - root - INFO - lr: 2.3962e-04 gnorm: 0.72 [ 7:04:26<15:00:05] +[titan] 2025-07-09 20:39:28,136 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:39:28,929 - root - INFO - step: 32050 loss: 16.4785 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5545 global_avg_mtp_loss: 13.9240 +[titan] 2025-07-09 20:39:28,930 - root - INFO - lr: 2.3960e-04 gnorm: 0.74 [ 7:04:30<15:00:00] +[titan] 2025-07-09 20:39:33,045 - root - INFO - step: 32055 loss: 16.8514 memory: 44.58GiB(31.99%) tps: 79,621 tflops: 274.78 mfu: 27.78% global_avg_ntp_loss: 2.6119 global_avg_mtp_loss: 14.2395 +[titan] 2025-07-09 20:39:33,046 - root - INFO - lr: 2.3958e-04 gnorm: 0.75 [ 7:04:34<14:59:57] +[titan] 2025-07-09 20:39:36,948 - root - INFO - step: 32060 loss: 16.6958 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.1186 +[titan] 2025-07-09 20:39:36,948 - root - INFO - lr: 2.3956e-04 gnorm: 0.76 [ 7:04:38<14:59:53] +[titan] 2025-07-09 20:39:40,861 - root - INFO - step: 32065 loss: 16.4116 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.8811 +[titan] 2025-07-09 20:39:40,862 - root - INFO - lr: 2.3954e-04 gnorm: 0.80 [ 7:04:42<14:59:49] +[titan] 2025-07-09 20:39:44,790 - root - INFO - step: 32070 loss: 16.9114 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6369 global_avg_mtp_loss: 14.2745 +[titan] 2025-07-09 20:39:44,790 - root - INFO - lr: 2.3953e-04 gnorm: 0.77 [ 7:04:46<14:59:44] +[titan] 2025-07-09 20:39:48,747 - root - INFO - step: 32075 loss: 16.6841 memory: 44.58GiB(31.99%) tps: 82,819 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.0960 +[titan] 2025-07-09 20:39:48,747 - root - INFO - lr: 2.3951e-04 gnorm: 0.77 [ 7:04:50<14:59:40] +[titan] 2025-07-09 20:39:52,655 - root - INFO - step: 32080 loss: 16.8464 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2328 +[titan] 2025-07-09 20:39:52,656 - root - INFO - lr: 2.3949e-04 gnorm: 0.83 [ 7:04:54<14:59:36] +[titan] 2025-07-09 20:39:56,549 - root - INFO - step: 32085 loss: 16.7134 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1259 +[titan] 2025-07-09 20:39:56,550 - root - INFO - lr: 2.3947e-04 gnorm: 0.80 [ 7:04:58<14:59:32] +[titan] 2025-07-09 20:40:00,480 - root - INFO - step: 32090 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0844 +[titan] 2025-07-09 20:40:00,481 - root - INFO - lr: 2.3946e-04 gnorm: 0.79 [ 7:05:02<14:59:28] +[titan] 2025-07-09 20:40:04,396 - root - INFO - step: 32095 loss: 16.9702 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.6418 global_avg_mtp_loss: 14.3284 +[titan] 2025-07-09 20:40:04,396 - root - INFO - lr: 2.3944e-04 gnorm: 0.76 [ 7:05:06<14:59:24] +[titan] 2025-07-09 20:40:07,502 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:40:08,291 - root - INFO - step: 32100 loss: 17.0410 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.6469 global_avg_mtp_loss: 14.3941 +[titan] 2025-07-09 20:40:08,291 - root - INFO - lr: 2.3942e-04 gnorm: 0.74 [ 7:05:10<14:59:20] +[titan] 2025-07-09 20:40:12,212 - root - INFO - step: 32105 loss: 16.9035 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.6205 global_avg_mtp_loss: 14.2830 +[titan] 2025-07-09 20:40:12,213 - root - INFO - lr: 2.3940e-04 gnorm: 0.81 [ 7:05:13<14:59:16] +[titan] 2025-07-09 20:40:16,173 - root - INFO - step: 32110 loss: 16.4531 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9127 +[titan] 2025-07-09 20:40:16,173 - root - INFO - lr: 2.3938e-04 gnorm: 0.84 [ 7:05:17<14:59:12] +[titan] 2025-07-09 20:40:20,070 - root - INFO - step: 32115 loss: 16.6810 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-09 20:40:20,070 - root - INFO - lr: 2.3937e-04 gnorm: 0.82 [ 7:05:21<14:59:08] +[titan] 2025-07-09 20:40:23,979 - root - INFO - step: 32120 loss: 17.0406 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6466 global_avg_mtp_loss: 14.3940 +[titan] 2025-07-09 20:40:23,979 - root - INFO - lr: 2.3935e-04 gnorm: 0.71 [ 7:05:25<14:59:04] +[titan] 2025-07-09 20:40:27,888 - root - INFO - step: 32125 loss: 16.6069 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0407 +[titan] 2025-07-09 20:40:27,888 - root - INFO - lr: 2.3933e-04 gnorm: 0.77 [ 7:05:29<14:58:59] +[titan] 2025-07-09 20:40:31,848 - root - INFO - step: 32130 loss: 16.8021 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5950 global_avg_mtp_loss: 14.2072 +[titan] 2025-07-09 20:40:31,848 - root - INFO - lr: 2.3931e-04 gnorm: 0.74 [ 7:05:33<14:58:55] +[titan] 2025-07-09 20:40:35,772 - root - INFO - step: 32135 loss: 16.7424 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.1543 +[titan] 2025-07-09 20:40:35,772 - root - INFO - lr: 2.3929e-04 gnorm: 0.84 [ 7:05:37<14:58:51] +[titan] 2025-07-09 20:40:39,716 - root - INFO - step: 32140 loss: 16.6141 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.0293 +[titan] 2025-07-09 20:40:39,716 - root - INFO - lr: 2.3928e-04 gnorm: 0.87 [ 7:05:41<14:58:47] +[titan] 2025-07-09 20:40:43,629 - root - INFO - step: 32145 loss: 16.9232 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6334 global_avg_mtp_loss: 14.2898 +[titan] 2025-07-09 20:40:43,630 - root - INFO - lr: 2.3926e-04 gnorm: 0.73 [ 7:05:45<14:58:43] +[titan] 2025-07-09 20:40:46,922 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:40:47,712 - root - INFO - step: 32150 loss: 16.8229 memory: 44.58GiB(31.99%) tps: 80,268 tflops: 277.02 mfu: 28.01% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.2190 +[titan] 2025-07-09 20:40:47,712 - root - INFO - lr: 2.3924e-04 gnorm: 0.76 [ 7:05:49<14:58:39] +[titan] 2025-07-09 20:40:51,603 - root - INFO - step: 32155 loss: 16.8150 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.6156 global_avg_mtp_loss: 14.1994 +[titan] 2025-07-09 20:40:51,603 - root - INFO - lr: 2.3922e-04 gnorm: 0.75 [ 7:05:53<14:58:35] +[titan] 2025-07-09 20:40:55,515 - root - INFO - step: 32160 loss: 16.9459 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.3265 +[titan] 2025-07-09 20:40:55,515 - root - INFO - lr: 2.3921e-04 gnorm: 0.81 [ 7:05:57<14:58:31] +[titan] 2025-07-09 20:40:59,458 - root - INFO - step: 32165 loss: 16.8375 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6051 global_avg_mtp_loss: 14.2323 +[titan] 2025-07-09 20:40:59,458 - root - INFO - lr: 2.3919e-04 gnorm: 0.72 [ 7:06:01<14:58:27] +[titan] 2025-07-09 20:41:03,408 - root - INFO - step: 32170 loss: 16.9427 memory: 44.58GiB(31.99%) tps: 82,952 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.3166 +[titan] 2025-07-09 20:41:03,409 - root - INFO - lr: 2.3917e-04 gnorm: 0.79 [ 7:06:05<14:58:23] +[titan] 2025-07-09 20:41:07,379 - root - INFO - step: 32175 loss: 16.9631 memory: 44.58GiB(31.99%) tps: 82,527 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3235 +[titan] 2025-07-09 20:41:07,380 - root - INFO - lr: 2.3915e-04 gnorm: 0.82 [ 7:06:09<14:58:19] +[titan] 2025-07-09 20:41:11,304 - root - INFO - step: 32180 loss: 16.8979 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.2629 +[titan] 2025-07-09 20:41:11,304 - root - INFO - lr: 2.3913e-04 gnorm: 0.78 [ 7:06:13<14:58:15] +[titan] 2025-07-09 20:41:15,246 - root - INFO - step: 32185 loss: 16.8514 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.6118 global_avg_mtp_loss: 14.2396 +[titan] 2025-07-09 20:41:15,246 - root - INFO - lr: 2.3912e-04 gnorm: 0.78 [ 7:06:16<14:58:11] +[titan] 2025-07-09 20:41:19,150 - root - INFO - step: 32190 loss: 16.5679 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0066 +[titan] 2025-07-09 20:41:19,150 - root - INFO - lr: 2.3910e-04 gnorm: 0.82 [ 7:06:20<14:58:07] +[titan] 2025-07-09 20:41:23,062 - root - INFO - step: 32195 loss: 16.8736 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2555 +[titan] 2025-07-09 20:41:23,062 - root - INFO - lr: 2.3908e-04 gnorm: 0.74 [ 7:06:24<14:58:03] +[titan] 2025-07-09 20:41:26,193 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:41:26,987 - root - INFO - step: 32200 loss: 16.8716 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2532 +[titan] 2025-07-09 20:41:26,988 - root - INFO - lr: 2.3906e-04 gnorm: 0.80 [ 7:06:28<14:57:59] +[titan] 2025-07-09 20:41:30,942 - root - INFO - step: 32205 loss: 16.5228 memory: 44.58GiB(31.99%) tps: 82,869 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9717 +[titan] 2025-07-09 20:41:30,942 - root - INFO - lr: 2.3904e-04 gnorm: 0.85 [ 7:06:32<14:57:55] +[titan] 2025-07-09 20:41:34,865 - root - INFO - step: 32210 loss: 16.6134 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0504 +[titan] 2025-07-09 20:41:34,865 - root - INFO - lr: 2.3903e-04 gnorm: 0.73 [ 7:06:36<14:57:51] +[titan] 2025-07-09 20:41:38,761 - root - INFO - step: 32215 loss: 16.7616 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.1670 +[titan] 2025-07-09 20:41:38,761 - root - INFO - lr: 2.3901e-04 gnorm: 0.77 [ 7:06:40<14:57:47] +[titan] 2025-07-09 20:41:42,658 - root - INFO - step: 32220 loss: 16.8328 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.2239 +[titan] 2025-07-09 20:41:42,658 - root - INFO - lr: 2.3899e-04 gnorm: 0.81 [ 7:06:44<14:57:42] +[titan] 2025-07-09 20:41:46,587 - root - INFO - step: 32225 loss: 16.8852 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.6370 global_avg_mtp_loss: 14.2482 +[titan] 2025-07-09 20:41:46,587 - root - INFO - lr: 2.3897e-04 gnorm: 0.82 [ 7:06:48<14:57:38] +[titan] 2025-07-09 20:41:50,514 - root - INFO - step: 32230 loss: 16.6457 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.0704 +[titan] 2025-07-09 20:41:50,514 - root - INFO - lr: 2.3895e-04 gnorm: 0.86 [ 7:06:52<14:57:34] +[titan] 2025-07-09 20:41:54,500 - root - INFO - step: 32235 loss: 16.4663 memory: 44.58GiB(31.99%) tps: 82,219 tflops: 283.75 mfu: 28.69% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9256 +[titan] 2025-07-09 20:41:54,500 - root - INFO - lr: 2.3894e-04 gnorm: 0.77 [ 7:06:56<14:57:30] +[titan] 2025-07-09 20:41:58,393 - root - INFO - step: 32240 loss: 16.8106 memory: 44.58GiB(31.99%) tps: 84,183 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.6158 global_avg_mtp_loss: 14.1948 +[titan] 2025-07-09 20:41:58,393 - root - INFO - lr: 2.3892e-04 gnorm: 0.79 [ 7:07:00<14:57:26] +[titan] 2025-07-09 20:42:02,299 - root - INFO - step: 32245 loss: 16.9301 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.3061 +[titan] 2025-07-09 20:42:02,299 - root - INFO - lr: 2.3890e-04 gnorm: 0.81 [ 7:07:03<14:57:22] +[titan] 2025-07-09 20:42:05,407 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:42:06,198 - root - INFO - step: 32250 loss: 16.5587 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5564 global_avg_mtp_loss: 14.0023 +[titan] 2025-07-09 20:42:06,199 - root - INFO - lr: 2.3888e-04 gnorm: 0.78 [ 7:07:07<14:57:18] +[titan] 2025-07-09 20:42:10,185 - root - INFO - step: 32255 loss: 16.8445 memory: 44.58GiB(31.99%) tps: 82,194 tflops: 283.67 mfu: 28.68% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.2158 +[titan] 2025-07-09 20:42:10,186 - root - INFO - lr: 2.3886e-04 gnorm: 0.70 [ 7:07:11<14:57:14] +[titan] 2025-07-09 20:42:11,141 - root - INFO - Dumping profiler traces at step 32256 +[titan] 2025-07-09 20:42:11,175 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:42:14,309 - root - INFO - step: 32260 loss: 16.6185 memory: 44.58GiB(31.99%) tps: 79,474 tflops: 274.28 mfu: 27.73% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.0434 +[titan] 2025-07-09 20:42:14,309 - root - INFO - lr: 2.3885e-04 gnorm: 0.74 [ 7:07:16<14:57:10] +[titan] 2025-07-09 20:42:18,229 - root - INFO - step: 32265 loss: 16.8829 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.6043 global_avg_mtp_loss: 14.2785 +[titan] 2025-07-09 20:42:18,229 - root - INFO - lr: 2.3883e-04 gnorm: 0.74 [ 7:07:19<14:57:06] +[titan] 2025-07-09 20:42:22,174 - root - INFO - step: 32270 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.6201 global_avg_mtp_loss: 14.2476 +[titan] 2025-07-09 20:42:22,175 - root - INFO - lr: 2.3881e-04 gnorm: 0.79 [ 7:07:23<14:57:02] +[titan] 2025-07-09 20:42:26,165 - root - INFO - step: 32275 loss: 16.5146 memory: 44.58GiB(31.99%) tps: 82,120 tflops: 283.41 mfu: 28.66% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 13.9661 +[titan] 2025-07-09 20:42:26,165 - root - INFO - lr: 2.3879e-04 gnorm: 0.83 [ 7:07:27<14:56:58] +[titan] 2025-07-09 20:42:30,084 - root - INFO - step: 32280 loss: 17.1235 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6588 global_avg_mtp_loss: 14.4647 +[titan] 2025-07-09 20:42:30,084 - root - INFO - lr: 2.3878e-04 gnorm: 0.79 [ 7:07:31<14:56:54] +[titan] 2025-07-09 20:42:33,987 - root - INFO - step: 32285 loss: 16.8095 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6108 global_avg_mtp_loss: 14.1987 +[titan] 2025-07-09 20:42:33,987 - root - INFO - lr: 2.3876e-04 gnorm: 0.79 [ 7:07:35<14:56:50] +[titan] 2025-07-09 20:42:37,923 - root - INFO - step: 32290 loss: 16.9723 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6511 global_avg_mtp_loss: 14.3211 +[titan] 2025-07-09 20:42:37,923 - root - INFO - lr: 2.3874e-04 gnorm: 0.73 [ 7:07:39<14:56:46] +[titan] 2025-07-09 20:42:41,811 - root - INFO - step: 32295 loss: 16.9245 memory: 44.58GiB(31.99%) tps: 84,298 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.2966 +[titan] 2025-07-09 20:42:41,811 - root - INFO - lr: 2.3872e-04 gnorm: 0.75 [ 7:07:43<14:56:42] +[titan] 2025-07-09 20:42:44,918 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:42:45,712 - root - INFO - step: 32300 loss: 16.7566 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1719 +[titan] 2025-07-09 20:42:45,712 - root - INFO - lr: 2.3870e-04 gnorm: 0.78 [ 7:07:47<14:56:38] +[titan] 2025-07-09 20:42:49,632 - root - INFO - step: 32305 loss: 16.9934 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6310 global_avg_mtp_loss: 14.3624 +[titan] 2025-07-09 20:42:49,632 - root - INFO - lr: 2.3869e-04 gnorm: 0.80 [ 7:07:51<14:56:34] +[titan] 2025-07-09 20:42:53,559 - root - INFO - step: 32310 loss: 16.8161 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.2192 +[titan] 2025-07-09 20:42:53,559 - root - INFO - lr: 2.3867e-04 gnorm: 0.75 [ 7:07:55<14:56:30] +[titan] 2025-07-09 20:42:57,470 - root - INFO - step: 32315 loss: 16.9208 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6277 global_avg_mtp_loss: 14.2931 +[titan] 2025-07-09 20:42:57,470 - root - INFO - lr: 2.3865e-04 gnorm: 0.81 [ 7:07:59<14:56:25] +[titan] 2025-07-09 20:43:01,364 - root - INFO - step: 32320 loss: 16.6293 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0629 +[titan] 2025-07-09 20:43:01,364 - root - INFO - lr: 2.3863e-04 gnorm: 0.78 [ 7:08:03<14:56:21] +[titan] 2025-07-09 20:43:05,276 - root - INFO - step: 32325 loss: 16.9086 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.6205 global_avg_mtp_loss: 14.2881 +[titan] 2025-07-09 20:43:05,276 - root - INFO - lr: 2.3861e-04 gnorm: 0.78 [ 7:08:06<14:56:17] +[titan] 2025-07-09 20:43:09,258 - root - INFO - step: 32330 loss: 16.7432 memory: 44.58GiB(31.99%) tps: 82,296 tflops: 284.02 mfu: 28.72% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.1419 +[titan] 2025-07-09 20:43:09,258 - root - INFO - lr: 2.3860e-04 gnorm: 0.73 [ 7:08:10<14:56:13] +[titan] 2025-07-09 20:43:13,197 - root - INFO - step: 32335 loss: 16.8907 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.6116 global_avg_mtp_loss: 14.2791 +[titan] 2025-07-09 20:43:13,198 - root - INFO - lr: 2.3858e-04 gnorm: 0.82 [ 7:08:14<14:56:09] +[titan] 2025-07-09 20:43:17,123 - root - INFO - step: 32340 loss: 16.7924 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.1942 +[titan] 2025-07-09 20:43:17,124 - root - INFO - lr: 2.3856e-04 gnorm: 0.77 [ 7:08:18<14:56:05] +[titan] 2025-07-09 20:43:21,025 - root - INFO - step: 32345 loss: 16.9129 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.2822 +[titan] 2025-07-09 20:43:21,025 - root - INFO - lr: 2.3854e-04 gnorm: 0.77 [ 7:08:22<14:56:01] +[titan] 2025-07-09 20:43:24,165 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:43:24,957 - root - INFO - step: 32350 loss: 16.7796 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.1818 +[titan] 2025-07-09 20:43:24,957 - root - INFO - lr: 2.3852e-04 gnorm: 0.77 [ 7:08:26<14:55:57] +[titan] 2025-07-09 20:43:28,883 - root - INFO - step: 32355 loss: 16.6613 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5749 global_avg_mtp_loss: 14.0864 +[titan] 2025-07-09 20:43:28,883 - root - INFO - lr: 2.3851e-04 gnorm: 0.76 [ 7:08:30<14:55:53] +[titan] 2025-07-09 20:43:32,810 - root - INFO - step: 32360 loss: 16.8122 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.6051 global_avg_mtp_loss: 14.2071 +[titan] 2025-07-09 20:43:32,810 - root - INFO - lr: 2.3849e-04 gnorm: 0.75 [ 7:08:34<14:55:49] +[titan] 2025-07-09 20:43:36,737 - root - INFO - step: 32365 loss: 16.9661 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6420 global_avg_mtp_loss: 14.3241 +[titan] 2025-07-09 20:43:36,737 - root - INFO - lr: 2.3847e-04 gnorm: 0.75 [ 7:08:38<14:55:45] +[titan] 2025-07-09 20:43:40,686 - root - INFO - step: 32370 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 13.9853 +[titan] 2025-07-09 20:43:40,686 - root - INFO - lr: 2.3845e-04 gnorm: 0.87 [ 7:08:42<14:55:41] +[titan] 2025-07-09 20:43:44,619 - root - INFO - step: 32375 loss: 16.7513 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5988 global_avg_mtp_loss: 14.1525 +[titan] 2025-07-09 20:43:44,619 - root - INFO - lr: 2.3843e-04 gnorm: 0.79 [ 7:08:46<14:55:37] +[titan] 2025-07-09 20:43:48,579 - root - INFO - step: 32380 loss: 16.9782 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.6314 global_avg_mtp_loss: 14.3468 +[titan] 2025-07-09 20:43:48,580 - root - INFO - lr: 2.3842e-04 gnorm: 0.71 [ 7:08:50<14:55:33] +[titan] 2025-07-09 20:43:52,480 - root - INFO - step: 32385 loss: 16.9962 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6401 global_avg_mtp_loss: 14.3561 +[titan] 2025-07-09 20:43:52,480 - root - INFO - lr: 2.3840e-04 gnorm: 0.78 [ 7:08:54<14:55:29] +[titan] 2025-07-09 20:43:56,418 - root - INFO - step: 32390 loss: 17.0249 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.6651 global_avg_mtp_loss: 14.3598 +[titan] 2025-07-09 20:43:56,418 - root - INFO - lr: 2.3838e-04 gnorm: 0.74 [ 7:08:58<14:55:25] +[titan] 2025-07-09 20:44:00,365 - root - INFO - step: 32395 loss: 16.7048 memory: 44.58GiB(31.99%) tps: 83,042 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1226 +[titan] 2025-07-09 20:44:00,365 - root - INFO - lr: 2.3836e-04 gnorm: 0.79 [ 7:09:02<14:55:20] +[titan] 2025-07-09 20:44:03,487 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:44:04,279 - root - INFO - step: 32400 loss: 16.5533 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9960 +[titan] 2025-07-09 20:44:04,280 - root - INFO - lr: 2.3834e-04 gnorm: 0.74 [ 7:09:05<14:55:16] +[titan] 2025-07-09 20:44:08,208 - root - INFO - step: 32405 loss: 16.6901 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5829 global_avg_mtp_loss: 14.1072 +[titan] 2025-07-09 20:44:08,208 - root - INFO - lr: 2.3833e-04 gnorm: 0.75 [ 7:09:09<14:55:12] +[titan] 2025-07-09 20:44:12,107 - root - INFO - step: 32410 loss: 16.6182 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-09 20:44:12,108 - root - INFO - lr: 2.3831e-04 gnorm: 0.76 [ 7:09:13<14:55:08] +[titan] 2025-07-09 20:44:16,003 - root - INFO - step: 32415 loss: 16.7739 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.6003 global_avg_mtp_loss: 14.1736 +[titan] 2025-07-09 20:44:16,003 - root - INFO - lr: 2.3829e-04 gnorm: 0.73 [ 7:09:17<14:55:04] +[titan] 2025-07-09 20:44:19,903 - root - INFO - step: 32420 loss: 16.9557 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.3249 +[titan] 2025-07-09 20:44:19,903 - root - INFO - lr: 2.3827e-04 gnorm: 0.77 [ 7:09:21<14:55:00] +[titan] 2025-07-09 20:44:23,812 - root - INFO - step: 32425 loss: 16.9834 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.6400 global_avg_mtp_loss: 14.3434 +[titan] 2025-07-09 20:44:23,813 - root - INFO - lr: 2.3825e-04 gnorm: 0.75 [ 7:09:25<14:54:56] +[titan] 2025-07-09 20:44:27,728 - root - INFO - step: 32430 loss: 16.6895 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.1135 +[titan] 2025-07-09 20:44:27,728 - root - INFO - lr: 2.3824e-04 gnorm: 0.75 [ 7:09:29<14:54:52] +[titan] 2025-07-09 20:44:31,631 - root - INFO - step: 32435 loss: 16.9690 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6314 global_avg_mtp_loss: 14.3376 +[titan] 2025-07-09 20:44:31,631 - root - INFO - lr: 2.3822e-04 gnorm: 0.76 [ 7:09:33<14:54:48] +[titan] 2025-07-09 20:44:35,550 - root - INFO - step: 32440 loss: 16.6015 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0410 +[titan] 2025-07-09 20:44:35,550 - root - INFO - lr: 2.3820e-04 gnorm: 0.77 [ 7:09:37<14:54:44] +[titan] 2025-07-09 20:44:39,493 - root - INFO - step: 32445 loss: 16.7175 memory: 44.58GiB(31.99%) tps: 83,108 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.6073 global_avg_mtp_loss: 14.1102 +[titan] 2025-07-09 20:44:39,494 - root - INFO - lr: 2.3818e-04 gnorm: 0.82 [ 7:09:41<14:54:39] +[titan] 2025-07-09 20:44:42,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:44:43,395 - root - INFO - step: 32450 loss: 16.9140 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6408 global_avg_mtp_loss: 14.2732 +[titan] 2025-07-09 20:44:43,396 - root - INFO - lr: 2.3816e-04 gnorm: 0.79 [ 7:09:45<14:54:35] +[titan] 2025-07-09 20:44:47,370 - root - INFO - step: 32455 loss: 16.8539 memory: 44.58GiB(31.99%) tps: 82,441 tflops: 284.52 mfu: 28.77% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2268 +[titan] 2025-07-09 20:44:47,371 - root - INFO - lr: 2.3815e-04 gnorm: 0.76 [ 7:09:49<14:54:31] +[titan] 2025-07-09 20:44:51,299 - root - INFO - step: 32460 loss: 16.7255 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5911 global_avg_mtp_loss: 14.1344 +[titan] 2025-07-09 20:44:51,299 - root - INFO - lr: 2.3813e-04 gnorm: 0.84 [ 7:09:52<14:54:27] +[titan] 2025-07-09 20:44:55,199 - root - INFO - step: 32465 loss: 16.8558 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2351 +[titan] 2025-07-09 20:44:55,199 - root - INFO - lr: 2.3811e-04 gnorm: 0.83 [ 7:09:56<14:54:23] +[titan] 2025-07-09 20:44:59,133 - root - INFO - step: 32470 loss: 16.8476 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.6131 global_avg_mtp_loss: 14.2346 +[titan] 2025-07-09 20:44:59,133 - root - INFO - lr: 2.3809e-04 gnorm: 0.80 [ 7:10:00<14:54:19] +[titan] 2025-07-09 20:45:03,068 - root - INFO - step: 32475 loss: 16.6542 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5878 global_avg_mtp_loss: 14.0665 +[titan] 2025-07-09 20:45:03,069 - root - INFO - lr: 2.3807e-04 gnorm: 0.84 [ 7:10:04<14:54:15] +[titan] 2025-07-09 20:45:06,978 - root - INFO - step: 32480 loss: 16.5914 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0265 +[titan] 2025-07-09 20:45:06,978 - root - INFO - lr: 2.3806e-04 gnorm: 0.79 [ 7:10:08<14:54:11] +[titan] 2025-07-09 20:45:10,903 - root - INFO - step: 32485 loss: 16.8116 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6045 global_avg_mtp_loss: 14.2070 +[titan] 2025-07-09 20:45:10,903 - root - INFO - lr: 2.3804e-04 gnorm: 0.79 [ 7:10:12<14:54:07] +[titan] 2025-07-09 20:45:14,859 - root - INFO - step: 32490 loss: 16.5121 memory: 44.58GiB(31.99%) tps: 82,847 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 13.9632 +[titan] 2025-07-09 20:45:14,859 - root - INFO - lr: 2.3802e-04 gnorm: 0.80 [ 7:10:16<14:54:03] +[titan] 2025-07-09 20:45:18,760 - root - INFO - step: 32495 loss: 16.7906 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6068 global_avg_mtp_loss: 14.1838 +[titan] 2025-07-09 20:45:18,760 - root - INFO - lr: 2.3800e-04 gnorm: 0.77 [ 7:10:20<14:53:59] +[titan] 2025-07-09 20:45:21,890 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:45:22,686 - root - INFO - step: 32500 loss: 16.6014 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5669 global_avg_mtp_loss: 14.0345 +[titan] 2025-07-09 20:45:22,686 - root - INFO - lr: 2.3798e-04 gnorm: 0.73 [ 7:10:24<14:53:55] +[titan] 2025-07-09 20:45:26,625 - root - INFO - step: 32505 loss: 16.7289 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5856 global_avg_mtp_loss: 14.1433 +[titan] 2025-07-09 20:45:26,626 - root - INFO - lr: 2.3797e-04 gnorm: 0.77 [ 7:10:28<14:53:51] +[titan] 2025-07-09 20:45:30,542 - root - INFO - step: 32510 loss: 16.8502 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.6150 global_avg_mtp_loss: 14.2353 +[titan] 2025-07-09 20:45:30,542 - root - INFO - lr: 2.3795e-04 gnorm: 0.73 [ 7:10:32<14:53:47] +[titan] 2025-07-09 20:45:34,474 - root - INFO - step: 32515 loss: 17.1331 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.6628 global_avg_mtp_loss: 14.4703 +[titan] 2025-07-09 20:45:34,474 - root - INFO - lr: 2.3793e-04 gnorm: 0.78 [ 7:10:36<14:53:43] +[titan] 2025-07-09 20:45:38,389 - root - INFO - step: 32520 loss: 16.7752 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.1856 +[titan] 2025-07-09 20:45:38,389 - root - INFO - lr: 2.3791e-04 gnorm: 0.80 [ 7:10:40<14:53:38] +[titan] 2025-07-09 20:45:42,308 - root - INFO - step: 32525 loss: 16.7583 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1613 +[titan] 2025-07-09 20:45:42,308 - root - INFO - lr: 2.3789e-04 gnorm: 0.77 [ 7:10:43<14:53:34] +[titan] 2025-07-09 20:45:46,227 - root - INFO - step: 32530 loss: 16.5916 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0347 +[titan] 2025-07-09 20:45:46,227 - root - INFO - lr: 2.3788e-04 gnorm: 0.80 [ 7:10:47<14:53:30] +[titan] 2025-07-09 20:45:50,163 - root - INFO - step: 32535 loss: 16.8733 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5991 global_avg_mtp_loss: 14.2743 +[titan] 2025-07-09 20:45:50,163 - root - INFO - lr: 2.3786e-04 gnorm: 0.78 [ 7:10:51<14:53:26] +[titan] 2025-07-09 20:45:54,063 - root - INFO - step: 32540 loss: 16.7036 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1114 +[titan] 2025-07-09 20:45:54,063 - root - INFO - lr: 2.3784e-04 gnorm: 0.83 [ 7:10:55<14:53:22] +[titan] 2025-07-09 20:45:58,046 - root - INFO - step: 32545 loss: 16.8250 memory: 44.58GiB(31.99%) tps: 82,283 tflops: 283.97 mfu: 28.71% global_avg_ntp_loss: 2.6104 global_avg_mtp_loss: 14.2146 +[titan] 2025-07-09 20:45:58,046 - root - INFO - lr: 2.3782e-04 gnorm: 0.74 [ 7:10:59<14:53:18] +[titan] 2025-07-09 20:46:01,180 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:46:01,980 - root - INFO - step: 32550 loss: 16.6508 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.0713 +[titan] 2025-07-09 20:46:01,980 - root - INFO - lr: 2.3780e-04 gnorm: 0.78 [ 7:11:03<14:53:14] +[titan] 2025-07-09 20:46:05,905 - root - INFO - step: 32555 loss: 16.7639 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1772 +[titan] 2025-07-09 20:46:05,905 - root - INFO - lr: 2.3779e-04 gnorm: 0.80 [ 7:11:07<14:53:10] +[titan] 2025-07-09 20:46:09,853 - root - INFO - step: 32560 loss: 16.7910 memory: 44.58GiB(31.99%) tps: 83,003 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5995 global_avg_mtp_loss: 14.1915 +[titan] 2025-07-09 20:46:09,853 - root - INFO - lr: 2.3777e-04 gnorm: 0.75 [ 7:11:11<14:53:06] +[titan] 2025-07-09 20:46:13,765 - root - INFO - step: 32565 loss: 16.6139 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.0434 +[titan] 2025-07-09 20:46:13,766 - root - INFO - lr: 2.3775e-04 gnorm: 0.81 [ 7:11:15<14:53:02] +[titan] 2025-07-09 20:46:17,662 - root - INFO - step: 32570 loss: 16.6860 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.1108 +[titan] 2025-07-09 20:46:17,662 - root - INFO - lr: 2.3773e-04 gnorm: 0.81 [ 7:11:19<14:52:58] +[titan] 2025-07-09 20:46:21,569 - root - INFO - step: 32575 loss: 17.1522 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.6756 global_avg_mtp_loss: 14.4767 +[titan] 2025-07-09 20:46:21,569 - root - INFO - lr: 2.3771e-04 gnorm: 0.88 [ 7:11:23<14:52:54] +[titan] 2025-07-09 20:46:25,484 - root - INFO - step: 32580 loss: 16.4448 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.8874 +[titan] 2025-07-09 20:46:25,484 - root - INFO - lr: 2.3770e-04 gnorm: 0.79 [ 7:11:27<14:52:50] +[titan] 2025-07-09 20:46:29,397 - root - INFO - step: 32585 loss: 16.5702 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 13.9933 +[titan] 2025-07-09 20:46:29,397 - root - INFO - lr: 2.3768e-04 gnorm: 0.77 [ 7:11:31<14:52:45] +[titan] 2025-07-09 20:46:33,300 - root - INFO - step: 32590 loss: 16.7665 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5954 global_avg_mtp_loss: 14.1711 +[titan] 2025-07-09 20:46:33,300 - root - INFO - lr: 2.3766e-04 gnorm: 0.80 [ 7:11:34<14:52:41] +[titan] 2025-07-09 20:46:37,231 - root - INFO - step: 32595 loss: 16.8569 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2396 +[titan] 2025-07-09 20:46:37,231 - root - INFO - lr: 2.3764e-04 gnorm: 0.71 [ 7:11:38<14:52:37] +[titan] 2025-07-09 20:46:40,359 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:46:41,155 - root - INFO - step: 32600 loss: 16.4825 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9397 +[titan] 2025-07-09 20:46:41,155 - root - INFO - lr: 2.3762e-04 gnorm: 0.79 [ 7:11:42<14:52:33] +[titan] 2025-07-09 20:46:45,087 - root - INFO - step: 32605 loss: 16.8229 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6101 global_avg_mtp_loss: 14.2128 +[titan] 2025-07-09 20:46:45,087 - root - INFO - lr: 2.3761e-04 gnorm: 0.80 [ 7:11:46<14:52:29] +[titan] 2025-07-09 20:46:48,994 - root - INFO - step: 32610 loss: 16.8641 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2418 +[titan] 2025-07-09 20:46:48,994 - root - INFO - lr: 2.3759e-04 gnorm: 0.80 [ 7:11:50<14:52:25] +[titan] 2025-07-09 20:46:52,889 - root - INFO - step: 32615 loss: 16.7064 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 20:46:52,889 - root - INFO - lr: 2.3757e-04 gnorm: 0.81 [ 7:11:54<14:52:21] +[titan] 2025-07-09 20:46:56,784 - root - INFO - step: 32620 loss: 16.6257 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-09 20:46:56,784 - root - INFO - lr: 2.3755e-04 gnorm: 0.74 [ 7:11:58<14:52:17] +[titan] 2025-07-09 20:47:00,692 - root - INFO - step: 32625 loss: 16.9506 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6259 global_avg_mtp_loss: 14.3247 +[titan] 2025-07-09 20:47:00,692 - root - INFO - lr: 2.3753e-04 gnorm: 0.76 [ 7:12:02<14:52:13] +[titan] 2025-07-09 20:47:04,610 - root - INFO - step: 32630 loss: 16.5002 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9528 +[titan] 2025-07-09 20:47:04,611 - root - INFO - lr: 2.3751e-04 gnorm: 0.74 [ 7:12:06<14:52:09] +[titan] 2025-07-09 20:47:08,512 - root - INFO - step: 32635 loss: 16.7118 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1216 +[titan] 2025-07-09 20:47:08,512 - root - INFO - lr: 2.3750e-04 gnorm: 0.86 [ 7:12:10<14:52:04] +[titan] 2025-07-09 20:47:12,509 - root - INFO - step: 32640 loss: 16.9943 memory: 44.58GiB(31.99%) tps: 81,982 tflops: 282.93 mfu: 28.61% global_avg_ntp_loss: 2.6395 global_avg_mtp_loss: 14.3548 +[titan] 2025-07-09 20:47:12,510 - root - INFO - lr: 2.3748e-04 gnorm: 0.76 [ 7:12:14<14:52:01] +[titan] 2025-07-09 20:47:16,407 - root - INFO - step: 32645 loss: 16.8165 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.2078 +[titan] 2025-07-09 20:47:16,407 - root - INFO - lr: 2.3746e-04 gnorm: 0.80 [ 7:12:18<14:51:56] +[titan] 2025-07-09 20:47:19,546 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:47:20,336 - root - INFO - step: 32650 loss: 16.7186 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.1334 +[titan] 2025-07-09 20:47:20,336 - root - INFO - lr: 2.3744e-04 gnorm: 0.75 [ 7:12:21<14:51:52] +[titan] 2025-07-09 20:47:24,240 - root - INFO - step: 32655 loss: 16.5906 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0240 +[titan] 2025-07-09 20:47:24,241 - root - INFO - lr: 2.3742e-04 gnorm: 0.82 [ 7:12:25<14:51:48] +[titan] 2025-07-09 20:47:28,170 - root - INFO - step: 32660 loss: 16.7490 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.1578 +[titan] 2025-07-09 20:47:28,170 - root - INFO - lr: 2.3741e-04 gnorm: 0.75 [ 7:12:29<14:51:44] +[titan] 2025-07-09 20:47:32,102 - root - INFO - step: 32665 loss: 16.7984 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5999 global_avg_mtp_loss: 14.1985 +[titan] 2025-07-09 20:47:32,102 - root - INFO - lr: 2.3739e-04 gnorm: 0.76 [ 7:12:33<14:51:40] +[titan] 2025-07-09 20:47:36,004 - root - INFO - step: 32670 loss: 16.5570 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 13.9915 +[titan] 2025-07-09 20:47:36,005 - root - INFO - lr: 2.3737e-04 gnorm: 0.77 [ 7:12:37<14:51:36] +[titan] 2025-07-09 20:47:39,960 - root - INFO - step: 32675 loss: 16.5774 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 14.0278 +[titan] 2025-07-09 20:47:39,960 - root - INFO - lr: 2.3735e-04 gnorm: 0.74 [ 7:12:41<14:51:32] +[titan] 2025-07-09 20:47:43,904 - root - INFO - step: 32680 loss: 16.7917 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.6012 global_avg_mtp_loss: 14.1905 +[titan] 2025-07-09 20:47:43,905 - root - INFO - lr: 2.3733e-04 gnorm: 0.75 [ 7:12:45<14:51:28] +[titan] 2025-07-09 20:47:47,860 - root - INFO - step: 32685 loss: 16.8105 memory: 44.58GiB(31.99%) tps: 82,843 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.2089 +[titan] 2025-07-09 20:47:47,860 - root - INFO - lr: 2.3732e-04 gnorm: 0.80 [ 7:12:49<14:51:24] +[titan] 2025-07-09 20:47:51,768 - root - INFO - step: 32690 loss: 16.7258 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.1319 +[titan] 2025-07-09 20:47:51,768 - root - INFO - lr: 2.3730e-04 gnorm: 0.89 [ 7:12:53<14:51:20] +[titan] 2025-07-09 20:47:55,691 - root - INFO - step: 32695 loss: 16.5699 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.0005 +[titan] 2025-07-09 20:47:55,692 - root - INFO - lr: 2.3728e-04 gnorm: 0.81 [ 7:12:57<14:51:16] +[titan] 2025-07-09 20:47:58,805 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:47:59,601 - root - INFO - step: 32700 loss: 16.7425 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5961 global_avg_mtp_loss: 14.1463 +[titan] 2025-07-09 20:47:59,602 - root - INFO - lr: 2.3726e-04 gnorm: 0.88 [ 7:13:01<14:51:12] +[titan] 2025-07-09 20:48:03,503 - root - INFO - step: 32705 loss: 16.7891 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.1931 +[titan] 2025-07-09 20:48:03,503 - root - INFO - lr: 2.3724e-04 gnorm: 0.84 [ 7:13:05<14:51:08] +[titan] 2025-07-09 20:48:07,417 - root - INFO - step: 32710 loss: 16.6164 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.0376 +[titan] 2025-07-09 20:48:07,417 - root - INFO - lr: 2.3723e-04 gnorm: 0.79 [ 7:13:09<14:51:03] +[titan] 2025-07-09 20:48:11,343 - root - INFO - step: 32715 loss: 16.6701 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.0678 +[titan] 2025-07-09 20:48:11,343 - root - INFO - lr: 2.3721e-04 gnorm: 0.76 [ 7:13:12<14:50:59] +[titan] 2025-07-09 20:48:15,305 - root - INFO - step: 32720 loss: 16.7733 memory: 44.58GiB(31.99%) tps: 82,708 tflops: 285.44 mfu: 28.86% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1770 +[titan] 2025-07-09 20:48:15,306 - root - INFO - lr: 2.3719e-04 gnorm: 0.75 [ 7:13:16<14:50:55] +[titan] 2025-07-09 20:48:19,236 - root - INFO - step: 32725 loss: 16.8109 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.1996 +[titan] 2025-07-09 20:48:19,236 - root - INFO - lr: 2.3717e-04 gnorm: 0.80 [ 7:13:20<14:50:51] +[titan] 2025-07-09 20:48:23,137 - root - INFO - step: 32730 loss: 16.6495 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.0610 +[titan] 2025-07-09 20:48:23,137 - root - INFO - lr: 2.3715e-04 gnorm: 0.71 [ 7:13:24<14:50:47] +[titan] 2025-07-09 20:48:27,062 - root - INFO - step: 32735 loss: 16.6092 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0559 +[titan] 2025-07-09 20:48:27,062 - root - INFO - lr: 2.3714e-04 gnorm: 0.76 [ 7:13:28<14:50:43] +[titan] 2025-07-09 20:48:30,971 - root - INFO - step: 32740 loss: 16.5435 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9929 +[titan] 2025-07-09 20:48:30,971 - root - INFO - lr: 2.3712e-04 gnorm: 0.74 [ 7:13:32<14:50:39] +[titan] 2025-07-09 20:48:34,900 - root - INFO - step: 32745 loss: 16.7202 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1257 +[titan] 2025-07-09 20:48:34,900 - root - INFO - lr: 2.3710e-04 gnorm: 0.80 [ 7:13:36<14:50:35] +[titan] 2025-07-09 20:48:38,036 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:48:38,829 - root - INFO - step: 32750 loss: 16.5674 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0080 +[titan] 2025-07-09 20:48:38,829 - root - INFO - lr: 2.3708e-04 gnorm: 0.79 [ 7:13:40<14:50:31] +[titan] 2025-07-09 20:48:42,747 - root - INFO - step: 32755 loss: 16.7495 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1554 +[titan] 2025-07-09 20:48:42,748 - root - INFO - lr: 2.3706e-04 gnorm: 0.78 [ 7:13:44<14:50:27] +[titan] 2025-07-09 20:48:46,636 - root - INFO - step: 32760 loss: 16.9609 memory: 44.58GiB(31.99%) tps: 84,272 tflops: 290.84 mfu: 29.41% global_avg_ntp_loss: 2.6437 global_avg_mtp_loss: 14.3172 +[titan] 2025-07-09 20:48:46,636 - root - INFO - lr: 2.3704e-04 gnorm: 0.76 [ 7:13:48<14:50:23] +[titan] 2025-07-09 20:48:50,609 - root - INFO - step: 32765 loss: 16.3289 memory: 44.58GiB(31.99%) tps: 82,484 tflops: 284.67 mfu: 28.78% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-09 20:48:50,609 - root - INFO - lr: 2.3703e-04 gnorm: 0.87 [ 7:13:52<14:50:19] +[titan] 2025-07-09 20:48:53,118 - root - INFO - Dumping profiler traces at step 32768 +[titan] 2025-07-09 20:48:53,152 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:48:54,792 - root - INFO - step: 32770 loss: 16.8476 memory: 44.58GiB(31.99%) tps: 78,351 tflops: 270.40 mfu: 27.34% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.2398 +[titan] 2025-07-09 20:48:54,792 - root - INFO - lr: 2.3701e-04 gnorm: 0.80 [ 7:13:56<14:50:15] +[titan] 2025-07-09 20:48:58,703 - root - INFO - step: 32775 loss: 17.0042 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.3640 +[titan] 2025-07-09 20:48:58,703 - root - INFO - lr: 2.3699e-04 gnorm: 0.83 [ 7:14:00<14:50:11] +[titan] 2025-07-09 20:49:02,627 - root - INFO - step: 32780 loss: 16.6773 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.0992 +[titan] 2025-07-09 20:49:02,627 - root - INFO - lr: 2.3697e-04 gnorm: 0.80 [ 7:14:04<14:50:07] +[titan] 2025-07-09 20:49:06,551 - root - INFO - step: 32785 loss: 16.8246 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.2202 +[titan] 2025-07-09 20:49:06,551 - root - INFO - lr: 2.3695e-04 gnorm: 0.75 [ 7:14:08<14:50:03] +[titan] 2025-07-09 20:49:10,484 - root - INFO - step: 32790 loss: 16.6107 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0373 +[titan] 2025-07-09 20:49:10,484 - root - INFO - lr: 2.3694e-04 gnorm: 0.77 [ 7:14:12<14:49:59] +[titan] 2025-07-09 20:49:14,455 - root - INFO - step: 32795 loss: 16.6704 memory: 44.58GiB(31.99%) tps: 82,525 tflops: 284.81 mfu: 28.80% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.0916 +[titan] 2025-07-09 20:49:14,455 - root - INFO - lr: 2.3692e-04 gnorm: 0.77 [ 7:14:16<14:49:55] +[titan] 2025-07-09 20:49:17,631 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:49:18,430 - root - INFO - step: 32800 loss: 16.7450 memory: 44.58GiB(31.99%) tps: 82,447 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1494 +[titan] 2025-07-09 20:49:18,430 - root - INFO - lr: 2.3690e-04 gnorm: 0.80 [ 7:14:20<14:49:51] +[titan] 2025-07-09 20:49:22,360 - root - INFO - step: 32805 loss: 16.6422 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0749 +[titan] 2025-07-09 20:49:22,361 - root - INFO - lr: 2.3688e-04 gnorm: 0.78 [ 7:14:24<14:49:47] +[titan] 2025-07-09 20:49:26,278 - root - INFO - step: 32810 loss: 16.6245 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-09 20:49:26,278 - root - INFO - lr: 2.3686e-04 gnorm: 0.76 [ 7:14:27<14:49:43] +[titan] 2025-07-09 20:49:30,228 - root - INFO - step: 32815 loss: 16.8349 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.6076 global_avg_mtp_loss: 14.2273 +[titan] 2025-07-09 20:49:30,229 - root - INFO - lr: 2.3685e-04 gnorm: 0.75 [ 7:14:31<14:49:39] +[titan] 2025-07-09 20:49:34,128 - root - INFO - step: 32820 loss: 16.7118 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1309 +[titan] 2025-07-09 20:49:34,128 - root - INFO - lr: 2.3683e-04 gnorm: 0.73 [ 7:14:35<14:49:35] +[titan] 2025-07-09 20:49:38,023 - root - INFO - step: 32825 loss: 16.9877 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.6393 global_avg_mtp_loss: 14.3485 +[titan] 2025-07-09 20:49:38,023 - root - INFO - lr: 2.3681e-04 gnorm: 0.77 [ 7:14:39<14:49:30] +[titan] 2025-07-09 20:49:41,955 - root - INFO - step: 32830 loss: 17.0028 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6380 global_avg_mtp_loss: 14.3648 +[titan] 2025-07-09 20:49:41,956 - root - INFO - lr: 2.3679e-04 gnorm: 0.73 [ 7:14:43<14:49:26] +[titan] 2025-07-09 20:49:45,898 - root - INFO - step: 32835 loss: 16.7305 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1412 +[titan] 2025-07-09 20:49:45,898 - root - INFO - lr: 2.3677e-04 gnorm: 0.74 [ 7:14:47<14:49:22] +[titan] 2025-07-09 20:49:49,834 - root - INFO - step: 32840 loss: 16.7713 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1791 +[titan] 2025-07-09 20:49:49,835 - root - INFO - lr: 2.3675e-04 gnorm: 0.83 [ 7:14:51<14:49:18] +[titan] 2025-07-09 20:49:53,732 - root - INFO - step: 32845 loss: 16.7973 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.6074 global_avg_mtp_loss: 14.1899 +[titan] 2025-07-09 20:49:53,732 - root - INFO - lr: 2.3674e-04 gnorm: 0.82 [ 7:14:55<14:49:14] +[titan] 2025-07-09 20:49:56,864 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:49:57,655 - root - INFO - step: 32850 loss: 16.6560 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0780 +[titan] 2025-07-09 20:49:57,655 - root - INFO - lr: 2.3672e-04 gnorm: 0.81 [ 7:14:59<14:49:10] +[titan] 2025-07-09 20:50:01,563 - root - INFO - step: 32855 loss: 16.5467 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-09 20:50:01,563 - root - INFO - lr: 2.3670e-04 gnorm: 0.78 [ 7:15:03<14:49:06] +[titan] 2025-07-09 20:50:05,509 - root - INFO - step: 32860 loss: 16.8501 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.2434 +[titan] 2025-07-09 20:50:05,509 - root - INFO - lr: 2.3668e-04 gnorm: 0.73 [ 7:15:07<14:49:02] +[titan] 2025-07-09 20:50:09,454 - root - INFO - step: 32865 loss: 16.9523 memory: 44.58GiB(31.99%) tps: 83,068 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.6250 global_avg_mtp_loss: 14.3273 +[titan] 2025-07-09 20:50:09,455 - root - INFO - lr: 2.3666e-04 gnorm: 0.78 [ 7:15:11<14:48:58] +[titan] 2025-07-09 20:50:13,399 - root - INFO - step: 32870 loss: 16.5817 memory: 44.58GiB(31.99%) tps: 83,076 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0236 +[titan] 2025-07-09 20:50:13,399 - root - INFO - lr: 2.3665e-04 gnorm: 0.74 [ 7:15:15<14:48:54] +[titan] 2025-07-09 20:50:17,337 - root - INFO - step: 32875 loss: 16.9794 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6457 global_avg_mtp_loss: 14.3338 +[titan] 2025-07-09 20:50:17,337 - root - INFO - lr: 2.3663e-04 gnorm: 0.74 [ 7:15:18<14:48:50] +[titan] 2025-07-09 20:50:21,247 - root - INFO - step: 32880 loss: 16.8282 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2144 +[titan] 2025-07-09 20:50:21,247 - root - INFO - lr: 2.3661e-04 gnorm: 0.77 [ 7:15:22<14:48:46] +[titan] 2025-07-09 20:50:25,159 - root - INFO - step: 32885 loss: 16.8984 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2798 +[titan] 2025-07-09 20:50:25,159 - root - INFO - lr: 2.3659e-04 gnorm: 0.73 [ 7:15:26<14:48:42] +[titan] 2025-07-09 20:50:29,106 - root - INFO - step: 32890 loss: 16.7138 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1298 +[titan] 2025-07-09 20:50:29,107 - root - INFO - lr: 2.3657e-04 gnorm: 0.79 [ 7:15:30<14:48:38] +[titan] 2025-07-09 20:50:33,030 - root - INFO - step: 32895 loss: 16.6152 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5805 global_avg_mtp_loss: 14.0347 +[titan] 2025-07-09 20:50:33,031 - root - INFO - lr: 2.3655e-04 gnorm: 0.75 [ 7:15:34<14:48:34] +[titan] 2025-07-09 20:50:36,157 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:50:36,948 - root - INFO - step: 32900 loss: 16.9971 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.6411 global_avg_mtp_loss: 14.3560 +[titan] 2025-07-09 20:50:36,949 - root - INFO - lr: 2.3654e-04 gnorm: 0.89 [ 7:15:38<14:48:29] +[titan] 2025-07-09 20:50:40,879 - root - INFO - step: 32905 loss: 16.5050 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9572 +[titan] 2025-07-09 20:50:40,879 - root - INFO - lr: 2.3652e-04 gnorm: 0.76 [ 7:15:42<14:48:25] +[titan] 2025-07-09 20:50:44,822 - root - INFO - step: 32910 loss: 16.5730 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0160 +[titan] 2025-07-09 20:50:44,822 - root - INFO - lr: 2.3650e-04 gnorm: 0.79 [ 7:15:46<14:48:21] +[titan] 2025-07-09 20:50:48,745 - root - INFO - step: 32915 loss: 16.5560 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 13.9850 +[titan] 2025-07-09 20:50:48,745 - root - INFO - lr: 2.3648e-04 gnorm: 0.80 [ 7:15:50<14:48:17] +[titan] 2025-07-09 20:50:52,659 - root - INFO - step: 32920 loss: 16.9138 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6354 global_avg_mtp_loss: 14.2784 +[titan] 2025-07-09 20:50:52,659 - root - INFO - lr: 2.3646e-04 gnorm: 0.75 [ 7:15:54<14:48:13] +[titan] 2025-07-09 20:50:56,560 - root - INFO - step: 32925 loss: 16.8347 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2261 +[titan] 2025-07-09 20:50:56,560 - root - INFO - lr: 2.3645e-04 gnorm: 0.77 [ 7:15:58<14:48:09] +[titan] 2025-07-09 20:51:00,509 - root - INFO - step: 32930 loss: 16.5442 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9924 +[titan] 2025-07-09 20:51:00,509 - root - INFO - lr: 2.3643e-04 gnorm: 0.80 [ 7:16:02<14:48:05] +[titan] 2025-07-09 20:51:04,428 - root - INFO - step: 32935 loss: 16.5534 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 13.9890 +[titan] 2025-07-09 20:51:04,428 - root - INFO - lr: 2.3641e-04 gnorm: 0.78 [ 7:16:06<14:48:01] +[titan] 2025-07-09 20:51:08,355 - root - INFO - step: 32940 loss: 17.0940 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6533 global_avg_mtp_loss: 14.4407 +[titan] 2025-07-09 20:51:08,355 - root - INFO - lr: 2.3639e-04 gnorm: 0.80 [ 7:16:09<14:47:57] +[titan] 2025-07-09 20:51:12,289 - root - INFO - step: 32945 loss: 16.8496 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6323 global_avg_mtp_loss: 14.2173 +[titan] 2025-07-09 20:51:12,289 - root - INFO - lr: 2.3637e-04 gnorm: 0.72 [ 7:16:13<14:47:53] +[titan] 2025-07-09 20:51:15,417 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:51:16,204 - root - INFO - step: 32950 loss: 16.5566 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 13.9996 +[titan] 2025-07-09 20:51:16,204 - root - INFO - lr: 2.3635e-04 gnorm: 0.85 [ 7:16:17<14:47:49] +[titan] 2025-07-09 20:51:20,159 - root - INFO - step: 32955 loss: 16.6835 memory: 44.58GiB(31.99%) tps: 82,853 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5786 global_avg_mtp_loss: 14.1049 +[titan] 2025-07-09 20:51:20,160 - root - INFO - lr: 2.3634e-04 gnorm: 0.77 [ 7:16:21<14:47:45] +[titan] 2025-07-09 20:51:24,116 - root - INFO - step: 32960 loss: 16.8961 memory: 44.58GiB(31.99%) tps: 82,835 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2736 +[titan] 2025-07-09 20:51:24,116 - root - INFO - lr: 2.3632e-04 gnorm: 0.79 [ 7:16:25<14:47:41] +[titan] 2025-07-09 20:51:28,037 - root - INFO - step: 32965 loss: 16.8509 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.6027 global_avg_mtp_loss: 14.2483 +[titan] 2025-07-09 20:51:28,038 - root - INFO - lr: 2.3630e-04 gnorm: 0.75 [ 7:16:29<14:47:37] +[titan] 2025-07-09 20:51:31,946 - root - INFO - step: 32970 loss: 16.6873 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.1048 +[titan] 2025-07-09 20:51:31,947 - root - INFO - lr: 2.3628e-04 gnorm: 0.75 [ 7:16:33<14:47:33] +[titan] 2025-07-09 20:51:35,884 - root - INFO - step: 32975 loss: 16.6719 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0993 +[titan] 2025-07-09 20:51:35,884 - root - INFO - lr: 2.3626e-04 gnorm: 0.77 [ 7:16:37<14:47:29] +[titan] 2025-07-09 20:51:39,815 - root - INFO - step: 32980 loss: 16.9131 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6245 global_avg_mtp_loss: 14.2886 +[titan] 2025-07-09 20:51:39,816 - root - INFO - lr: 2.3625e-04 gnorm: 0.81 [ 7:16:41<14:47:25] +[titan] 2025-07-09 20:51:43,723 - root - INFO - step: 32985 loss: 16.8632 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.2518 +[titan] 2025-07-09 20:51:43,724 - root - INFO - lr: 2.3623e-04 gnorm: 0.87 [ 7:16:45<14:47:20] +[titan] 2025-07-09 20:51:47,632 - root - INFO - step: 32990 loss: 16.8441 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6095 global_avg_mtp_loss: 14.2345 +[titan] 2025-07-09 20:51:47,632 - root - INFO - lr: 2.3621e-04 gnorm: 0.85 [ 7:16:49<14:47:16] +[titan] 2025-07-09 20:51:51,584 - root - INFO - step: 32995 loss: 16.3573 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8423 +[titan] 2025-07-09 20:51:51,585 - root - INFO - lr: 2.3619e-04 gnorm: 0.86 [ 7:16:53<14:47:12] +[titan] 2025-07-09 20:51:54,737 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:51:55,540 - root - INFO - step: 33000 loss: 16.6361 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.0489 +[titan] 2025-07-09 20:51:55,540 - root - INFO - lr: 2.3617e-04 gnorm: 0.74 [ 7:16:57<14:47:08] +[titan] 2025-07-09 20:51:59,539 - root - INFO - step: 33005 loss: 16.7090 memory: 44.58GiB(31.99%) tps: 81,943 tflops: 282.80 mfu: 28.59% global_avg_ntp_loss: 2.5909 global_avg_mtp_loss: 14.1181 +[titan] 2025-07-09 20:51:59,539 - root - INFO - lr: 2.3615e-04 gnorm: 0.82 [ 7:17:01<14:47:04] +[titan] 2025-07-09 20:52:03,494 - root - INFO - step: 33010 loss: 16.8136 memory: 44.58GiB(31.99%) tps: 82,849 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.1983 +[titan] 2025-07-09 20:52:03,495 - root - INFO - lr: 2.3614e-04 gnorm: 0.76 [ 7:17:05<14:47:00] +[titan] 2025-07-09 20:52:07,440 - root - INFO - step: 33015 loss: 16.5964 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0230 +[titan] 2025-07-09 20:52:07,440 - root - INFO - lr: 2.3612e-04 gnorm: 0.82 [ 7:17:09<14:46:56] +[titan] 2025-07-09 20:52:11,358 - root - INFO - step: 33020 loss: 16.6634 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0942 +[titan] 2025-07-09 20:52:11,358 - root - INFO - lr: 2.3610e-04 gnorm: 0.73 [ 7:17:12<14:46:52] +[titan] 2025-07-09 20:52:15,258 - root - INFO - step: 33025 loss: 16.9915 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6480 global_avg_mtp_loss: 14.3435 +[titan] 2025-07-09 20:52:15,258 - root - INFO - lr: 2.3608e-04 gnorm: 0.79 [ 7:17:16<14:46:48] +[titan] 2025-07-09 20:52:19,167 - root - INFO - step: 33030 loss: 16.7853 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5984 global_avg_mtp_loss: 14.1868 +[titan] 2025-07-09 20:52:19,167 - root - INFO - lr: 2.3606e-04 gnorm: 0.71 [ 7:17:20<14:46:44] +[titan] 2025-07-09 20:52:23,066 - root - INFO - step: 33035 loss: 16.5285 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 13.9710 +[titan] 2025-07-09 20:52:23,066 - root - INFO - lr: 2.3605e-04 gnorm: 0.75 [ 7:17:24<14:46:40] +[titan] 2025-07-09 20:52:26,979 - root - INFO - step: 33040 loss: 16.7334 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.1342 +[titan] 2025-07-09 20:52:26,979 - root - INFO - lr: 2.3603e-04 gnorm: 0.81 [ 7:17:28<14:46:36] +[titan] 2025-07-09 20:52:30,890 - root - INFO - step: 33045 loss: 16.3566 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-09 20:52:30,890 - root - INFO - lr: 2.3601e-04 gnorm: 0.82 [ 7:17:32<14:46:32] +[titan] 2025-07-09 20:52:34,007 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:52:34,794 - root - INFO - step: 33050 loss: 16.7089 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 14.1252 +[titan] 2025-07-09 20:52:34,794 - root - INFO - lr: 2.3599e-04 gnorm: 0.77 [ 7:17:36<14:46:28] +[titan] 2025-07-09 20:52:38,722 - root - INFO - step: 33055 loss: 16.4975 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9550 +[titan] 2025-07-09 20:52:38,723 - root - INFO - lr: 2.3597e-04 gnorm: 0.74 [ 7:17:40<14:46:24] +[titan] 2025-07-09 20:52:42,641 - root - INFO - step: 33060 loss: 16.9743 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6390 global_avg_mtp_loss: 14.3353 +[titan] 2025-07-09 20:52:42,641 - root - INFO - lr: 2.3595e-04 gnorm: 0.79 [ 7:17:44<14:46:19] +[titan] 2025-07-09 20:52:46,543 - root - INFO - step: 33065 loss: 16.5787 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.0100 +[titan] 2025-07-09 20:52:46,543 - root - INFO - lr: 2.3594e-04 gnorm: 0.87 [ 7:17:48<14:46:15] +[titan] 2025-07-09 20:52:50,490 - root - INFO - step: 33070 loss: 16.8667 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.6080 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 20:52:50,490 - root - INFO - lr: 2.3592e-04 gnorm: 0.82 [ 7:17:52<14:46:11] +[titan] 2025-07-09 20:52:54,417 - root - INFO - step: 33075 loss: 16.8538 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6122 global_avg_mtp_loss: 14.2416 +[titan] 2025-07-09 20:52:54,418 - root - INFO - lr: 2.3590e-04 gnorm: 0.79 [ 7:17:56<14:46:07] +[titan] 2025-07-09 20:52:58,334 - root - INFO - step: 33080 loss: 16.8172 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.2224 +[titan] 2025-07-09 20:52:58,334 - root - INFO - lr: 2.3588e-04 gnorm: 0.73 [ 7:17:59<14:46:03] +[titan] 2025-07-09 20:53:02,269 - root - INFO - step: 33085 loss: 16.4915 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 13.9406 +[titan] 2025-07-09 20:53:02,269 - root - INFO - lr: 2.3586e-04 gnorm: 0.78 [ 7:18:03<14:45:59] +[titan] 2025-07-09 20:53:06,189 - root - INFO - step: 33090 loss: 16.6223 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-09 20:53:06,189 - root - INFO - lr: 2.3585e-04 gnorm: 0.73 [ 7:18:07<14:45:55] +[titan] 2025-07-09 20:53:10,139 - root - INFO - step: 33095 loss: 16.9931 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.6375 global_avg_mtp_loss: 14.3556 +[titan] 2025-07-09 20:53:10,140 - root - INFO - lr: 2.3583e-04 gnorm: 0.78 [ 7:18:11<14:45:51] +[titan] 2025-07-09 20:53:13,280 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:53:14,076 - root - INFO - step: 33100 loss: 16.7770 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1805 +[titan] 2025-07-09 20:53:14,077 - root - INFO - lr: 2.3581e-04 gnorm: 0.77 [ 7:18:15<14:45:47] +[titan] 2025-07-09 20:53:18,020 - root - INFO - step: 33105 loss: 16.6681 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-09 20:53:18,021 - root - INFO - lr: 2.3579e-04 gnorm: 0.78 [ 7:18:19<14:45:43] +[titan] 2025-07-09 20:53:21,921 - root - INFO - step: 33110 loss: 16.7758 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1849 +[titan] 2025-07-09 20:53:21,921 - root - INFO - lr: 2.3577e-04 gnorm: 0.77 [ 7:18:23<14:45:39] +[titan] 2025-07-09 20:53:25,857 - root - INFO - step: 33115 loss: 16.7789 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.1879 +[titan] 2025-07-09 20:53:25,858 - root - INFO - lr: 2.3575e-04 gnorm: 0.72 [ 7:18:27<14:45:35] +[titan] 2025-07-09 20:53:29,770 - root - INFO - step: 33120 loss: 16.5495 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 14.0007 +[titan] 2025-07-09 20:53:29,770 - root - INFO - lr: 2.3574e-04 gnorm: 0.80 [ 7:18:31<14:45:31] +[titan] 2025-07-09 20:53:33,670 - root - INFO - step: 33125 loss: 16.8506 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.6092 global_avg_mtp_loss: 14.2414 +[titan] 2025-07-09 20:53:33,670 - root - INFO - lr: 2.3572e-04 gnorm: 0.77 [ 7:18:35<14:45:27] +[titan] 2025-07-09 20:53:37,586 - root - INFO - step: 33130 loss: 16.9124 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.2804 +[titan] 2025-07-09 20:53:37,586 - root - INFO - lr: 2.3570e-04 gnorm: 0.80 [ 7:18:39<14:45:22] +[titan] 2025-07-09 20:53:41,524 - root - INFO - step: 33135 loss: 16.7037 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1191 +[titan] 2025-07-09 20:53:41,524 - root - INFO - lr: 2.3568e-04 gnorm: 0.76 [ 7:18:43<14:45:18] +[titan] 2025-07-09 20:53:45,471 - root - INFO - step: 33140 loss: 16.8866 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.6124 global_avg_mtp_loss: 14.2742 +[titan] 2025-07-09 20:53:45,472 - root - INFO - lr: 2.3566e-04 gnorm: 0.77 [ 7:18:47<14:45:14] +[titan] 2025-07-09 20:53:49,484 - root - INFO - step: 33145 loss: 16.8043 memory: 44.58GiB(31.99%) tps: 81,673 tflops: 281.87 mfu: 28.50% global_avg_ntp_loss: 2.5895 global_avg_mtp_loss: 14.2147 +[titan] 2025-07-09 20:53:49,484 - root - INFO - lr: 2.3564e-04 gnorm: 0.73 [ 7:18:51<14:45:11] +[titan] 2025-07-09 20:53:52,586 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:53:53,375 - root - INFO - step: 33150 loss: 16.8219 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.2290 +[titan] 2025-07-09 20:53:53,375 - root - INFO - lr: 2.3563e-04 gnorm: 0.75 [ 7:18:54<14:45:06] +[titan] 2025-07-09 20:53:57,292 - root - INFO - step: 33155 loss: 16.9247 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.3124 +[titan] 2025-07-09 20:53:57,292 - root - INFO - lr: 2.3561e-04 gnorm: 0.75 [ 7:18:58<14:45:02] +[titan] 2025-07-09 20:54:01,192 - root - INFO - step: 33160 loss: 16.6774 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.1038 +[titan] 2025-07-09 20:54:01,192 - root - INFO - lr: 2.3559e-04 gnorm: 0.73 [ 7:19:02<14:44:58] +[titan] 2025-07-09 20:54:05,103 - root - INFO - step: 33165 loss: 16.8035 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.2012 +[titan] 2025-07-09 20:54:05,103 - root - INFO - lr: 2.3557e-04 gnorm: 0.71 [ 7:19:06<14:44:54] +[titan] 2025-07-09 20:54:08,995 - root - INFO - step: 33170 loss: 16.6513 memory: 44.58GiB(31.99%) tps: 84,205 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0818 +[titan] 2025-07-09 20:54:08,995 - root - INFO - lr: 2.3555e-04 gnorm: 0.73 [ 7:19:10<14:44:50] +[titan] 2025-07-09 20:54:12,928 - root - INFO - step: 33175 loss: 16.7864 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.1757 +[titan] 2025-07-09 20:54:12,929 - root - INFO - lr: 2.3553e-04 gnorm: 0.77 [ 7:19:14<14:44:46] +[titan] 2025-07-09 20:54:16,833 - root - INFO - step: 33180 loss: 16.5194 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 13.9637 +[titan] 2025-07-09 20:54:16,833 - root - INFO - lr: 2.3552e-04 gnorm: 0.83 [ 7:19:18<14:44:42] +[titan] 2025-07-09 20:54:20,750 - root - INFO - step: 33185 loss: 16.6589 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5595 global_avg_mtp_loss: 14.0993 +[titan] 2025-07-09 20:54:20,750 - root - INFO - lr: 2.3550e-04 gnorm: 0.72 [ 7:19:22<14:44:38] +[titan] 2025-07-09 20:54:24,676 - root - INFO - step: 33190 loss: 16.4051 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-09 20:54:24,677 - root - INFO - lr: 2.3548e-04 gnorm: 0.80 [ 7:19:26<14:44:34] +[titan] 2025-07-09 20:54:28,601 - root - INFO - step: 33195 loss: 16.7050 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.1328 +[titan] 2025-07-09 20:54:28,602 - root - INFO - lr: 2.3546e-04 gnorm: 0.73 [ 7:19:30<14:44:30] +[titan] 2025-07-09 20:54:31,720 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:54:32,513 - root - INFO - step: 33200 loss: 16.5147 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9719 +[titan] 2025-07-09 20:54:32,513 - root - INFO - lr: 2.3544e-04 gnorm: 0.77 [ 7:19:34<14:44:26] +[titan] 2025-07-09 20:54:36,443 - root - INFO - step: 33205 loss: 16.8526 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.6145 global_avg_mtp_loss: 14.2382 +[titan] 2025-07-09 20:54:36,443 - root - INFO - lr: 2.3543e-04 gnorm: 0.85 [ 7:19:38<14:44:21] +[titan] 2025-07-09 20:54:40,373 - root - INFO - step: 33210 loss: 16.9443 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6291 global_avg_mtp_loss: 14.3152 +[titan] 2025-07-09 20:54:40,373 - root - INFO - lr: 2.3541e-04 gnorm: 0.82 [ 7:19:41<14:44:17] +[titan] 2025-07-09 20:54:44,314 - root - INFO - step: 33215 loss: 16.7785 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.6070 global_avg_mtp_loss: 14.1715 +[titan] 2025-07-09 20:54:44,315 - root - INFO - lr: 2.3539e-04 gnorm: 0.86 [ 7:19:45<14:44:13] +[titan] 2025-07-09 20:54:48,239 - root - INFO - step: 33220 loss: 16.7562 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.1532 +[titan] 2025-07-09 20:54:48,239 - root - INFO - lr: 2.3537e-04 gnorm: 0.84 [ 7:19:49<14:44:09] +[titan] 2025-07-09 20:54:52,151 - root - INFO - step: 33225 loss: 16.8670 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6041 global_avg_mtp_loss: 14.2630 +[titan] 2025-07-09 20:54:52,151 - root - INFO - lr: 2.3535e-04 gnorm: 0.79 [ 7:19:53<14:44:05] +[titan] 2025-07-09 20:54:56,087 - root - INFO - step: 33230 loss: 16.7236 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1414 +[titan] 2025-07-09 20:54:56,087 - root - INFO - lr: 2.3533e-04 gnorm: 0.76 [ 7:19:57<14:44:01] +[titan] 2025-07-09 20:55:00,003 - root - INFO - step: 33235 loss: 16.5989 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 20:55:00,004 - root - INFO - lr: 2.3532e-04 gnorm: 0.80 [ 7:20:01<14:43:57] +[titan] 2025-07-09 20:55:03,934 - root - INFO - step: 33240 loss: 16.6194 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.0356 +[titan] 2025-07-09 20:55:03,934 - root - INFO - lr: 2.3530e-04 gnorm: 0.78 [ 7:20:05<14:43:53] +[titan] 2025-07-09 20:55:07,863 - root - INFO - step: 33245 loss: 16.6080 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0370 +[titan] 2025-07-09 20:55:07,863 - root - INFO - lr: 2.3528e-04 gnorm: 0.82 [ 7:20:09<14:43:49] +[titan] 2025-07-09 20:55:10,974 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:55:11,759 - root - INFO - step: 33250 loss: 16.5348 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 13.9804 +[titan] 2025-07-09 20:55:11,759 - root - INFO - lr: 2.3526e-04 gnorm: 0.81 [ 7:20:13<14:43:45] +[titan] 2025-07-09 20:55:15,684 - root - INFO - step: 33255 loss: 16.5899 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0226 +[titan] 2025-07-09 20:55:15,684 - root - INFO - lr: 2.3524e-04 gnorm: 0.79 [ 7:20:17<14:43:41] +[titan] 2025-07-09 20:55:19,607 - root - INFO - step: 33260 loss: 16.7204 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1241 +[titan] 2025-07-09 20:55:19,608 - root - INFO - lr: 2.3522e-04 gnorm: 0.75 [ 7:20:21<14:43:37] +[titan] 2025-07-09 20:55:23,518 - root - INFO - step: 33265 loss: 16.7098 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1219 +[titan] 2025-07-09 20:55:23,518 - root - INFO - lr: 2.3521e-04 gnorm: 0.81 [ 7:20:25<14:43:33] +[titan] 2025-07-09 20:55:27,439 - root - INFO - step: 33270 loss: 16.4488 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9075 +[titan] 2025-07-09 20:55:27,439 - root - INFO - lr: 2.3519e-04 gnorm: 0.77 [ 7:20:29<14:43:29] +[titan] 2025-07-09 20:55:31,342 - root - INFO - step: 33275 loss: 16.5890 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.0189 +[titan] 2025-07-09 20:55:31,342 - root - INFO - lr: 2.3517e-04 gnorm: 0.75 [ 7:20:32<14:43:24] +[titan] 2025-07-09 20:55:35,323 - root - INFO - step: 33280 loss: 16.7506 memory: 44.58GiB(31.99%) tps: 82,315 tflops: 284.08 mfu: 28.72% global_avg_ntp_loss: 2.5887 global_avg_mtp_loss: 14.1618 +[titan] 2025-07-09 20:55:35,324 - root - INFO - lr: 2.3515e-04 gnorm: 0.78 [ 7:20:36<14:43:20] +[titan] 2025-07-09 20:55:35,477 - root - INFO - Dumping profiler traces at step 33280 +[titan] 2025-07-09 20:55:35,509 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 20:55:39,414 - root - INFO - step: 33285 loss: 16.8396 memory: 44.58GiB(31.99%) tps: 80,104 tflops: 276.45 mfu: 27.95% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.2388 +[titan] 2025-07-09 20:55:39,415 - root - INFO - lr: 2.3513e-04 gnorm: 0.76 [ 7:20:41<14:43:17] +[titan] 2025-07-09 20:55:43,333 - root - INFO - step: 33290 loss: 16.6974 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5768 global_avg_mtp_loss: 14.1206 +[titan] 2025-07-09 20:55:43,334 - root - INFO - lr: 2.3511e-04 gnorm: 0.79 [ 7:20:44<14:43:13] +[titan] 2025-07-09 20:55:47,222 - root - INFO - step: 33295 loss: 16.6345 memory: 44.58GiB(31.99%) tps: 84,280 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0701 +[titan] 2025-07-09 20:55:47,222 - root - INFO - lr: 2.3510e-04 gnorm: 0.74 [ 7:20:48<14:43:08] +[titan] 2025-07-09 20:55:50,322 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:55:51,111 - root - INFO - step: 33300 loss: 16.7608 memory: 44.58GiB(31.99%) tps: 84,266 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.1599 +[titan] 2025-07-09 20:55:51,111 - root - INFO - lr: 2.3508e-04 gnorm: 0.70 [ 7:20:52<14:43:04] +[titan] 2025-07-09 20:55:55,009 - root - INFO - step: 33305 loss: 16.5508 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-09 20:55:55,009 - root - INFO - lr: 2.3506e-04 gnorm: 0.74 [ 7:20:56<14:43:00] +[titan] 2025-07-09 20:55:58,910 - root - INFO - step: 33310 loss: 16.7568 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1509 +[titan] 2025-07-09 20:55:58,910 - root - INFO - lr: 2.3504e-04 gnorm: 0.86 [ 7:21:00<14:42:56] +[titan] 2025-07-09 20:56:02,797 - root - INFO - step: 33315 loss: 16.8258 memory: 44.58GiB(31.99%) tps: 84,304 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2084 +[titan] 2025-07-09 20:56:02,798 - root - INFO - lr: 2.3502e-04 gnorm: 0.85 [ 7:21:04<14:42:52] +[titan] 2025-07-09 20:56:06,706 - root - INFO - step: 33320 loss: 16.7417 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.1519 +[titan] 2025-07-09 20:56:06,706 - root - INFO - lr: 2.3500e-04 gnorm: 0.82 [ 7:21:08<14:42:48] +[titan] 2025-07-09 20:56:10,593 - root - INFO - step: 33325 loss: 16.8162 memory: 44.58GiB(31.99%) tps: 84,310 tflops: 290.97 mfu: 29.42% global_avg_ntp_loss: 2.6127 global_avg_mtp_loss: 14.2035 +[titan] 2025-07-09 20:56:10,593 - root - INFO - lr: 2.3499e-04 gnorm: 0.74 [ 7:21:12<14:42:44] +[titan] 2025-07-09 20:56:14,508 - root - INFO - step: 33330 loss: 16.6420 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0680 +[titan] 2025-07-09 20:56:14,508 - root - INFO - lr: 2.3497e-04 gnorm: 0.74 [ 7:21:16<14:42:40] +[titan] 2025-07-09 20:56:18,454 - root - INFO - step: 33335 loss: 16.7283 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 20:56:18,454 - root - INFO - lr: 2.3495e-04 gnorm: 0.78 [ 7:21:20<14:42:36] +[titan] 2025-07-09 20:56:22,388 - root - INFO - step: 33340 loss: 16.8333 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2093 +[titan] 2025-07-09 20:56:22,388 - root - INFO - lr: 2.3493e-04 gnorm: 0.77 [ 7:21:23<14:42:32] +[titan] 2025-07-09 20:56:26,272 - root - INFO - step: 33345 loss: 16.7917 memory: 44.58GiB(31.99%) tps: 84,370 tflops: 291.18 mfu: 29.44% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.1869 +[titan] 2025-07-09 20:56:26,272 - root - INFO - lr: 2.3491e-04 gnorm: 0.80 [ 7:21:27<14:42:27] +[titan] 2025-07-09 20:56:29,388 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:56:30,201 - root - INFO - step: 33350 loss: 16.5412 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-09 20:56:30,201 - root - INFO - lr: 2.3489e-04 gnorm: 0.76 [ 7:21:31<14:42:23] +[titan] 2025-07-09 20:56:34,097 - root - INFO - step: 33355 loss: 16.6273 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0631 +[titan] 2025-07-09 20:56:34,097 - root - INFO - lr: 2.3488e-04 gnorm: 0.82 [ 7:21:35<14:42:19] +[titan] 2025-07-09 20:56:38,013 - root - INFO - step: 33360 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-09 20:56:38,013 - root - INFO - lr: 2.3486e-04 gnorm: 0.79 [ 7:21:39<14:42:15] +[titan] 2025-07-09 20:56:41,929 - root - INFO - step: 33365 loss: 16.3677 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8439 +[titan] 2025-07-09 20:56:41,929 - root - INFO - lr: 2.3484e-04 gnorm: 0.78 [ 7:21:43<14:42:11] +[titan] 2025-07-09 20:56:45,839 - root - INFO - step: 33370 loss: 16.7389 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1496 +[titan] 2025-07-09 20:56:45,840 - root - INFO - lr: 2.3482e-04 gnorm: 0.71 [ 7:21:47<14:42:07] +[titan] 2025-07-09 20:56:49,734 - root - INFO - step: 33375 loss: 16.5689 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 13.9984 +[titan] 2025-07-09 20:56:49,734 - root - INFO - lr: 2.3480e-04 gnorm: 0.75 [ 7:21:51<14:42:03] +[titan] 2025-07-09 20:56:53,628 - root - INFO - step: 33380 loss: 16.7018 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.1121 +[titan] 2025-07-09 20:56:53,628 - root - INFO - lr: 2.3478e-04 gnorm: 0.84 [ 7:21:55<14:41:59] +[titan] 2025-07-09 20:56:57,543 - root - INFO - step: 33385 loss: 16.8057 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.2101 +[titan] 2025-07-09 20:56:57,543 - root - INFO - lr: 2.3477e-04 gnorm: 0.80 [ 7:21:59<14:41:55] +[titan] 2025-07-09 20:57:01,431 - root - INFO - step: 33390 loss: 16.5902 memory: 44.58GiB(31.99%) tps: 84,280 tflops: 290.87 mfu: 29.41% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0259 +[titan] 2025-07-09 20:57:01,432 - root - INFO - lr: 2.3475e-04 gnorm: 0.82 [ 7:22:03<14:41:51] +[titan] 2025-07-09 20:57:05,349 - root - INFO - step: 33395 loss: 17.0534 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6482 global_avg_mtp_loss: 14.4052 +[titan] 2025-07-09 20:57:05,349 - root - INFO - lr: 2.3473e-04 gnorm: 0.76 [ 7:22:06<14:41:46] +[titan] 2025-07-09 20:57:08,460 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:57:09,246 - root - INFO - step: 33400 loss: 16.8292 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.2183 +[titan] 2025-07-09 20:57:09,246 - root - INFO - lr: 2.3471e-04 gnorm: 0.80 [ 7:22:10<14:41:42] +[titan] 2025-07-09 20:57:13,143 - root - INFO - step: 33405 loss: 16.5070 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 13.9526 +[titan] 2025-07-09 20:57:13,143 - root - INFO - lr: 2.3469e-04 gnorm: 0.78 [ 7:22:14<14:41:38] +[titan] 2025-07-09 20:57:17,066 - root - INFO - step: 33410 loss: 16.7034 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1207 +[titan] 2025-07-09 20:57:17,066 - root - INFO - lr: 2.3467e-04 gnorm: 0.73 [ 7:22:18<14:41:34] +[titan] 2025-07-09 20:57:20,979 - root - INFO - step: 33415 loss: 16.5660 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0166 +[titan] 2025-07-09 20:57:20,979 - root - INFO - lr: 2.3466e-04 gnorm: 0.77 [ 7:22:22<14:41:30] +[titan] 2025-07-09 20:57:24,917 - root - INFO - step: 33420 loss: 16.6792 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.1088 +[titan] 2025-07-09 20:57:24,917 - root - INFO - lr: 2.3464e-04 gnorm: 0.77 [ 7:22:26<14:41:26] +[titan] 2025-07-09 20:57:28,849 - root - INFO - step: 33425 loss: 16.6569 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5792 global_avg_mtp_loss: 14.0777 +[titan] 2025-07-09 20:57:28,849 - root - INFO - lr: 2.3462e-04 gnorm: 0.77 [ 7:22:30<14:41:22] +[titan] 2025-07-09 20:57:32,766 - root - INFO - step: 33430 loss: 16.8335 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.2282 +[titan] 2025-07-09 20:57:32,766 - root - INFO - lr: 2.3460e-04 gnorm: 0.80 [ 7:22:34<14:41:18] +[titan] 2025-07-09 20:57:36,682 - root - INFO - step: 33435 loss: 16.7608 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.6035 global_avg_mtp_loss: 14.1573 +[titan] 2025-07-09 20:57:36,682 - root - INFO - lr: 2.3458e-04 gnorm: 0.76 [ 7:22:38<14:41:14] +[titan] 2025-07-09 20:57:40,583 - root - INFO - step: 33440 loss: 16.7363 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.1532 +[titan] 2025-07-09 20:57:40,584 - root - INFO - lr: 2.3456e-04 gnorm: 0.76 [ 7:22:42<14:41:10] +[titan] 2025-07-09 20:57:44,519 - root - INFO - step: 33445 loss: 16.7054 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1220 +[titan] 2025-07-09 20:57:44,519 - root - INFO - lr: 2.3455e-04 gnorm: 0.78 [ 7:22:46<14:41:06] +[titan] 2025-07-09 20:57:47,651 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:57:48,449 - root - INFO - step: 33450 loss: 16.7159 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1324 +[titan] 2025-07-09 20:57:48,450 - root - INFO - lr: 2.3453e-04 gnorm: 0.79 [ 7:22:50<14:41:02] +[titan] 2025-07-09 20:57:52,383 - root - INFO - step: 33455 loss: 16.5477 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9903 +[titan] 2025-07-09 20:57:52,383 - root - INFO - lr: 2.3451e-04 gnorm: 0.74 [ 7:22:53<14:40:58] +[titan] 2025-07-09 20:57:56,283 - root - INFO - step: 33460 loss: 16.8344 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2208 +[titan] 2025-07-09 20:57:56,283 - root - INFO - lr: 2.3449e-04 gnorm: 0.76 [ 7:22:57<14:40:53] +[titan] 2025-07-09 20:58:00,229 - root - INFO - step: 33465 loss: 17.0235 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.6539 global_avg_mtp_loss: 14.3696 +[titan] 2025-07-09 20:58:00,230 - root - INFO - lr: 2.3447e-04 gnorm: 0.75 [ 7:23:01<14:40:49] +[titan] 2025-07-09 20:58:04,165 - root - INFO - step: 33470 loss: 16.5243 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 13.9767 +[titan] 2025-07-09 20:58:04,166 - root - INFO - lr: 2.3445e-04 gnorm: 0.80 [ 7:23:05<14:40:45] +[titan] 2025-07-09 20:58:08,066 - root - INFO - step: 33475 loss: 16.7886 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1979 +[titan] 2025-07-09 20:58:08,066 - root - INFO - lr: 2.3444e-04 gnorm: 0.77 [ 7:23:09<14:40:41] +[titan] 2025-07-09 20:58:12,000 - root - INFO - step: 33480 loss: 16.8952 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.6225 global_avg_mtp_loss: 14.2727 +[titan] 2025-07-09 20:58:12,000 - root - INFO - lr: 2.3442e-04 gnorm: 0.82 [ 7:23:13<14:40:37] +[titan] 2025-07-09 20:58:15,885 - root - INFO - step: 33485 loss: 16.7578 memory: 44.58GiB(31.99%) tps: 84,340 tflops: 291.07 mfu: 29.43% global_avg_ntp_loss: 2.6026 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 20:58:15,885 - root - INFO - lr: 2.3440e-04 gnorm: 0.79 [ 7:23:17<14:40:33] +[titan] 2025-07-09 20:58:19,822 - root - INFO - step: 33490 loss: 17.1243 memory: 44.58GiB(31.99%) tps: 83,244 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.6631 global_avg_mtp_loss: 14.4612 +[titan] 2025-07-09 20:58:19,822 - root - INFO - lr: 2.3438e-04 gnorm: 0.76 [ 7:23:21<14:40:29] +[titan] 2025-07-09 20:58:23,755 - root - INFO - step: 33495 loss: 16.5462 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 14.0073 +[titan] 2025-07-09 20:58:23,755 - root - INFO - lr: 2.3436e-04 gnorm: 0.72 [ 7:23:25<14:40:25] +[titan] 2025-07-09 20:58:26,895 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:58:27,681 - root - INFO - step: 33500 loss: 16.6304 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-09 20:58:27,681 - root - INFO - lr: 2.3434e-04 gnorm: 0.69 [ 7:23:29<14:40:21] +[titan] 2025-07-09 20:58:31,582 - root - INFO - step: 33505 loss: 16.9433 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6443 global_avg_mtp_loss: 14.2990 +[titan] 2025-07-09 20:58:31,582 - root - INFO - lr: 2.3433e-04 gnorm: 0.72 [ 7:23:33<14:40:17] +[titan] 2025-07-09 20:58:35,511 - root - INFO - step: 33510 loss: 16.6962 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.1166 +[titan] 2025-07-09 20:58:35,511 - root - INFO - lr: 2.3431e-04 gnorm: 0.86 [ 7:23:37<14:40:13] +[titan] 2025-07-09 20:58:39,434 - root - INFO - step: 33515 loss: 16.9601 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6371 global_avg_mtp_loss: 14.3229 +[titan] 2025-07-09 20:58:39,434 - root - INFO - lr: 2.3429e-04 gnorm: 0.73 [ 7:23:41<14:40:09] +[titan] 2025-07-09 20:58:43,336 - root - INFO - step: 33520 loss: 16.6527 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5808 global_avg_mtp_loss: 14.0720 +[titan] 2025-07-09 20:58:43,337 - root - INFO - lr: 2.3427e-04 gnorm: 1.21 [ 7:23:44<14:40:05] +[titan] 2025-07-09 20:58:47,261 - root - INFO - step: 33525 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.3098 +[titan] 2025-07-09 20:58:47,261 - root - INFO - lr: 2.3425e-04 gnorm: 0.87 [ 7:23:48<14:40:00] +[titan] 2025-07-09 20:58:51,236 - root - INFO - step: 33530 loss: 16.9522 memory: 44.58GiB(31.99%) tps: 82,439 tflops: 284.51 mfu: 28.77% global_avg_ntp_loss: 2.6378 global_avg_mtp_loss: 14.3144 +[titan] 2025-07-09 20:58:51,237 - root - INFO - lr: 2.3423e-04 gnorm: 0.76 [ 7:23:52<14:39:56] +[titan] 2025-07-09 20:58:55,177 - root - INFO - step: 33535 loss: 16.8122 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5971 global_avg_mtp_loss: 14.2151 +[titan] 2025-07-09 20:58:55,177 - root - INFO - lr: 2.3422e-04 gnorm: 0.76 [ 7:23:56<14:39:52] +[titan] 2025-07-09 20:58:59,097 - root - INFO - step: 33540 loss: 16.8947 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.6368 global_avg_mtp_loss: 14.2579 +[titan] 2025-07-09 20:58:59,097 - root - INFO - lr: 2.3420e-04 gnorm: 0.75 [ 7:24:00<14:39:48] +[titan] 2025-07-09 20:59:02,988 - root - INFO - step: 33545 loss: 16.9107 memory: 44.58GiB(31.99%) tps: 84,216 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2829 +[titan] 2025-07-09 20:59:02,988 - root - INFO - lr: 2.3418e-04 gnorm: 0.77 [ 7:24:04<14:39:44] +[titan] 2025-07-09 20:59:06,088 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:59:06,882 - root - INFO - step: 33550 loss: 17.0554 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.6551 global_avg_mtp_loss: 14.4003 +[titan] 2025-07-09 20:59:06,882 - root - INFO - lr: 2.3416e-04 gnorm: 0.78 [ 7:24:08<14:39:40] +[titan] 2025-07-09 20:59:10,777 - root - INFO - step: 33555 loss: 16.7588 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1673 +[titan] 2025-07-09 20:59:10,778 - root - INFO - lr: 2.3414e-04 gnorm: 0.75 [ 7:24:12<14:39:36] +[titan] 2025-07-09 20:59:14,671 - root - INFO - step: 33560 loss: 16.6214 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0526 +[titan] 2025-07-09 20:59:14,671 - root - INFO - lr: 2.3412e-04 gnorm: 0.73 [ 7:24:16<14:39:32] +[titan] 2025-07-09 20:59:18,561 - root - INFO - step: 33565 loss: 16.4643 memory: 44.58GiB(31.99%) tps: 84,238 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9233 +[titan] 2025-07-09 20:59:18,561 - root - INFO - lr: 2.3410e-04 gnorm: 0.83 [ 7:24:20<14:39:28] +[titan] 2025-07-09 20:59:22,511 - root - INFO - step: 33570 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 13.9688 +[titan] 2025-07-09 20:59:22,511 - root - INFO - lr: 2.3409e-04 gnorm: 0.81 [ 7:24:24<14:39:24] +[titan] 2025-07-09 20:59:26,426 - root - INFO - step: 33575 loss: 16.5700 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0128 +[titan] 2025-07-09 20:59:26,427 - root - INFO - lr: 2.3407e-04 gnorm: 0.76 [ 7:24:27<14:39:20] +[titan] 2025-07-09 20:59:30,538 - root - INFO - step: 33580 loss: 16.6770 memory: 44.58GiB(31.99%) tps: 79,706 tflops: 275.08 mfu: 27.81% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-09 20:59:30,538 - root - INFO - lr: 2.3405e-04 gnorm: 0.82 [ 7:24:32<14:39:16] +[titan] 2025-07-09 20:59:34,486 - root - INFO - step: 33585 loss: 16.6995 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.1296 +[titan] 2025-07-09 20:59:34,486 - root - INFO - lr: 2.3403e-04 gnorm: 0.74 [ 7:24:36<14:39:12] +[titan] 2025-07-09 20:59:38,421 - root - INFO - step: 33590 loss: 16.7404 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.1464 +[titan] 2025-07-09 20:59:38,422 - root - INFO - lr: 2.3401e-04 gnorm: 0.76 [ 7:24:39<14:39:08] +[titan] 2025-07-09 20:59:42,356 - root - INFO - step: 33595 loss: 16.5706 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 14.0239 +[titan] 2025-07-09 20:59:42,356 - root - INFO - lr: 2.3399e-04 gnorm: 0.71 [ 7:24:43<14:39:04] +[titan] 2025-07-09 20:59:45,470 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 20:59:46,280 - root - INFO - step: 33600 loss: 16.9393 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6412 global_avg_mtp_loss: 14.2981 +[titan] 2025-07-09 20:59:46,280 - root - INFO - lr: 2.3398e-04 gnorm: 0.78 [ 7:24:47<14:39:00] +[titan] 2025-07-09 20:59:50,230 - root - INFO - step: 33605 loss: 16.9996 memory: 44.58GiB(31.99%) tps: 82,953 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.6421 global_avg_mtp_loss: 14.3575 +[titan] 2025-07-09 20:59:50,231 - root - INFO - lr: 2.3396e-04 gnorm: 0.78 [ 7:24:51<14:38:56] +[titan] 2025-07-09 20:59:54,154 - root - INFO - step: 33610 loss: 16.8398 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.2316 +[titan] 2025-07-09 20:59:54,154 - root - INFO - lr: 2.3394e-04 gnorm: 0.79 [ 7:24:55<14:38:52] +[titan] 2025-07-09 20:59:58,174 - root - INFO - step: 33615 loss: 16.7129 memory: 44.58GiB(31.99%) tps: 81,520 tflops: 281.34 mfu: 28.45% global_avg_ntp_loss: 2.5871 global_avg_mtp_loss: 14.1258 +[titan] 2025-07-09 20:59:58,174 - root - INFO - lr: 2.3392e-04 gnorm: 0.79 [ 7:24:59<14:38:48] +[titan] 2025-07-09 21:00:02,064 - root - INFO - step: 33620 loss: 16.6902 memory: 44.58GiB(31.99%) tps: 84,244 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1053 +[titan] 2025-07-09 21:00:02,064 - root - INFO - lr: 2.3390e-04 gnorm: 0.79 [ 7:25:03<14:38:44] +[titan] 2025-07-09 21:00:05,983 - root - INFO - step: 33625 loss: 16.7319 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1426 +[titan] 2025-07-09 21:00:05,984 - root - INFO - lr: 2.3388e-04 gnorm: 0.75 [ 7:25:07<14:38:40] +[titan] 2025-07-09 21:00:09,895 - root - INFO - step: 33630 loss: 16.9159 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2960 +[titan] 2025-07-09 21:00:09,895 - root - INFO - lr: 2.3387e-04 gnorm: 0.80 [ 7:25:11<14:38:36] +[titan] 2025-07-09 21:00:13,814 - root - INFO - step: 33635 loss: 16.6537 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.0677 +[titan] 2025-07-09 21:00:13,814 - root - INFO - lr: 2.3385e-04 gnorm: 0.72 [ 7:25:15<14:38:31] +[titan] 2025-07-09 21:00:17,714 - root - INFO - step: 33640 loss: 16.6267 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0654 +[titan] 2025-07-09 21:00:17,715 - root - INFO - lr: 2.3383e-04 gnorm: 0.77 [ 7:25:19<14:38:27] +[titan] 2025-07-09 21:00:21,618 - root - INFO - step: 33645 loss: 16.6113 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0526 +[titan] 2025-07-09 21:00:21,618 - root - INFO - lr: 2.3381e-04 gnorm: 0.78 [ 7:25:23<14:38:23] +[titan] 2025-07-09 21:00:24,713 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:00:25,503 - root - INFO - step: 33650 loss: 16.6606 memory: 44.58GiB(31.99%) tps: 84,353 tflops: 291.12 mfu: 29.44% global_avg_ntp_loss: 2.5661 global_avg_mtp_loss: 14.0945 +[titan] 2025-07-09 21:00:25,503 - root - INFO - lr: 2.3379e-04 gnorm: 0.76 [ 7:25:27<14:38:19] +[titan] 2025-07-09 21:00:29,413 - root - INFO - step: 33655 loss: 16.8879 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2656 +[titan] 2025-07-09 21:00:29,413 - root - INFO - lr: 2.3377e-04 gnorm: 0.74 [ 7:25:30<14:38:15] +[titan] 2025-07-09 21:00:33,314 - root - INFO - step: 33660 loss: 16.6964 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5814 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-09 21:00:33,314 - root - INFO - lr: 2.3375e-04 gnorm: 0.79 [ 7:25:34<14:38:11] +[titan] 2025-07-09 21:00:37,209 - root - INFO - step: 33665 loss: 16.3142 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.7948 +[titan] 2025-07-09 21:00:37,209 - root - INFO - lr: 2.3374e-04 gnorm: 0.77 [ 7:25:38<14:38:07] +[titan] 2025-07-09 21:00:41,120 - root - INFO - step: 33670 loss: 17.0329 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6486 global_avg_mtp_loss: 14.3843 +[titan] 2025-07-09 21:00:41,121 - root - INFO - lr: 2.3372e-04 gnorm: 0.88 [ 7:25:42<14:38:03] +[titan] 2025-07-09 21:00:45,082 - root - INFO - step: 33675 loss: 16.9196 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.6254 global_avg_mtp_loss: 14.2942 +[titan] 2025-07-09 21:00:45,082 - root - INFO - lr: 2.3370e-04 gnorm: 0.82 [ 7:25:46<14:37:59] +[titan] 2025-07-09 21:00:48,985 - root - INFO - step: 33680 loss: 16.8060 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.1998 +[titan] 2025-07-09 21:00:48,985 - root - INFO - lr: 2.3368e-04 gnorm: 0.84 [ 7:25:50<14:37:55] +[titan] 2025-07-09 21:00:52,881 - root - INFO - step: 33685 loss: 16.5605 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 14.0031 +[titan] 2025-07-09 21:00:52,882 - root - INFO - lr: 2.3366e-04 gnorm: 0.79 [ 7:25:54<14:37:50] +[titan] 2025-07-09 21:00:56,779 - root - INFO - step: 33690 loss: 16.9797 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.6390 global_avg_mtp_loss: 14.3407 +[titan] 2025-07-09 21:00:56,779 - root - INFO - lr: 2.3364e-04 gnorm: 0.76 [ 7:25:58<14:37:46] +[titan] 2025-07-09 21:01:00,728 - root - INFO - step: 33695 loss: 16.6433 memory: 44.58GiB(31.99%) tps: 82,974 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.0671 +[titan] 2025-07-09 21:01:00,729 - root - INFO - lr: 2.3363e-04 gnorm: 0.73 [ 7:26:02<14:37:42] +[titan] 2025-07-09 21:01:03,846 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:01:04,634 - root - INFO - step: 33700 loss: 16.8369 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6079 global_avg_mtp_loss: 14.2290 +[titan] 2025-07-09 21:01:04,634 - root - INFO - lr: 2.3361e-04 gnorm: 0.78 [ 7:26:06<14:37:38] +[titan] 2025-07-09 21:01:08,537 - root - INFO - step: 33705 loss: 16.6696 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.0863 +[titan] 2025-07-09 21:01:08,538 - root - INFO - lr: 2.3359e-04 gnorm: 0.75 [ 7:26:10<14:37:34] +[titan] 2025-07-09 21:01:12,536 - root - INFO - step: 33710 loss: 16.5180 memory: 44.58GiB(31.99%) tps: 81,951 tflops: 282.83 mfu: 28.60% global_avg_ntp_loss: 2.5568 global_avg_mtp_loss: 13.9612 +[titan] 2025-07-09 21:01:12,536 - root - INFO - lr: 2.3357e-04 gnorm: 0.86 [ 7:26:14<14:37:30] +[titan] 2025-07-09 21:01:16,480 - root - INFO - step: 33715 loss: 16.8820 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.6307 global_avg_mtp_loss: 14.2513 +[titan] 2025-07-09 21:01:16,480 - root - INFO - lr: 2.3355e-04 gnorm: 0.80 [ 7:26:18<14:37:26] +[titan] 2025-07-09 21:01:20,397 - root - INFO - step: 33720 loss: 16.6604 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0892 +[titan] 2025-07-09 21:01:20,398 - root - INFO - lr: 2.3353e-04 gnorm: 0.76 [ 7:26:21<14:37:22] +[titan] 2025-07-09 21:01:24,302 - root - INFO - step: 33725 loss: 16.4796 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 13.9269 +[titan] 2025-07-09 21:01:24,302 - root - INFO - lr: 2.3352e-04 gnorm: 0.75 [ 7:26:25<14:37:18] +[titan] 2025-07-09 21:01:28,212 - root - INFO - step: 33730 loss: 16.5218 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 13.9660 +[titan] 2025-07-09 21:01:28,212 - root - INFO - lr: 2.3350e-04 gnorm: 0.81 [ 7:26:29<14:37:14] +[titan] 2025-07-09 21:01:32,122 - root - INFO - step: 33735 loss: 16.7062 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.1243 +[titan] 2025-07-09 21:01:32,122 - root - INFO - lr: 2.3348e-04 gnorm: 0.76 [ 7:26:33<14:37:10] +[titan] 2025-07-09 21:01:36,032 - root - INFO - step: 33740 loss: 16.6722 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.1010 +[titan] 2025-07-09 21:01:36,033 - root - INFO - lr: 2.3346e-04 gnorm: 0.76 [ 7:26:37<14:37:06] +[titan] 2025-07-09 21:01:39,973 - root - INFO - step: 33745 loss: 16.7734 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5990 global_avg_mtp_loss: 14.1745 +[titan] 2025-07-09 21:01:39,973 - root - INFO - lr: 2.3344e-04 gnorm: 0.79 [ 7:26:41<14:37:02] +[titan] 2025-07-09 21:01:43,083 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:01:43,868 - root - INFO - step: 33750 loss: 16.7131 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 21:01:43,868 - root - INFO - lr: 2.3342e-04 gnorm: 0.74 [ 7:26:45<14:36:58] +[titan] 2025-07-09 21:01:47,768 - root - INFO - step: 33755 loss: 16.8499 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6133 global_avg_mtp_loss: 14.2366 +[titan] 2025-07-09 21:01:47,768 - root - INFO - lr: 2.3340e-04 gnorm: 0.80 [ 7:26:49<14:36:53] +[titan] 2025-07-09 21:01:51,703 - root - INFO - step: 33760 loss: 16.6508 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 14.0818 +[titan] 2025-07-09 21:01:51,703 - root - INFO - lr: 2.3339e-04 gnorm: 0.76 [ 7:26:53<14:36:49] +[titan] 2025-07-09 21:01:55,620 - root - INFO - step: 33765 loss: 16.8052 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.2004 +[titan] 2025-07-09 21:01:55,621 - root - INFO - lr: 2.3337e-04 gnorm: 0.74 [ 7:26:57<14:36:45] +[titan] 2025-07-09 21:01:59,554 - root - INFO - step: 33770 loss: 16.6974 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1117 +[titan] 2025-07-09 21:01:59,555 - root - INFO - lr: 2.3335e-04 gnorm: 0.81 [ 7:27:01<14:36:41] +[titan] 2025-07-09 21:02:03,483 - root - INFO - step: 33775 loss: 16.6012 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5768 global_avg_mtp_loss: 14.0244 +[titan] 2025-07-09 21:02:03,483 - root - INFO - lr: 2.3333e-04 gnorm: 0.78 [ 7:27:05<14:36:37] +[titan] 2025-07-09 21:02:07,420 - root - INFO - step: 33780 loss: 16.8095 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.2028 +[titan] 2025-07-09 21:02:07,420 - root - INFO - lr: 2.3331e-04 gnorm: 0.76 [ 7:27:08<14:36:33] +[titan] 2025-07-09 21:02:11,335 - root - INFO - step: 33785 loss: 16.6703 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.0877 +[titan] 2025-07-09 21:02:11,336 - root - INFO - lr: 2.3329e-04 gnorm: 0.75 [ 7:27:12<14:36:29] +[titan] 2025-07-09 21:02:15,333 - root - INFO - step: 33790 loss: 16.7359 memory: 44.58GiB(31.99%) tps: 81,976 tflops: 282.91 mfu: 28.61% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1512 +[titan] 2025-07-09 21:02:15,333 - root - INFO - lr: 2.3328e-04 gnorm: 0.73 [ 7:27:16<14:36:25] +[titan] 2025-07-09 21:02:17,047 - root - INFO - Dumping profiler traces at step 33792 +[titan] 2025-07-09 21:02:17,079 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:02:19,463 - root - INFO - step: 33795 loss: 16.3095 memory: 44.58GiB(31.99%) tps: 79,341 tflops: 273.82 mfu: 27.69% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.7882 +[titan] 2025-07-09 21:02:19,464 - root - INFO - lr: 2.3326e-04 gnorm: 0.80 [ 7:27:21<14:36:22] +[titan] 2025-07-09 21:02:22,578 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:02:23,366 - root - INFO - step: 33800 loss: 16.5967 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5597 global_avg_mtp_loss: 14.0371 +[titan] 2025-07-09 21:02:23,366 - root - INFO - lr: 2.3324e-04 gnorm: 0.74 [ 7:27:24<14:36:17] +[titan] 2025-07-09 21:02:27,282 - root - INFO - step: 33805 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 13.9919 +[titan] 2025-07-09 21:02:27,282 - root - INFO - lr: 2.3322e-04 gnorm: 0.75 [ 7:27:28<14:36:13] +[titan] 2025-07-09 21:02:31,195 - root - INFO - step: 33810 loss: 16.7916 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.2018 +[titan] 2025-07-09 21:02:31,195 - root - INFO - lr: 2.3320e-04 gnorm: 0.74 [ 7:27:32<14:36:09] +[titan] 2025-07-09 21:02:35,091 - root - INFO - step: 33815 loss: 16.7435 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5880 global_avg_mtp_loss: 14.1554 +[titan] 2025-07-09 21:02:35,091 - root - INFO - lr: 2.3318e-04 gnorm: 0.76 [ 7:27:36<14:36:05] +[titan] 2025-07-09 21:02:39,029 - root - INFO - step: 33820 loss: 16.7742 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5964 global_avg_mtp_loss: 14.1778 +[titan] 2025-07-09 21:02:39,029 - root - INFO - lr: 2.3316e-04 gnorm: 0.75 [ 7:27:40<14:36:01] +[titan] 2025-07-09 21:02:42,985 - root - INFO - step: 33825 loss: 16.8830 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.2529 +[titan] 2025-07-09 21:02:42,985 - root - INFO - lr: 2.3315e-04 gnorm: 0.79 [ 7:27:44<14:35:57] +[titan] 2025-07-09 21:02:46,881 - root - INFO - step: 33830 loss: 16.8646 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6173 global_avg_mtp_loss: 14.2473 +[titan] 2025-07-09 21:02:46,882 - root - INFO - lr: 2.3313e-04 gnorm: 0.79 [ 7:27:48<14:35:53] +[titan] 2025-07-09 21:02:50,812 - root - INFO - step: 33835 loss: 16.8582 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6114 global_avg_mtp_loss: 14.2469 +[titan] 2025-07-09 21:02:50,812 - root - INFO - lr: 2.3311e-04 gnorm: 0.77 [ 7:27:52<14:35:49] +[titan] 2025-07-09 21:02:54,726 - root - INFO - step: 33840 loss: 16.6985 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.1228 +[titan] 2025-07-09 21:02:54,727 - root - INFO - lr: 2.3309e-04 gnorm: 0.76 [ 7:27:56<14:35:45] +[titan] 2025-07-09 21:02:58,644 - root - INFO - step: 33845 loss: 16.8088 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2055 +[titan] 2025-07-09 21:02:58,644 - root - INFO - lr: 2.3307e-04 gnorm: 0.80 [ 7:28:00<14:35:41] +[titan] 2025-07-09 21:03:01,764 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:03:02,556 - root - INFO - step: 33850 loss: 16.7815 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6107 global_avg_mtp_loss: 14.1709 +[titan] 2025-07-09 21:03:02,556 - root - INFO - lr: 2.3305e-04 gnorm: 0.79 [ 7:28:04<14:35:37] +[titan] 2025-07-09 21:03:06,488 - root - INFO - step: 33855 loss: 16.5002 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.9649 +[titan] 2025-07-09 21:03:06,488 - root - INFO - lr: 2.3303e-04 gnorm: 0.74 [ 7:28:08<14:35:33] +[titan] 2025-07-09 21:03:10,382 - root - INFO - step: 33860 loss: 16.7044 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1222 +[titan] 2025-07-09 21:03:10,382 - root - INFO - lr: 2.3302e-04 gnorm: 0.77 [ 7:28:11<14:35:29] +[titan] 2025-07-09 21:03:14,288 - root - INFO - step: 33865 loss: 16.6287 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.0486 +[titan] 2025-07-09 21:03:14,288 - root - INFO - lr: 2.3300e-04 gnorm: 0.80 [ 7:28:15<14:35:24] +[titan] 2025-07-09 21:03:18,262 - root - INFO - step: 33870 loss: 16.9387 memory: 44.58GiB(31.99%) tps: 82,456 tflops: 284.57 mfu: 28.77% global_avg_ntp_loss: 2.6358 global_avg_mtp_loss: 14.3029 +[titan] 2025-07-09 21:03:18,263 - root - INFO - lr: 2.3298e-04 gnorm: 0.87 [ 7:28:19<14:35:20] +[titan] 2025-07-09 21:03:22,190 - root - INFO - step: 33875 loss: 16.7109 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5803 global_avg_mtp_loss: 14.1306 +[titan] 2025-07-09 21:03:22,191 - root - INFO - lr: 2.3296e-04 gnorm: 0.78 [ 7:28:23<14:35:16] +[titan] 2025-07-09 21:03:26,110 - root - INFO - step: 33880 loss: 16.4213 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.8938 +[titan] 2025-07-09 21:03:26,110 - root - INFO - lr: 2.3294e-04 gnorm: 0.78 [ 7:28:27<14:35:12] +[titan] 2025-07-09 21:03:30,137 - root - INFO - step: 33885 loss: 16.6049 memory: 44.58GiB(31.99%) tps: 81,392 tflops: 280.90 mfu: 28.40% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0404 +[titan] 2025-07-09 21:03:30,137 - root - INFO - lr: 2.3292e-04 gnorm: 0.77 [ 7:28:31<14:35:08] +[titan] 2025-07-09 21:03:34,059 - root - INFO - step: 33890 loss: 17.1044 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.6520 global_avg_mtp_loss: 14.4524 +[titan] 2025-07-09 21:03:34,059 - root - INFO - lr: 2.3291e-04 gnorm: 0.81 [ 7:28:35<14:35:04] +[titan] 2025-07-09 21:03:37,994 - root - INFO - step: 33895 loss: 16.8249 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.2231 +[titan] 2025-07-09 21:03:37,994 - root - INFO - lr: 2.3289e-04 gnorm: 0.76 [ 7:28:39<14:35:00] +[titan] 2025-07-09 21:03:41,149 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:03:41,944 - root - INFO - step: 33900 loss: 16.6157 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0492 +[titan] 2025-07-09 21:03:41,945 - root - INFO - lr: 2.3287e-04 gnorm: 0.76 [ 7:28:43<14:34:56] +[titan] 2025-07-09 21:03:45,902 - root - INFO - step: 33905 loss: 16.9341 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.6373 global_avg_mtp_loss: 14.2968 +[titan] 2025-07-09 21:03:45,902 - root - INFO - lr: 2.3285e-04 gnorm: 0.99 [ 7:28:47<14:34:52] +[titan] 2025-07-09 21:03:49,841 - root - INFO - step: 33910 loss: 16.6618 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5635 global_avg_mtp_loss: 14.0984 +[titan] 2025-07-09 21:03:49,841 - root - INFO - lr: 2.3283e-04 gnorm: 0.77 [ 7:28:51<14:34:48] +[titan] 2025-07-09 21:03:53,735 - root - INFO - step: 33915 loss: 16.9030 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.6230 global_avg_mtp_loss: 14.2799 +[titan] 2025-07-09 21:03:53,735 - root - INFO - lr: 2.3281e-04 gnorm: 0.81 [ 7:28:55<14:34:44] +[titan] 2025-07-09 21:03:57,654 - root - INFO - step: 33920 loss: 16.7588 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.1542 +[titan] 2025-07-09 21:03:57,654 - root - INFO - lr: 2.3279e-04 gnorm: 0.74 [ 7:28:59<14:34:40] +[titan] 2025-07-09 21:04:01,579 - root - INFO - step: 33925 loss: 16.5927 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 14.0252 +[titan] 2025-07-09 21:04:01,579 - root - INFO - lr: 2.3278e-04 gnorm: 0.76 [ 7:29:03<14:34:36] +[titan] 2025-07-09 21:04:05,472 - root - INFO - step: 33930 loss: 16.8271 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.6105 global_avg_mtp_loss: 14.2165 +[titan] 2025-07-09 21:04:05,472 - root - INFO - lr: 2.3276e-04 gnorm: 0.77 [ 7:29:07<14:34:32] +[titan] 2025-07-09 21:04:09,370 - root - INFO - step: 33935 loss: 16.8624 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6209 global_avg_mtp_loss: 14.2415 +[titan] 2025-07-09 21:04:09,371 - root - INFO - lr: 2.3274e-04 gnorm: 0.78 [ 7:29:10<14:34:28] +[titan] 2025-07-09 21:04:13,336 - root - INFO - step: 33940 loss: 16.7318 memory: 44.58GiB(31.99%) tps: 82,633 tflops: 285.18 mfu: 28.84% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1410 +[titan] 2025-07-09 21:04:13,337 - root - INFO - lr: 2.3272e-04 gnorm: 0.77 [ 7:29:14<14:34:24] +[titan] 2025-07-09 21:04:17,245 - root - INFO - step: 33945 loss: 16.8207 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6052 global_avg_mtp_loss: 14.2155 +[titan] 2025-07-09 21:04:17,246 - root - INFO - lr: 2.3270e-04 gnorm: 0.76 [ 7:29:18<14:34:20] +[titan] 2025-07-09 21:04:20,389 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:04:21,172 - root - INFO - step: 33950 loss: 16.6967 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1121 +[titan] 2025-07-09 21:04:21,172 - root - INFO - lr: 2.3268e-04 gnorm: 0.77 [ 7:29:22<14:34:16] +[titan] 2025-07-09 21:04:25,061 - root - INFO - step: 33955 loss: 16.4400 memory: 44.58GiB(31.99%) tps: 84,267 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.9110 +[titan] 2025-07-09 21:04:25,061 - root - INFO - lr: 2.3266e-04 gnorm: 0.74 [ 7:29:26<14:34:12] +[titan] 2025-07-09 21:04:28,966 - root - INFO - step: 33960 loss: 16.8462 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6191 global_avg_mtp_loss: 14.2271 +[titan] 2025-07-09 21:04:28,966 - root - INFO - lr: 2.3265e-04 gnorm: 0.75 [ 7:29:30<14:34:07] +[titan] 2025-07-09 21:04:32,902 - root - INFO - step: 33965 loss: 16.8697 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.6138 global_avg_mtp_loss: 14.2560 +[titan] 2025-07-09 21:04:32,902 - root - INFO - lr: 2.3263e-04 gnorm: 0.75 [ 7:29:34<14:34:03] +[titan] 2025-07-09 21:04:36,806 - root - INFO - step: 33970 loss: 16.9536 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.6316 global_avg_mtp_loss: 14.3220 +[titan] 2025-07-09 21:04:36,806 - root - INFO - lr: 2.3261e-04 gnorm: 0.79 [ 7:29:38<14:33:59] +[titan] 2025-07-09 21:04:40,713 - root - INFO - step: 33975 loss: 16.7541 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1633 +[titan] 2025-07-09 21:04:40,713 - root - INFO - lr: 2.3259e-04 gnorm: 0.85 [ 7:29:42<14:33:55] +[titan] 2025-07-09 21:04:44,628 - root - INFO - step: 33980 loss: 16.8580 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6091 global_avg_mtp_loss: 14.2488 +[titan] 2025-07-09 21:04:44,628 - root - INFO - lr: 2.3257e-04 gnorm: 0.85 [ 7:29:46<14:33:51] +[titan] 2025-07-09 21:04:48,556 - root - INFO - step: 33985 loss: 16.7263 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5975 global_avg_mtp_loss: 14.1289 +[titan] 2025-07-09 21:04:48,556 - root - INFO - lr: 2.3255e-04 gnorm: 0.82 [ 7:29:50<14:33:47] +[titan] 2025-07-09 21:04:52,518 - root - INFO - step: 33990 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 82,715 tflops: 285.46 mfu: 28.86% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1795 +[titan] 2025-07-09 21:04:52,518 - root - INFO - lr: 2.3253e-04 gnorm: 0.76 [ 7:29:54<14:33:43] +[titan] 2025-07-09 21:04:56,446 - root - INFO - step: 33995 loss: 16.9877 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6330 global_avg_mtp_loss: 14.3548 +[titan] 2025-07-09 21:04:56,447 - root - INFO - lr: 2.3252e-04 gnorm: 0.75 [ 7:29:57<14:33:39] +[titan] 2025-07-09 21:04:59,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:05:00,379 - root - INFO - step: 34000 loss: 17.0226 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6543 global_avg_mtp_loss: 14.3682 +[titan] 2025-07-09 21:05:00,379 - root - INFO - lr: 2.3250e-04 gnorm: 0.72 [ 7:30:01<14:33:35] +[titan] 2025-07-09 21:05:04,299 - root - INFO - step: 34005 loss: 16.8542 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6245 global_avg_mtp_loss: 14.2297 +[titan] 2025-07-09 21:05:04,299 - root - INFO - lr: 2.3248e-04 gnorm: 0.76 [ 7:30:05<14:33:31] +[titan] 2025-07-09 21:05:08,224 - root - INFO - step: 34010 loss: 16.7027 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.1299 +[titan] 2025-07-09 21:05:08,224 - root - INFO - lr: 2.3246e-04 gnorm: 0.78 [ 7:30:09<14:33:27] +[titan] 2025-07-09 21:05:12,165 - root - INFO - step: 34015 loss: 16.8641 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.6112 global_avg_mtp_loss: 14.2530 +[titan] 2025-07-09 21:05:12,166 - root - INFO - lr: 2.3244e-04 gnorm: 0.74 [ 7:30:13<14:33:23] +[titan] 2025-07-09 21:05:16,089 - root - INFO - step: 34020 loss: 16.5178 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9776 +[titan] 2025-07-09 21:05:16,090 - root - INFO - lr: 2.3242e-04 gnorm: 0.76 [ 7:30:17<14:33:19] +[titan] 2025-07-09 21:05:20,009 - root - INFO - step: 34025 loss: 16.7119 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1235 +[titan] 2025-07-09 21:05:20,009 - root - INFO - lr: 2.3240e-04 gnorm: 0.72 [ 7:30:21<14:33:15] +[titan] 2025-07-09 21:05:23,936 - root - INFO - step: 34030 loss: 16.9283 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.3076 +[titan] 2025-07-09 21:05:23,936 - root - INFO - lr: 2.3239e-04 gnorm: 0.72 [ 7:30:25<14:33:11] +[titan] 2025-07-09 21:05:27,853 - root - INFO - step: 34035 loss: 16.5076 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 13.9556 +[titan] 2025-07-09 21:05:27,853 - root - INFO - lr: 2.3237e-04 gnorm: 0.79 [ 7:30:29<14:33:07] +[titan] 2025-07-09 21:05:31,766 - root - INFO - step: 34040 loss: 16.7114 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.1380 +[titan] 2025-07-09 21:05:31,767 - root - INFO - lr: 2.3235e-04 gnorm: 0.77 [ 7:30:33<14:33:02] +[titan] 2025-07-09 21:05:35,681 - root - INFO - step: 34045 loss: 16.6964 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5810 global_avg_mtp_loss: 14.1154 +[titan] 2025-07-09 21:05:35,682 - root - INFO - lr: 2.3233e-04 gnorm: 0.75 [ 7:30:37<14:32:58] +[titan] 2025-07-09 21:05:38,802 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:05:39,593 - root - INFO - step: 34050 loss: 16.7120 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5850 global_avg_mtp_loss: 14.1270 +[titan] 2025-07-09 21:05:39,594 - root - INFO - lr: 2.3231e-04 gnorm: 0.79 [ 7:30:41<14:32:54] +[titan] 2025-07-09 21:05:43,498 - root - INFO - step: 34055 loss: 16.6083 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0532 +[titan] 2025-07-09 21:05:43,498 - root - INFO - lr: 2.3229e-04 gnorm: 0.82 [ 7:30:45<14:32:50] +[titan] 2025-07-09 21:05:47,446 - root - INFO - step: 34060 loss: 17.1414 memory: 44.58GiB(31.99%) tps: 83,001 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.6763 global_avg_mtp_loss: 14.4651 +[titan] 2025-07-09 21:05:47,446 - root - INFO - lr: 2.3227e-04 gnorm: 0.80 [ 7:30:48<14:32:46] +[titan] 2025-07-09 21:05:51,430 - root - INFO - step: 34065 loss: 16.5678 memory: 44.58GiB(31.99%) tps: 82,255 tflops: 283.87 mfu: 28.70% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.0093 +[titan] 2025-07-09 21:05:51,430 - root - INFO - lr: 2.3226e-04 gnorm: 0.76 [ 7:30:52<14:32:42] +[titan] 2025-07-09 21:05:55,342 - root - INFO - step: 34070 loss: 16.2937 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-09 21:05:55,343 - root - INFO - lr: 2.3224e-04 gnorm: 0.81 [ 7:30:56<14:32:38] +[titan] 2025-07-09 21:05:59,295 - root - INFO - step: 34075 loss: 16.5066 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 13.9597 +[titan] 2025-07-09 21:05:59,296 - root - INFO - lr: 2.3222e-04 gnorm: 0.76 [ 7:31:00<14:32:34] +[titan] 2025-07-09 21:06:03,201 - root - INFO - step: 34080 loss: 16.9631 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6359 global_avg_mtp_loss: 14.3272 +[titan] 2025-07-09 21:06:03,202 - root - INFO - lr: 2.3220e-04 gnorm: 0.79 [ 7:31:04<14:32:30] +[titan] 2025-07-09 21:06:07,138 - root - INFO - step: 34085 loss: 16.7898 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.6047 global_avg_mtp_loss: 14.1851 +[titan] 2025-07-09 21:06:07,139 - root - INFO - lr: 2.3218e-04 gnorm: 0.74 [ 7:31:08<14:32:26] +[titan] 2025-07-09 21:06:11,089 - root - INFO - step: 34090 loss: 16.6821 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5813 global_avg_mtp_loss: 14.1008 +[titan] 2025-07-09 21:06:11,090 - root - INFO - lr: 2.3216e-04 gnorm: 0.76 [ 7:31:12<14:32:22] +[titan] 2025-07-09 21:06:15,027 - root - INFO - step: 34095 loss: 16.7188 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5928 global_avg_mtp_loss: 14.1260 +[titan] 2025-07-09 21:06:15,027 - root - INFO - lr: 2.3214e-04 gnorm: 0.77 [ 7:31:16<14:32:18] +[titan] 2025-07-09 21:06:18,185 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:06:18,973 - root - INFO - step: 34100 loss: 16.6600 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0892 +[titan] 2025-07-09 21:06:18,973 - root - INFO - lr: 2.3213e-04 gnorm: 0.84 [ 7:31:20<14:32:14] +[titan] 2025-07-09 21:06:22,869 - root - INFO - step: 34105 loss: 16.8727 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6187 global_avg_mtp_loss: 14.2540 +[titan] 2025-07-09 21:06:22,870 - root - INFO - lr: 2.3211e-04 gnorm: 0.76 [ 7:31:24<14:32:10] +[titan] 2025-07-09 21:06:26,788 - root - INFO - step: 34110 loss: 16.8551 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.6096 global_avg_mtp_loss: 14.2455 +[titan] 2025-07-09 21:06:26,788 - root - INFO - lr: 2.3209e-04 gnorm: 0.75 [ 7:31:28<14:32:06] +[titan] 2025-07-09 21:06:30,681 - root - INFO - step: 34115 loss: 16.7540 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1651 +[titan] 2025-07-09 21:06:30,681 - root - INFO - lr: 2.3207e-04 gnorm: 0.77 [ 7:31:32<14:32:02] +[titan] 2025-07-09 21:06:34,579 - root - INFO - step: 34120 loss: 16.5718 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0016 +[titan] 2025-07-09 21:06:34,579 - root - INFO - lr: 2.3205e-04 gnorm: 0.77 [ 7:31:36<14:31:58] +[titan] 2025-07-09 21:06:38,476 - root - INFO - step: 34125 loss: 16.7191 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1268 +[titan] 2025-07-09 21:06:38,476 - root - INFO - lr: 2.3203e-04 gnorm: 0.82 [ 7:31:39<14:31:53] +[titan] 2025-07-09 21:06:42,398 - root - INFO - step: 34130 loss: 16.6880 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.1077 +[titan] 2025-07-09 21:06:42,398 - root - INFO - lr: 2.3201e-04 gnorm: 0.77 [ 7:31:43<14:31:49] +[titan] 2025-07-09 21:06:46,294 - root - INFO - step: 34135 loss: 16.6284 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.0456 +[titan] 2025-07-09 21:06:46,294 - root - INFO - lr: 2.3200e-04 gnorm: 0.77 [ 7:31:47<14:31:45] +[titan] 2025-07-09 21:06:50,232 - root - INFO - step: 34140 loss: 16.8832 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 21:06:50,232 - root - INFO - lr: 2.3198e-04 gnorm: 0.75 [ 7:31:51<14:31:41] +[titan] 2025-07-09 21:06:54,140 - root - INFO - step: 34145 loss: 16.8111 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.6198 global_avg_mtp_loss: 14.1913 +[titan] 2025-07-09 21:06:54,140 - root - INFO - lr: 2.3196e-04 gnorm: 0.75 [ 7:31:55<14:31:37] +[titan] 2025-07-09 21:06:57,244 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:06:58,034 - root - INFO - step: 34150 loss: 16.7936 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5986 global_avg_mtp_loss: 14.1950 +[titan] 2025-07-09 21:06:58,034 - root - INFO - lr: 2.3194e-04 gnorm: 0.77 [ 7:31:59<14:31:33] +[titan] 2025-07-09 21:07:01,946 - root - INFO - step: 34155 loss: 16.9058 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6198 global_avg_mtp_loss: 14.2861 +[titan] 2025-07-09 21:07:01,946 - root - INFO - lr: 2.3192e-04 gnorm: 0.77 [ 7:32:03<14:31:29] +[titan] 2025-07-09 21:07:05,882 - root - INFO - step: 34160 loss: 16.6576 memory: 44.58GiB(31.99%) tps: 83,247 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.0778 +[titan] 2025-07-09 21:07:05,882 - root - INFO - lr: 2.3190e-04 gnorm: 0.79 [ 7:32:07<14:31:25] +[titan] 2025-07-09 21:07:09,779 - root - INFO - step: 34165 loss: 16.6029 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 21:07:09,779 - root - INFO - lr: 2.3188e-04 gnorm: 0.73 [ 7:32:11<14:31:21] +[titan] 2025-07-09 21:07:13,676 - root - INFO - step: 34170 loss: 16.4794 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5564 global_avg_mtp_loss: 13.9230 +[titan] 2025-07-09 21:07:13,677 - root - INFO - lr: 2.3187e-04 gnorm: 0.80 [ 7:32:15<14:31:17] +[titan] 2025-07-09 21:07:17,588 - root - INFO - step: 34175 loss: 16.5780 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.0069 +[titan] 2025-07-09 21:07:17,588 - root - INFO - lr: 2.3185e-04 gnorm: 0.76 [ 7:32:19<14:31:13] +[titan] 2025-07-09 21:07:21,519 - root - INFO - step: 34180 loss: 16.6485 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.0721 +[titan] 2025-07-09 21:07:21,519 - root - INFO - lr: 2.3183e-04 gnorm: 0.72 [ 7:32:23<14:31:09] +[titan] 2025-07-09 21:07:25,451 - root - INFO - step: 34185 loss: 16.9450 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.6262 global_avg_mtp_loss: 14.3188 +[titan] 2025-07-09 21:07:25,451 - root - INFO - lr: 2.3181e-04 gnorm: 0.70 [ 7:32:26<14:31:04] +[titan] 2025-07-09 21:07:29,356 - root - INFO - step: 34190 loss: 16.5461 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5679 global_avg_mtp_loss: 13.9782 +[titan] 2025-07-09 21:07:29,357 - root - INFO - lr: 2.3179e-04 gnorm: 0.81 [ 7:32:30<14:31:00] +[titan] 2025-07-09 21:07:33,276 - root - INFO - step: 34195 loss: 16.8162 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6119 global_avg_mtp_loss: 14.2042 +[titan] 2025-07-09 21:07:33,277 - root - INFO - lr: 2.3177e-04 gnorm: 0.76 [ 7:32:34<14:30:56] +[titan] 2025-07-09 21:07:36,392 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:07:37,182 - root - INFO - step: 34200 loss: 16.5071 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9571 +[titan] 2025-07-09 21:07:37,182 - root - INFO - lr: 2.3175e-04 gnorm: 0.76 [ 7:32:38<14:30:52] +[titan] 2025-07-09 21:07:41,155 - root - INFO - step: 34205 loss: 16.8777 memory: 44.58GiB(31.99%) tps: 82,486 tflops: 284.67 mfu: 28.78% global_avg_ntp_loss: 2.6174 global_avg_mtp_loss: 14.2603 +[titan] 2025-07-09 21:07:41,155 - root - INFO - lr: 2.3174e-04 gnorm: 0.74 [ 7:32:42<14:30:48] +[titan] 2025-07-09 21:07:45,081 - root - INFO - step: 34210 loss: 16.7034 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.1280 +[titan] 2025-07-09 21:07:45,082 - root - INFO - lr: 2.3172e-04 gnorm: 0.76 [ 7:32:46<14:30:44] +[titan] 2025-07-09 21:07:49,019 - root - INFO - step: 34215 loss: 16.6311 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0609 +[titan] 2025-07-09 21:07:49,020 - root - INFO - lr: 2.3170e-04 gnorm: 0.77 [ 7:32:50<14:30:40] +[titan] 2025-07-09 21:07:52,912 - root - INFO - step: 34220 loss: 16.6401 memory: 44.58GiB(31.99%) tps: 84,177 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0728 +[titan] 2025-07-09 21:07:52,913 - root - INFO - lr: 2.3168e-04 gnorm: 0.83 [ 7:32:54<14:30:36] +[titan] 2025-07-09 21:07:56,830 - root - INFO - step: 34225 loss: 16.8072 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.2043 +[titan] 2025-07-09 21:07:56,830 - root - INFO - lr: 2.3166e-04 gnorm: 0.77 [ 7:32:58<14:30:32] +[titan] 2025-07-09 21:08:00,756 - root - INFO - step: 34230 loss: 16.6993 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.1230 +[titan] 2025-07-09 21:08:00,756 - root - INFO - lr: 2.3164e-04 gnorm: 0.78 [ 7:33:02<14:30:28] +[titan] 2025-07-09 21:08:04,678 - root - INFO - step: 34235 loss: 16.1936 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-09 21:08:04,679 - root - INFO - lr: 2.3162e-04 gnorm: 0.84 [ 7:33:06<14:30:24] +[titan] 2025-07-09 21:08:08,586 - root - INFO - step: 34240 loss: 16.7948 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.1860 +[titan] 2025-07-09 21:08:08,586 - root - INFO - lr: 2.3161e-04 gnorm: 0.75 [ 7:33:10<14:30:20] +[titan] 2025-07-09 21:08:12,490 - root - INFO - step: 34245 loss: 16.8816 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6272 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 21:08:12,491 - root - INFO - lr: 2.3159e-04 gnorm: 0.74 [ 7:33:14<14:30:16] +[titan] 2025-07-09 21:08:15,600 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:08:16,384 - root - INFO - step: 34250 loss: 16.8570 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.2398 +[titan] 2025-07-09 21:08:16,385 - root - INFO - lr: 2.3157e-04 gnorm: 0.83 [ 7:33:17<14:30:12] +[titan] 2025-07-09 21:08:20,320 - root - INFO - step: 34255 loss: 16.8023 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.2134 +[titan] 2025-07-09 21:08:20,320 - root - INFO - lr: 2.3155e-04 gnorm: 0.75 [ 7:33:21<14:30:07] +[titan] 2025-07-09 21:08:24,251 - root - INFO - step: 34260 loss: 16.7227 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5876 global_avg_mtp_loss: 14.1350 +[titan] 2025-07-09 21:08:24,251 - root - INFO - lr: 2.3153e-04 gnorm: 0.75 [ 7:33:25<14:30:03] +[titan] 2025-07-09 21:08:28,162 - root - INFO - step: 34265 loss: 16.6097 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5803 global_avg_mtp_loss: 14.0294 +[titan] 2025-07-09 21:08:28,163 - root - INFO - lr: 2.3151e-04 gnorm: 0.78 [ 7:33:29<14:29:59] +[titan] 2025-07-09 21:08:32,083 - root - INFO - step: 34270 loss: 16.7038 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.1215 +[titan] 2025-07-09 21:08:32,084 - root - INFO - lr: 2.3149e-04 gnorm: 0.71 [ 7:33:33<14:29:55] +[titan] 2025-07-09 21:08:35,982 - root - INFO - step: 34275 loss: 16.9355 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6299 global_avg_mtp_loss: 14.3056 +[titan] 2025-07-09 21:08:35,982 - root - INFO - lr: 2.3147e-04 gnorm: 0.73 [ 7:33:37<14:29:51] +[titan] 2025-07-09 21:08:39,871 - root - INFO - step: 34280 loss: 16.9126 memory: 44.58GiB(31.99%) tps: 84,252 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.6327 global_avg_mtp_loss: 14.2799 +[titan] 2025-07-09 21:08:39,872 - root - INFO - lr: 2.3146e-04 gnorm: 0.82 [ 7:33:41<14:29:47] +[titan] 2025-07-09 21:08:43,790 - root - INFO - step: 34285 loss: 16.7647 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1801 +[titan] 2025-07-09 21:08:43,791 - root - INFO - lr: 2.3144e-04 gnorm: 0.78 [ 7:33:45<14:29:43] +[titan] 2025-07-09 21:08:47,703 - root - INFO - step: 34290 loss: 16.8153 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6143 global_avg_mtp_loss: 14.2010 +[titan] 2025-07-09 21:08:47,703 - root - INFO - lr: 2.3142e-04 gnorm: 0.80 [ 7:33:49<14:29:39] +[titan] 2025-07-09 21:08:51,648 - root - INFO - step: 34295 loss: 16.6951 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1100 +[titan] 2025-07-09 21:08:51,648 - root - INFO - lr: 2.3140e-04 gnorm: 0.79 [ 7:33:53<14:29:35] +[titan] 2025-07-09 21:08:54,809 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:08:55,602 - root - INFO - step: 34300 loss: 16.8429 memory: 44.58GiB(31.99%) tps: 82,868 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2346 +[titan] 2025-07-09 21:08:55,603 - root - INFO - lr: 2.3138e-04 gnorm: 0.84 [ 7:33:57<14:29:31] +[titan] 2025-07-09 21:08:58,994 - root - INFO - Dumping profiler traces at step 34304 +[titan] 2025-07-09 21:08:59,027 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:08:59,822 - root - INFO - step: 34305 loss: 16.6527 memory: 44.58GiB(31.99%) tps: 77,663 tflops: 268.03 mfu: 27.10% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.0757 +[titan] 2025-07-09 21:08:59,822 - root - INFO - lr: 2.3136e-04 gnorm: 0.75 [ 7:34:01<14:29:27] +[titan] 2025-07-09 21:09:03,708 - root - INFO - step: 34310 loss: 16.6513 memory: 44.58GiB(31.99%) tps: 84,330 tflops: 291.04 mfu: 29.43% global_avg_ntp_loss: 2.5746 global_avg_mtp_loss: 14.0767 +[titan] 2025-07-09 21:09:03,708 - root - INFO - lr: 2.3134e-04 gnorm: 0.81 [ 7:34:05<14:29:23] +[titan] 2025-07-09 21:09:07,630 - root - INFO - step: 34315 loss: 16.7093 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5814 global_avg_mtp_loss: 14.1279 +[titan] 2025-07-09 21:09:07,630 - root - INFO - lr: 2.3133e-04 gnorm: 0.88 [ 7:34:09<14:29:19] +[titan] 2025-07-09 21:09:11,541 - root - INFO - step: 34320 loss: 16.6856 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.1036 +[titan] 2025-07-09 21:09:11,541 - root - INFO - lr: 2.3131e-04 gnorm: 0.80 [ 7:34:13<14:29:15] +[titan] 2025-07-09 21:09:15,484 - root - INFO - step: 34325 loss: 16.8875 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.6280 global_avg_mtp_loss: 14.2594 +[titan] 2025-07-09 21:09:15,485 - root - INFO - lr: 2.3129e-04 gnorm: 0.74 [ 7:34:16<14:29:11] +[titan] 2025-07-09 21:09:19,437 - root - INFO - step: 34330 loss: 16.7468 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5856 global_avg_mtp_loss: 14.1613 +[titan] 2025-07-09 21:09:19,438 - root - INFO - lr: 2.3127e-04 gnorm: 0.73 [ 7:34:20<14:29:07] +[titan] 2025-07-09 21:09:23,345 - root - INFO - step: 34335 loss: 16.9123 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6422 global_avg_mtp_loss: 14.2701 +[titan] 2025-07-09 21:09:23,345 - root - INFO - lr: 2.3125e-04 gnorm: 0.76 [ 7:34:24<14:29:03] +[titan] 2025-07-09 21:09:27,262 - root - INFO - step: 34340 loss: 16.7313 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1422 +[titan] 2025-07-09 21:09:27,263 - root - INFO - lr: 2.3123e-04 gnorm: 0.79 [ 7:34:28<14:28:59] +[titan] 2025-07-09 21:09:31,167 - root - INFO - step: 34345 loss: 16.8531 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2308 +[titan] 2025-07-09 21:09:31,168 - root - INFO - lr: 2.3121e-04 gnorm: 0.81 [ 7:34:32<14:28:55] +[titan] 2025-07-09 21:09:34,308 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:09:35,112 - root - INFO - step: 34350 loss: 16.7356 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.1465 +[titan] 2025-07-09 21:09:35,112 - root - INFO - lr: 2.3119e-04 gnorm: 0.81 [ 7:34:36<14:28:51] +[titan] 2025-07-09 21:09:39,014 - root - INFO - step: 34355 loss: 16.7876 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.1793 +[titan] 2025-07-09 21:09:39,014 - root - INFO - lr: 2.3118e-04 gnorm: 0.78 [ 7:34:40<14:28:47] +[titan] 2025-07-09 21:09:42,908 - root - INFO - step: 34360 loss: 16.8898 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.6104 global_avg_mtp_loss: 14.2794 +[titan] 2025-07-09 21:09:42,908 - root - INFO - lr: 2.3116e-04 gnorm: 0.74 [ 7:34:44<14:28:43] +[titan] 2025-07-09 21:09:46,825 - root - INFO - step: 34365 loss: 16.5188 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 13.9680 +[titan] 2025-07-09 21:09:46,825 - root - INFO - lr: 2.3114e-04 gnorm: 0.75 [ 7:34:48<14:28:38] +[titan] 2025-07-09 21:09:50,718 - root - INFO - step: 34370 loss: 16.5840 memory: 44.58GiB(31.99%) tps: 84,177 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.0142 +[titan] 2025-07-09 21:09:50,718 - root - INFO - lr: 2.3112e-04 gnorm: 0.73 [ 7:34:52<14:28:34] +[titan] 2025-07-09 21:09:54,618 - root - INFO - step: 34375 loss: 16.8839 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.2764 +[titan] 2025-07-09 21:09:54,618 - root - INFO - lr: 2.3110e-04 gnorm: 0.78 [ 7:34:56<14:28:30] +[titan] 2025-07-09 21:09:58,526 - root - INFO - step: 34380 loss: 16.8517 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.6248 global_avg_mtp_loss: 14.2269 +[titan] 2025-07-09 21:09:58,527 - root - INFO - lr: 2.3108e-04 gnorm: 0.79 [ 7:35:00<14:28:26] +[titan] 2025-07-09 21:10:02,444 - root - INFO - step: 34385 loss: 16.7690 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1725 +[titan] 2025-07-09 21:10:02,444 - root - INFO - lr: 2.3106e-04 gnorm: 0.81 [ 7:35:03<14:28:22] +[titan] 2025-07-09 21:10:06,369 - root - INFO - step: 34390 loss: 16.9835 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.6397 global_avg_mtp_loss: 14.3438 +[titan] 2025-07-09 21:10:06,369 - root - INFO - lr: 2.3105e-04 gnorm: 0.77 [ 7:35:07<14:28:18] +[titan] 2025-07-09 21:10:10,284 - root - INFO - step: 34395 loss: 16.9214 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.6295 global_avg_mtp_loss: 14.2918 +[titan] 2025-07-09 21:10:10,284 - root - INFO - lr: 2.3103e-04 gnorm: 0.90 [ 7:35:11<14:28:14] +[titan] 2025-07-09 21:10:13,439 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:10:14,233 - root - INFO - step: 34400 loss: 16.6169 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.0456 +[titan] 2025-07-09 21:10:14,234 - root - INFO - lr: 2.3101e-04 gnorm: 0.77 [ 7:35:15<14:28:10] +[titan] 2025-07-09 21:10:18,177 - root - INFO - step: 34405 loss: 16.5557 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 13.9845 +[titan] 2025-07-09 21:10:18,177 - root - INFO - lr: 2.3099e-04 gnorm: 0.84 [ 7:35:19<14:28:06] +[titan] 2025-07-09 21:10:22,123 - root - INFO - step: 34410 loss: 16.6982 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5792 global_avg_mtp_loss: 14.1190 +[titan] 2025-07-09 21:10:22,123 - root - INFO - lr: 2.3097e-04 gnorm: 0.80 [ 7:35:23<14:28:02] +[titan] 2025-07-09 21:10:26,024 - root - INFO - step: 34415 loss: 16.8377 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.2133 +[titan] 2025-07-09 21:10:26,024 - root - INFO - lr: 2.3095e-04 gnorm: 0.83 [ 7:35:27<14:27:58] +[titan] 2025-07-09 21:10:29,981 - root - INFO - step: 34420 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 82,811 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1735 +[titan] 2025-07-09 21:10:29,981 - root - INFO - lr: 2.3093e-04 gnorm: 0.77 [ 7:35:31<14:27:54] +[titan] 2025-07-09 21:10:33,935 - root - INFO - step: 34425 loss: 16.6354 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.0617 +[titan] 2025-07-09 21:10:33,935 - root - INFO - lr: 2.3091e-04 gnorm: 0.77 [ 7:35:35<14:27:50] +[titan] 2025-07-09 21:10:37,847 - root - INFO - step: 34430 loss: 16.6210 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0618 +[titan] 2025-07-09 21:10:37,848 - root - INFO - lr: 2.3090e-04 gnorm: 0.80 [ 7:35:39<14:27:46] +[titan] 2025-07-09 21:10:41,779 - root - INFO - step: 34435 loss: 16.7662 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5928 global_avg_mtp_loss: 14.1734 +[titan] 2025-07-09 21:10:41,779 - root - INFO - lr: 2.3088e-04 gnorm: 0.78 [ 7:35:43<14:27:42] +[titan] 2025-07-09 21:10:45,698 - root - INFO - step: 34440 loss: 16.9512 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.6261 global_avg_mtp_loss: 14.3252 +[titan] 2025-07-09 21:10:45,699 - root - INFO - lr: 2.3086e-04 gnorm: 0.73 [ 7:35:47<14:27:38] +[titan] 2025-07-09 21:10:49,624 - root - INFO - step: 34445 loss: 16.7825 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.1785 +[titan] 2025-07-09 21:10:49,625 - root - INFO - lr: 2.3084e-04 gnorm: 0.78 [ 7:35:51<14:27:34] +[titan] 2025-07-09 21:10:52,738 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:10:53,527 - root - INFO - step: 34450 loss: 16.6976 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1211 +[titan] 2025-07-09 21:10:53,527 - root - INFO - lr: 2.3082e-04 gnorm: 0.79 [ 7:35:55<14:27:29] +[titan] 2025-07-09 21:10:57,432 - root - INFO - step: 34455 loss: 16.7715 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6015 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 21:10:57,432 - root - INFO - lr: 2.3080e-04 gnorm: 0.76 [ 7:35:58<14:27:25] +[titan] 2025-07-09 21:11:01,346 - root - INFO - step: 34460 loss: 16.5103 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9674 +[titan] 2025-07-09 21:11:01,346 - root - INFO - lr: 2.3078e-04 gnorm: 0.79 [ 7:36:02<14:27:21] +[titan] 2025-07-09 21:11:05,250 - root - INFO - step: 34465 loss: 16.6216 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0588 +[titan] 2025-07-09 21:11:05,250 - root - INFO - lr: 2.3077e-04 gnorm: 0.81 [ 7:36:06<14:27:17] +[titan] 2025-07-09 21:11:09,149 - root - INFO - step: 34470 loss: 16.7423 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5941 global_avg_mtp_loss: 14.1482 +[titan] 2025-07-09 21:11:09,149 - root - INFO - lr: 2.3075e-04 gnorm: 0.73 [ 7:36:10<14:27:13] +[titan] 2025-07-09 21:11:13,054 - root - INFO - step: 34475 loss: 16.9734 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6398 global_avg_mtp_loss: 14.3336 +[titan] 2025-07-09 21:11:13,054 - root - INFO - lr: 2.3073e-04 gnorm: 0.75 [ 7:36:14<14:27:09] +[titan] 2025-07-09 21:11:16,976 - root - INFO - step: 34480 loss: 17.2601 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.7079 global_avg_mtp_loss: 14.5523 +[titan] 2025-07-09 21:11:16,976 - root - INFO - lr: 2.3071e-04 gnorm: 0.79 [ 7:36:18<14:27:05] +[titan] 2025-07-09 21:11:20,878 - root - INFO - step: 34485 loss: 16.8800 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6301 global_avg_mtp_loss: 14.2499 +[titan] 2025-07-09 21:11:20,878 - root - INFO - lr: 2.3069e-04 gnorm: 0.76 [ 7:36:22<14:27:01] +[titan] 2025-07-09 21:11:24,800 - root - INFO - step: 34490 loss: 16.5090 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5433 global_avg_mtp_loss: 13.9657 +[titan] 2025-07-09 21:11:24,801 - root - INFO - lr: 2.3067e-04 gnorm: 0.78 [ 7:36:26<14:26:57] +[titan] 2025-07-09 21:11:28,705 - root - INFO - step: 34495 loss: 16.5920 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0244 +[titan] 2025-07-09 21:11:28,705 - root - INFO - lr: 2.3065e-04 gnorm: 0.80 [ 7:36:30<14:26:53] +[titan] 2025-07-09 21:11:31,815 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:11:32,652 - root - INFO - step: 34500 loss: 16.9011 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.6176 global_avg_mtp_loss: 14.2834 +[titan] 2025-07-09 21:11:32,652 - root - INFO - lr: 2.3063e-04 gnorm: 0.73 [ 7:36:34<14:26:49] +[titan] 2025-07-09 21:11:36,576 - root - INFO - step: 34505 loss: 16.7613 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1742 +[titan] 2025-07-09 21:11:36,576 - root - INFO - lr: 2.3062e-04 gnorm: 0.77 [ 7:36:38<14:26:45] +[titan] 2025-07-09 21:11:40,477 - root - INFO - step: 34510 loss: 16.7918 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.1932 +[titan] 2025-07-09 21:11:40,478 - root - INFO - lr: 2.3060e-04 gnorm: 0.76 [ 7:36:41<14:26:41] +[titan] 2025-07-09 21:11:44,437 - root - INFO - step: 34515 loss: 16.8032 memory: 44.58GiB(31.99%) tps: 82,764 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1917 +[titan] 2025-07-09 21:11:44,437 - root - INFO - lr: 2.3058e-04 gnorm: 0.71 [ 7:36:45<14:26:37] +[titan] 2025-07-09 21:11:48,352 - root - INFO - step: 34520 loss: 16.9793 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6338 global_avg_mtp_loss: 14.3454 +[titan] 2025-07-09 21:11:48,352 - root - INFO - lr: 2.3056e-04 gnorm: 0.78 [ 7:36:49<14:26:32] +[titan] 2025-07-09 21:11:52,272 - root - INFO - step: 34525 loss: 16.9254 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.3033 +[titan] 2025-07-09 21:11:52,272 - root - INFO - lr: 2.3054e-04 gnorm: 0.77 [ 7:36:53<14:26:28] +[titan] 2025-07-09 21:11:56,190 - root - INFO - step: 34530 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5883 global_avg_mtp_loss: 14.1133 +[titan] 2025-07-09 21:11:56,190 - root - INFO - lr: 2.3052e-04 gnorm: 0.80 [ 7:36:57<14:26:24] +[titan] 2025-07-09 21:12:00,098 - root - INFO - step: 34535 loss: 16.9511 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6355 global_avg_mtp_loss: 14.3156 +[titan] 2025-07-09 21:12:00,099 - root - INFO - lr: 2.3050e-04 gnorm: 0.82 [ 7:37:01<14:26:20] +[titan] 2025-07-09 21:12:04,025 - root - INFO - step: 34540 loss: 16.9593 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.6195 global_avg_mtp_loss: 14.3398 +[titan] 2025-07-09 21:12:04,025 - root - INFO - lr: 2.3048e-04 gnorm: 0.74 [ 7:37:05<14:26:16] +[titan] 2025-07-09 21:12:07,963 - root - INFO - step: 34545 loss: 16.9256 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.6415 global_avg_mtp_loss: 14.2841 +[titan] 2025-07-09 21:12:07,964 - root - INFO - lr: 2.3047e-04 gnorm: 0.81 [ 7:37:09<14:26:12] +[titan] 2025-07-09 21:12:11,098 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:12:11,892 - root - INFO - step: 34550 loss: 16.9608 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6375 global_avg_mtp_loss: 14.3233 +[titan] 2025-07-09 21:12:11,892 - root - INFO - lr: 2.3045e-04 gnorm: 0.75 [ 7:37:13<14:26:08] +[titan] 2025-07-09 21:12:15,791 - root - INFO - step: 34555 loss: 16.7999 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.2017 +[titan] 2025-07-09 21:12:15,791 - root - INFO - lr: 2.3043e-04 gnorm: 0.71 [ 7:37:17<14:26:04] +[titan] 2025-07-09 21:12:19,701 - root - INFO - step: 34560 loss: 16.7319 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1604 +[titan] 2025-07-09 21:12:19,701 - root - INFO - lr: 2.3041e-04 gnorm: 0.73 [ 7:37:21<14:26:00] +[titan] 2025-07-09 21:12:23,604 - root - INFO - step: 34565 loss: 16.6106 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-09 21:12:23,604 - root - INFO - lr: 2.3039e-04 gnorm: 0.80 [ 7:37:25<14:25:56] +[titan] 2025-07-09 21:12:27,508 - root - INFO - step: 34570 loss: 16.5685 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5679 global_avg_mtp_loss: 14.0007 +[titan] 2025-07-09 21:12:27,508 - root - INFO - lr: 2.3037e-04 gnorm: 0.76 [ 7:37:28<14:25:52] +[titan] 2025-07-09 21:12:31,422 - root - INFO - step: 34575 loss: 16.6093 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0400 +[titan] 2025-07-09 21:12:31,423 - root - INFO - lr: 2.3035e-04 gnorm: 0.87 [ 7:37:32<14:25:48] +[titan] 2025-07-09 21:12:35,337 - root - INFO - step: 34580 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.8751 +[titan] 2025-07-09 21:12:35,337 - root - INFO - lr: 2.3033e-04 gnorm: 0.75 [ 7:37:36<14:25:44] +[titan] 2025-07-09 21:12:39,298 - root - INFO - step: 34585 loss: 16.6727 memory: 44.58GiB(31.99%) tps: 82,735 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.1031 +[titan] 2025-07-09 21:12:39,298 - root - INFO - lr: 2.3032e-04 gnorm: 0.79 [ 7:37:40<14:25:40] +[titan] 2025-07-09 21:12:43,207 - root - INFO - step: 34590 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 21:12:43,207 - root - INFO - lr: 2.3030e-04 gnorm: 0.73 [ 7:37:44<14:25:35] +[titan] 2025-07-09 21:12:47,128 - root - INFO - step: 34595 loss: 16.5997 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0359 +[titan] 2025-07-09 21:12:47,128 - root - INFO - lr: 2.3028e-04 gnorm: 0.76 [ 7:37:48<14:25:31] +[titan] 2025-07-09 21:12:50,313 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:12:51,099 - root - INFO - step: 34600 loss: 16.3608 memory: 44.58GiB(31.99%) tps: 82,527 tflops: 284.81 mfu: 28.80% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.8390 +[titan] 2025-07-09 21:12:51,099 - root - INFO - lr: 2.3026e-04 gnorm: 0.78 [ 7:37:52<14:25:27] +[titan] 2025-07-09 21:12:55,019 - root - INFO - step: 34605 loss: 16.6729 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.0955 +[titan] 2025-07-09 21:12:55,019 - root - INFO - lr: 2.3024e-04 gnorm: 0.75 [ 7:37:56<14:25:23] +[titan] 2025-07-09 21:12:58,983 - root - INFO - step: 34610 loss: 16.7486 memory: 44.58GiB(31.99%) tps: 82,657 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1629 +[titan] 2025-07-09 21:12:58,984 - root - INFO - lr: 2.3022e-04 gnorm: 0.80 [ 7:38:00<14:25:19] +[titan] 2025-07-09 21:13:02,906 - root - INFO - step: 34615 loss: 16.5367 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 13.9865 +[titan] 2025-07-09 21:13:02,906 - root - INFO - lr: 2.3020e-04 gnorm: 0.77 [ 7:38:04<14:25:15] +[titan] 2025-07-09 21:13:06,842 - root - INFO - step: 34620 loss: 16.7581 memory: 44.58GiB(31.99%) tps: 83,256 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.1558 +[titan] 2025-07-09 21:13:06,842 - root - INFO - lr: 2.3018e-04 gnorm: 0.84 [ 7:38:08<14:25:11] +[titan] 2025-07-09 21:13:10,760 - root - INFO - step: 34625 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0791 +[titan] 2025-07-09 21:13:10,760 - root - INFO - lr: 2.3017e-04 gnorm: 0.78 [ 7:38:12<14:25:07] +[titan] 2025-07-09 21:13:14,710 - root - INFO - step: 34630 loss: 16.9451 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.3287 +[titan] 2025-07-09 21:13:14,710 - root - INFO - lr: 2.3015e-04 gnorm: 0.82 [ 7:38:16<14:25:03] +[titan] 2025-07-09 21:13:18,647 - root - INFO - step: 34635 loss: 16.7953 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5970 global_avg_mtp_loss: 14.1983 +[titan] 2025-07-09 21:13:18,648 - root - INFO - lr: 2.3013e-04 gnorm: 0.78 [ 7:38:20<14:24:59] +[titan] 2025-07-09 21:13:22,565 - root - INFO - step: 34640 loss: 16.8263 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.6121 global_avg_mtp_loss: 14.2142 +[titan] 2025-07-09 21:13:22,565 - root - INFO - lr: 2.3011e-04 gnorm: 0.82 [ 7:38:24<14:24:55] +[titan] 2025-07-09 21:13:26,488 - root - INFO - step: 34645 loss: 16.6716 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0937 +[titan] 2025-07-09 21:13:26,488 - root - INFO - lr: 2.3009e-04 gnorm: 0.74 [ 7:38:27<14:24:51] +[titan] 2025-07-09 21:13:29,636 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:13:30,421 - root - INFO - step: 34650 loss: 16.3350 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8178 +[titan] 2025-07-09 21:13:30,421 - root - INFO - lr: 2.3007e-04 gnorm: 0.84 [ 7:38:31<14:24:47] +[titan] 2025-07-09 21:13:34,381 - root - INFO - step: 34655 loss: 16.6961 memory: 44.58GiB(31.99%) tps: 82,753 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1089 +[titan] 2025-07-09 21:13:34,381 - root - INFO - lr: 2.3005e-04 gnorm: 0.74 [ 7:38:35<14:24:43] +[titan] 2025-07-09 21:13:38,383 - root - INFO - step: 34660 loss: 16.6363 memory: 44.58GiB(31.99%) tps: 81,886 tflops: 282.60 mfu: 28.57% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0719 +[titan] 2025-07-09 21:13:38,383 - root - INFO - lr: 2.3003e-04 gnorm: 0.82 [ 7:38:39<14:24:39] +[titan] 2025-07-09 21:13:42,291 - root - INFO - step: 34665 loss: 16.5219 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 13.9662 +[titan] 2025-07-09 21:13:42,291 - root - INFO - lr: 2.3002e-04 gnorm: 0.70 [ 7:38:43<14:24:35] +[titan] 2025-07-09 21:13:46,247 - root - INFO - step: 34670 loss: 16.5583 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 14.0118 +[titan] 2025-07-09 21:13:46,247 - root - INFO - lr: 2.3000e-04 gnorm: 0.73 [ 7:38:47<14:24:31] +[titan] 2025-07-09 21:13:50,144 - root - INFO - step: 34675 loss: 16.7764 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.1936 +[titan] 2025-07-09 21:13:50,145 - root - INFO - lr: 2.2998e-04 gnorm: 0.73 [ 7:38:51<14:24:27] +[titan] 2025-07-09 21:13:54,039 - root - INFO - step: 34680 loss: 16.4862 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 13.9393 +[titan] 2025-07-09 21:13:54,039 - root - INFO - lr: 2.2996e-04 gnorm: 0.75 [ 7:38:55<14:24:23] +[titan] 2025-07-09 21:13:57,946 - root - INFO - step: 34685 loss: 16.8190 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5999 global_avg_mtp_loss: 14.2191 +[titan] 2025-07-09 21:13:57,947 - root - INFO - lr: 2.2994e-04 gnorm: 0.75 [ 7:38:59<14:24:19] +[titan] 2025-07-09 21:14:01,892 - root - INFO - step: 34690 loss: 16.9034 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2756 +[titan] 2025-07-09 21:14:01,893 - root - INFO - lr: 2.2992e-04 gnorm: 0.84 [ 7:39:03<14:24:15] +[titan] 2025-07-09 21:14:05,804 - root - INFO - step: 34695 loss: 16.5459 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 13.9897 +[titan] 2025-07-09 21:14:05,804 - root - INFO - lr: 2.2990e-04 gnorm: 0.77 [ 7:39:07<14:24:11] +[titan] 2025-07-09 21:14:08,917 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:14:09,717 - root - INFO - step: 34700 loss: 16.8444 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.6176 global_avg_mtp_loss: 14.2268 +[titan] 2025-07-09 21:14:09,717 - root - INFO - lr: 2.2988e-04 gnorm: 0.80 [ 7:39:11<14:24:07] +[titan] 2025-07-09 21:14:13,654 - root - INFO - step: 34705 loss: 16.6897 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.1078 +[titan] 2025-07-09 21:14:13,654 - root - INFO - lr: 2.2987e-04 gnorm: 0.74 [ 7:39:15<14:24:03] +[titan] 2025-07-09 21:14:17,558 - root - INFO - step: 34710 loss: 16.6397 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.0680 +[titan] 2025-07-09 21:14:17,558 - root - INFO - lr: 2.2985e-04 gnorm: 0.77 [ 7:39:19<14:23:58] +[titan] 2025-07-09 21:14:21,484 - root - INFO - step: 34715 loss: 16.7403 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5900 global_avg_mtp_loss: 14.1503 +[titan] 2025-07-09 21:14:21,484 - root - INFO - lr: 2.2983e-04 gnorm: 0.75 [ 7:39:22<14:23:54] +[titan] 2025-07-09 21:14:25,394 - root - INFO - step: 34720 loss: 16.6121 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-09 21:14:25,394 - root - INFO - lr: 2.2981e-04 gnorm: 0.75 [ 7:39:26<14:23:50] +[titan] 2025-07-09 21:14:29,335 - root - INFO - step: 34725 loss: 16.6313 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.0584 +[titan] 2025-07-09 21:14:29,335 - root - INFO - lr: 2.2979e-04 gnorm: 0.85 [ 7:39:30<14:23:46] +[titan] 2025-07-09 21:14:33,268 - root - INFO - step: 34730 loss: 16.7901 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6011 global_avg_mtp_loss: 14.1890 +[titan] 2025-07-09 21:14:33,268 - root - INFO - lr: 2.2977e-04 gnorm: 0.77 [ 7:39:34<14:23:42] +[titan] 2025-07-09 21:14:37,226 - root - INFO - step: 34735 loss: 16.4955 memory: 44.58GiB(31.99%) tps: 82,785 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 13.9431 +[titan] 2025-07-09 21:14:37,226 - root - INFO - lr: 2.2975e-04 gnorm: 0.80 [ 7:39:38<14:23:38] +[titan] 2025-07-09 21:14:41,136 - root - INFO - step: 34740 loss: 16.5354 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 13.9752 +[titan] 2025-07-09 21:14:41,136 - root - INFO - lr: 2.2973e-04 gnorm: 0.77 [ 7:39:42<14:23:34] +[titan] 2025-07-09 21:14:45,046 - root - INFO - step: 34745 loss: 16.5954 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 14.0468 +[titan] 2025-07-09 21:14:45,046 - root - INFO - lr: 2.2971e-04 gnorm: 0.79 [ 7:39:46<14:23:30] +[titan] 2025-07-09 21:14:48,186 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:14:48,986 - root - INFO - step: 34750 loss: 16.8472 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.2441 +[titan] 2025-07-09 21:14:48,986 - root - INFO - lr: 2.2970e-04 gnorm: 0.76 [ 7:39:50<14:23:26] +[titan] 2025-07-09 21:14:52,936 - root - INFO - step: 34755 loss: 16.6181 memory: 44.58GiB(31.99%) tps: 82,968 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 21:14:52,936 - root - INFO - lr: 2.2968e-04 gnorm: 0.72 [ 7:39:54<14:23:22] +[titan] 2025-07-09 21:14:56,868 - root - INFO - step: 34760 loss: 16.9133 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.6265 global_avg_mtp_loss: 14.2867 +[titan] 2025-07-09 21:14:56,869 - root - INFO - lr: 2.2966e-04 gnorm: 0.81 [ 7:39:58<14:23:18] +[titan] 2025-07-09 21:15:00,761 - root - INFO - step: 34765 loss: 16.5695 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 14.0189 +[titan] 2025-07-09 21:15:00,761 - root - INFO - lr: 2.2964e-04 gnorm: 0.81 [ 7:40:02<14:23:14] +[titan] 2025-07-09 21:15:04,664 - root - INFO - step: 34770 loss: 16.5986 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0375 +[titan] 2025-07-09 21:15:04,665 - root - INFO - lr: 2.2962e-04 gnorm: 0.76 [ 7:40:06<14:23:10] +[titan] 2025-07-09 21:15:08,602 - root - INFO - step: 34775 loss: 16.6902 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5741 global_avg_mtp_loss: 14.1161 +[titan] 2025-07-09 21:15:08,602 - root - INFO - lr: 2.2960e-04 gnorm: 0.73 [ 7:40:10<14:23:06] +[titan] 2025-07-09 21:15:12,522 - root - INFO - step: 34780 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.1727 +[titan] 2025-07-09 21:15:12,522 - root - INFO - lr: 2.2958e-04 gnorm: 0.75 [ 7:40:13<14:23:02] +[titan] 2025-07-09 21:15:16,479 - root - INFO - step: 34785 loss: 16.7419 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5991 global_avg_mtp_loss: 14.1427 +[titan] 2025-07-09 21:15:16,479 - root - INFO - lr: 2.2956e-04 gnorm: 0.82 [ 7:40:17<14:22:58] +[titan] 2025-07-09 21:15:20,398 - root - INFO - step: 34790 loss: 16.7498 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5909 global_avg_mtp_loss: 14.1588 +[titan] 2025-07-09 21:15:20,398 - root - INFO - lr: 2.2955e-04 gnorm: 0.76 [ 7:40:21<14:22:54] +[titan] 2025-07-09 21:15:24,323 - root - INFO - step: 34795 loss: 16.9162 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.6129 global_avg_mtp_loss: 14.3033 +[titan] 2025-07-09 21:15:24,323 - root - INFO - lr: 2.2953e-04 gnorm: 0.79 [ 7:40:25<14:22:50] +[titan] 2025-07-09 21:15:27,462 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:15:28,248 - root - INFO - step: 34800 loss: 16.6521 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0778 +[titan] 2025-07-09 21:15:28,248 - root - INFO - lr: 2.2951e-04 gnorm: 0.78 [ 7:40:29<14:22:46] +[titan] 2025-07-09 21:15:32,154 - root - INFO - step: 34805 loss: 16.6386 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0661 +[titan] 2025-07-09 21:15:32,155 - root - INFO - lr: 2.2949e-04 gnorm: 0.73 [ 7:40:33<14:22:41] +[titan] 2025-07-09 21:15:36,066 - root - INFO - step: 34810 loss: 16.7850 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1925 +[titan] 2025-07-09 21:15:36,066 - root - INFO - lr: 2.2947e-04 gnorm: 0.78 [ 7:40:37<14:22:37] +[titan] 2025-07-09 21:15:40,055 - root - INFO - step: 34815 loss: 16.6704 memory: 44.58GiB(31.99%) tps: 82,150 tflops: 283.51 mfu: 28.67% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.1053 +[titan] 2025-07-09 21:15:40,055 - root - INFO - lr: 2.2945e-04 gnorm: 0.74 [ 7:40:41<14:22:33] +[titan] 2025-07-09 21:15:40,996 - root - INFO - Dumping profiler traces at step 34816 +[titan] 2025-07-09 21:15:41,027 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:15:44,182 - root - INFO - step: 34820 loss: 16.4209 memory: 44.58GiB(31.99%) tps: 79,402 tflops: 274.03 mfu: 27.71% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-09 21:15:44,182 - root - INFO - lr: 2.2943e-04 gnorm: 0.75 [ 7:40:45<14:22:30] +[titan] 2025-07-09 21:15:48,097 - root - INFO - step: 34825 loss: 16.5637 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0060 +[titan] 2025-07-09 21:15:48,097 - root - INFO - lr: 2.2941e-04 gnorm: 0.85 [ 7:40:49<14:22:26] +[titan] 2025-07-09 21:15:52,012 - root - INFO - step: 34830 loss: 16.7563 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1597 +[titan] 2025-07-09 21:15:52,012 - root - INFO - lr: 2.2940e-04 gnorm: 0.85 [ 7:40:53<14:22:22] +[titan] 2025-07-09 21:15:55,927 - root - INFO - step: 34835 loss: 16.8482 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.2404 +[titan] 2025-07-09 21:15:55,927 - root - INFO - lr: 2.2938e-04 gnorm: 0.75 [ 7:40:57<14:22:18] +[titan] 2025-07-09 21:15:59,907 - root - INFO - step: 34840 loss: 16.6646 memory: 44.58GiB(31.99%) tps: 82,335 tflops: 284.15 mfu: 28.73% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.0867 +[titan] 2025-07-09 21:15:59,907 - root - INFO - lr: 2.2936e-04 gnorm: 0.76 [ 7:41:01<14:22:14] +[titan] 2025-07-09 21:16:03,818 - root - INFO - step: 34845 loss: 16.6345 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5635 global_avg_mtp_loss: 14.0710 +[titan] 2025-07-09 21:16:03,818 - root - INFO - lr: 2.2934e-04 gnorm: 0.77 [ 7:41:05<14:22:09] +[titan] 2025-07-09 21:16:06,955 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:16:07,775 - root - INFO - step: 34850 loss: 16.4495 memory: 44.58GiB(31.99%) tps: 82,813 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9184 +[titan] 2025-07-09 21:16:07,775 - root - INFO - lr: 2.2932e-04 gnorm: 0.77 [ 7:41:09<14:22:05] +[titan] 2025-07-09 21:16:11,688 - root - INFO - step: 34855 loss: 16.6518 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.0666 +[titan] 2025-07-09 21:16:11,688 - root - INFO - lr: 2.2930e-04 gnorm: 0.79 [ 7:41:13<14:22:01] +[titan] 2025-07-09 21:16:15,611 - root - INFO - step: 34860 loss: 16.5729 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0232 +[titan] 2025-07-09 21:16:15,611 - root - INFO - lr: 2.2928e-04 gnorm: 0.78 [ 7:41:17<14:21:57] +[titan] 2025-07-09 21:16:19,538 - root - INFO - step: 34865 loss: 16.6764 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.0969 +[titan] 2025-07-09 21:16:19,538 - root - INFO - lr: 2.2926e-04 gnorm: 0.75 [ 7:41:20<14:21:53] +[titan] 2025-07-09 21:16:23,445 - root - INFO - step: 34870 loss: 16.7074 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1274 +[titan] 2025-07-09 21:16:23,445 - root - INFO - lr: 2.2924e-04 gnorm: 0.76 [ 7:41:24<14:21:49] +[titan] 2025-07-09 21:16:27,372 - root - INFO - step: 34875 loss: 16.6618 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.1040 +[titan] 2025-07-09 21:16:27,373 - root - INFO - lr: 2.2923e-04 gnorm: 0.77 [ 7:41:28<14:21:45] +[titan] 2025-07-09 21:16:31,325 - root - INFO - step: 34880 loss: 16.7636 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.1592 +[titan] 2025-07-09 21:16:31,326 - root - INFO - lr: 2.2921e-04 gnorm: 0.83 [ 7:41:32<14:21:41] +[titan] 2025-07-09 21:16:35,294 - root - INFO - step: 34885 loss: 16.6913 memory: 44.58GiB(31.99%) tps: 82,571 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.1092 +[titan] 2025-07-09 21:16:35,294 - root - INFO - lr: 2.2919e-04 gnorm: 0.85 [ 7:41:36<14:21:37] +[titan] 2025-07-09 21:16:39,201 - root - INFO - step: 34890 loss: 17.0327 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6565 global_avg_mtp_loss: 14.3762 +[titan] 2025-07-09 21:16:39,202 - root - INFO - lr: 2.2917e-04 gnorm: 0.78 [ 7:41:40<14:21:33] +[titan] 2025-07-09 21:16:43,133 - root - INFO - step: 34895 loss: 16.6706 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.0847 +[titan] 2025-07-09 21:16:43,133 - root - INFO - lr: 2.2915e-04 gnorm: 0.74 [ 7:41:44<14:21:29] +[titan] 2025-07-09 21:16:46,272 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:16:47,072 - root - INFO - step: 34900 loss: 16.8972 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.6190 global_avg_mtp_loss: 14.2782 +[titan] 2025-07-09 21:16:47,072 - root - INFO - lr: 2.2913e-04 gnorm: 0.77 [ 7:41:48<14:21:25] +[titan] 2025-07-09 21:16:50,998 - root - INFO - step: 34905 loss: 16.7354 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1434 +[titan] 2025-07-09 21:16:50,998 - root - INFO - lr: 2.2911e-04 gnorm: 0.76 [ 7:41:52<14:21:21] +[titan] 2025-07-09 21:16:54,914 - root - INFO - step: 34910 loss: 16.7181 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.1267 +[titan] 2025-07-09 21:16:54,915 - root - INFO - lr: 2.2909e-04 gnorm: 0.74 [ 7:41:56<14:21:17] +[titan] 2025-07-09 21:16:58,827 - root - INFO - step: 34915 loss: 16.6978 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 21:16:58,828 - root - INFO - lr: 2.2907e-04 gnorm: 0.76 [ 7:42:00<14:21:13] +[titan] 2025-07-09 21:17:02,747 - root - INFO - step: 34920 loss: 16.6828 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.1052 +[titan] 2025-07-09 21:17:02,747 - root - INFO - lr: 2.2906e-04 gnorm: 0.79 [ 7:42:04<14:21:09] +[titan] 2025-07-09 21:17:06,669 - root - INFO - step: 34925 loss: 16.7930 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.6038 global_avg_mtp_loss: 14.1891 +[titan] 2025-07-09 21:17:06,669 - root - INFO - lr: 2.2904e-04 gnorm: 0.73 [ 7:42:08<14:21:05] +[titan] 2025-07-09 21:17:10,619 - root - INFO - step: 34930 loss: 16.8425 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.6012 global_avg_mtp_loss: 14.2412 +[titan] 2025-07-09 21:17:10,619 - root - INFO - lr: 2.2902e-04 gnorm: 0.77 [ 7:42:12<14:21:01] +[titan] 2025-07-09 21:17:14,527 - root - INFO - step: 34935 loss: 16.7423 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5957 global_avg_mtp_loss: 14.1466 +[titan] 2025-07-09 21:17:14,527 - root - INFO - lr: 2.2900e-04 gnorm: 0.75 [ 7:42:15<14:20:57] +[titan] 2025-07-09 21:17:18,445 - root - INFO - step: 34940 loss: 16.6456 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-09 21:17:18,445 - root - INFO - lr: 2.2898e-04 gnorm: 0.74 [ 7:42:19<14:20:53] +[titan] 2025-07-09 21:17:22,367 - root - INFO - step: 34945 loss: 16.4861 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9431 +[titan] 2025-07-09 21:17:22,367 - root - INFO - lr: 2.2896e-04 gnorm: 0.76 [ 7:42:23<14:20:48] +[titan] 2025-07-09 21:17:25,484 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:17:26,268 - root - INFO - step: 34950 loss: 16.9816 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6572 global_avg_mtp_loss: 14.3245 +[titan] 2025-07-09 21:17:26,268 - root - INFO - lr: 2.2894e-04 gnorm: 0.74 [ 7:42:27<14:20:44] +[titan] 2025-07-09 21:17:30,200 - root - INFO - step: 34955 loss: 16.5070 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9565 +[titan] 2025-07-09 21:17:30,200 - root - INFO - lr: 2.2892e-04 gnorm: 0.75 [ 7:42:31<14:20:40] +[titan] 2025-07-09 21:17:34,117 - root - INFO - step: 34960 loss: 16.4951 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9492 +[titan] 2025-07-09 21:17:34,117 - root - INFO - lr: 2.2891e-04 gnorm: 0.83 [ 7:42:35<14:20:36] +[titan] 2025-07-09 21:17:38,013 - root - INFO - step: 34965 loss: 16.7280 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.1413 +[titan] 2025-07-09 21:17:38,014 - root - INFO - lr: 2.2889e-04 gnorm: 0.77 [ 7:42:39<14:20:32] +[titan] 2025-07-09 21:17:41,976 - root - INFO - step: 34970 loss: 16.7304 memory: 44.58GiB(31.99%) tps: 82,706 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.1504 +[titan] 2025-07-09 21:17:41,976 - root - INFO - lr: 2.2887e-04 gnorm: 0.76 [ 7:42:43<14:20:28] +[titan] 2025-07-09 21:17:45,884 - root - INFO - step: 34975 loss: 16.6920 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1036 +[titan] 2025-07-09 21:17:45,884 - root - INFO - lr: 2.2885e-04 gnorm: 0.79 [ 7:42:47<14:20:24] +[titan] 2025-07-09 21:17:49,802 - root - INFO - step: 34980 loss: 16.7228 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.1451 +[titan] 2025-07-09 21:17:49,802 - root - INFO - lr: 2.2883e-04 gnorm: 0.78 [ 7:42:51<14:20:20] +[titan] 2025-07-09 21:17:53,705 - root - INFO - step: 34985 loss: 16.6764 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.0890 +[titan] 2025-07-09 21:17:53,705 - root - INFO - lr: 2.2881e-04 gnorm: 0.80 [ 7:42:55<14:20:16] +[titan] 2025-07-09 21:17:57,619 - root - INFO - step: 34990 loss: 16.7408 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5865 global_avg_mtp_loss: 14.1544 +[titan] 2025-07-09 21:17:57,620 - root - INFO - lr: 2.2879e-04 gnorm: 0.75 [ 7:42:59<14:20:12] +[titan] 2025-07-09 21:18:01,534 - root - INFO - step: 34995 loss: 16.6729 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5719 global_avg_mtp_loss: 14.1011 +[titan] 2025-07-09 21:18:01,535 - root - INFO - lr: 2.2877e-04 gnorm: 0.81 [ 7:43:02<14:20:08] +[titan] 2025-07-09 21:18:04,652 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:18:05,446 - root - INFO - step: 35000 loss: 16.8880 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.2716 +[titan] 2025-07-09 21:18:05,446 - root - INFO - lr: 2.2875e-04 gnorm: 0.76 [ 7:43:06<14:20:04] +[titan] 2025-07-09 21:18:09,378 - root - INFO - step: 35005 loss: 16.5785 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5618 global_avg_mtp_loss: 14.0168 +[titan] 2025-07-09 21:18:09,378 - root - INFO - lr: 2.2874e-04 gnorm: 0.76 [ 7:43:10<14:20:00] +[titan] 2025-07-09 21:18:13,288 - root - INFO - step: 35010 loss: 16.7562 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1718 +[titan] 2025-07-09 21:18:13,288 - root - INFO - lr: 2.2872e-04 gnorm: 0.78 [ 7:43:14<14:19:56] +[titan] 2025-07-09 21:18:17,194 - root - INFO - step: 35015 loss: 16.6801 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.1046 +[titan] 2025-07-09 21:18:17,194 - root - INFO - lr: 2.2870e-04 gnorm: 0.76 [ 7:43:18<14:19:52] +[titan] 2025-07-09 21:18:21,134 - root - INFO - step: 35020 loss: 16.5687 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 14.0161 +[titan] 2025-07-09 21:18:21,134 - root - INFO - lr: 2.2868e-04 gnorm: 0.75 [ 7:43:22<14:19:47] +[titan] 2025-07-09 21:18:25,053 - root - INFO - step: 35025 loss: 16.7431 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1584 +[titan] 2025-07-09 21:18:25,054 - root - INFO - lr: 2.2866e-04 gnorm: 0.80 [ 7:43:26<14:19:43] +[titan] 2025-07-09 21:18:28,969 - root - INFO - step: 35030 loss: 16.4199 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.9004 +[titan] 2025-07-09 21:18:28,970 - root - INFO - lr: 2.2864e-04 gnorm: 0.76 [ 7:43:30<14:19:39] +[titan] 2025-07-09 21:18:32,890 - root - INFO - step: 35035 loss: 16.9368 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6273 global_avg_mtp_loss: 14.3094 +[titan] 2025-07-09 21:18:32,890 - root - INFO - lr: 2.2862e-04 gnorm: 0.77 [ 7:43:34<14:19:35] +[titan] 2025-07-09 21:18:36,789 - root - INFO - step: 35040 loss: 16.8851 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6202 global_avg_mtp_loss: 14.2649 +[titan] 2025-07-09 21:18:36,790 - root - INFO - lr: 2.2860e-04 gnorm: 0.75 [ 7:43:38<14:19:31] +[titan] 2025-07-09 21:18:40,713 - root - INFO - step: 35045 loss: 16.8288 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.2100 +[titan] 2025-07-09 21:18:40,714 - root - INFO - lr: 2.2858e-04 gnorm: 0.74 [ 7:43:42<14:19:27] +[titan] 2025-07-09 21:18:43,835 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:18:44,633 - root - INFO - step: 35050 loss: 16.5959 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0390 +[titan] 2025-07-09 21:18:44,633 - root - INFO - lr: 2.2857e-04 gnorm: 0.73 [ 7:43:46<14:19:23] +[titan] 2025-07-09 21:18:48,545 - root - INFO - step: 35055 loss: 16.7387 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5941 global_avg_mtp_loss: 14.1446 +[titan] 2025-07-09 21:18:48,545 - root - INFO - lr: 2.2855e-04 gnorm: 0.81 [ 7:43:49<14:19:19] +[titan] 2025-07-09 21:18:52,452 - root - INFO - step: 35060 loss: 16.6072 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0346 +[titan] 2025-07-09 21:18:52,452 - root - INFO - lr: 2.2853e-04 gnorm: 0.74 [ 7:43:53<14:19:15] +[titan] 2025-07-09 21:18:56,366 - root - INFO - step: 35065 loss: 16.4518 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9112 +[titan] 2025-07-09 21:18:56,366 - root - INFO - lr: 2.2851e-04 gnorm: 0.76 [ 7:43:57<14:19:11] +[titan] 2025-07-09 21:19:00,274 - root - INFO - step: 35070 loss: 16.7218 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.1351 +[titan] 2025-07-09 21:19:00,275 - root - INFO - lr: 2.2849e-04 gnorm: 0.73 [ 7:44:01<14:19:07] +[titan] 2025-07-09 21:19:04,210 - root - INFO - step: 35075 loss: 16.6770 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1047 +[titan] 2025-07-09 21:19:04,211 - root - INFO - lr: 2.2847e-04 gnorm: 0.73 [ 7:44:05<14:19:03] +[titan] 2025-07-09 21:19:08,142 - root - INFO - step: 35080 loss: 16.6655 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0899 +[titan] 2025-07-09 21:19:08,142 - root - INFO - lr: 2.2845e-04 gnorm: 0.76 [ 7:44:09<14:18:59] +[titan] 2025-07-09 21:19:12,055 - root - INFO - step: 35085 loss: 16.7175 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.1408 +[titan] 2025-07-09 21:19:12,055 - root - INFO - lr: 2.2843e-04 gnorm: 0.76 [ 7:44:13<14:18:55] +[titan] 2025-07-09 21:19:15,969 - root - INFO - step: 35090 loss: 16.7798 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5991 global_avg_mtp_loss: 14.1807 +[titan] 2025-07-09 21:19:15,969 - root - INFO - lr: 2.2841e-04 gnorm: 0.79 [ 7:44:17<14:18:51] +[titan] 2025-07-09 21:19:19,891 - root - INFO - step: 35095 loss: 16.4847 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9496 +[titan] 2025-07-09 21:19:19,892 - root - INFO - lr: 2.2840e-04 gnorm: 0.77 [ 7:44:21<14:18:46] +[titan] 2025-07-09 21:19:23,009 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:19:23,797 - root - INFO - step: 35100 loss: 16.5545 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 13.9898 +[titan] 2025-07-09 21:19:23,797 - root - INFO - lr: 2.2838e-04 gnorm: 0.78 [ 7:44:25<14:18:42] +[titan] 2025-07-09 21:19:27,708 - root - INFO - step: 35105 loss: 16.7248 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.1451 +[titan] 2025-07-09 21:19:27,708 - root - INFO - lr: 2.2836e-04 gnorm: 0.78 [ 7:44:29<14:18:38] +[titan] 2025-07-09 21:19:31,646 - root - INFO - step: 35110 loss: 16.7422 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.1547 +[titan] 2025-07-09 21:19:31,646 - root - INFO - lr: 2.2834e-04 gnorm: 0.81 [ 7:44:33<14:18:34] +[titan] 2025-07-09 21:19:35,554 - root - INFO - step: 35115 loss: 16.3801 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.8540 +[titan] 2025-07-09 21:19:35,555 - root - INFO - lr: 2.2832e-04 gnorm: 0.81 [ 7:44:36<14:18:30] +[titan] 2025-07-09 21:19:39,479 - root - INFO - step: 35120 loss: 16.5642 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5732 global_avg_mtp_loss: 13.9910 +[titan] 2025-07-09 21:19:39,480 - root - INFO - lr: 2.2830e-04 gnorm: 1.61 [ 7:44:40<14:18:26] +[titan] 2025-07-09 21:19:43,388 - root - INFO - step: 35125 loss: 16.8762 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6139 global_avg_mtp_loss: 14.2623 +[titan] 2025-07-09 21:19:43,388 - root - INFO - lr: 2.2828e-04 gnorm: 0.79 [ 7:44:44<14:18:22] +[titan] 2025-07-09 21:19:47,295 - root - INFO - step: 35130 loss: 16.4133 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.8809 +[titan] 2025-07-09 21:19:47,295 - root - INFO - lr: 2.2826e-04 gnorm: 0.79 [ 7:44:48<14:18:18] +[titan] 2025-07-09 21:19:51,225 - root - INFO - step: 35135 loss: 16.6267 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0591 +[titan] 2025-07-09 21:19:51,226 - root - INFO - lr: 2.2824e-04 gnorm: 0.73 [ 7:44:52<14:18:14] +[titan] 2025-07-09 21:19:55,141 - root - INFO - step: 35140 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5597 global_avg_mtp_loss: 14.0571 +[titan] 2025-07-09 21:19:55,142 - root - INFO - lr: 2.2822e-04 gnorm: 0.77 [ 7:44:56<14:18:10] +[titan] 2025-07-09 21:19:59,069 - root - INFO - step: 35145 loss: 16.7320 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1415 +[titan] 2025-07-09 21:19:59,069 - root - INFO - lr: 2.2821e-04 gnorm: 0.80 [ 7:45:00<14:18:06] +[titan] 2025-07-09 21:20:02,181 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:20:02,975 - root - INFO - step: 35150 loss: 16.6910 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.1090 +[titan] 2025-07-09 21:20:02,975 - root - INFO - lr: 2.2819e-04 gnorm: 0.91 [ 7:45:04<14:18:02] +[titan] 2025-07-09 21:20:06,889 - root - INFO - step: 35155 loss: 16.6631 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.0749 +[titan] 2025-07-09 21:20:06,890 - root - INFO - lr: 2.2817e-04 gnorm: 0.76 [ 7:45:08<14:17:58] +[titan] 2025-07-09 21:20:10,794 - root - INFO - step: 35160 loss: 16.3912 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.8619 +[titan] 2025-07-09 21:20:10,794 - root - INFO - lr: 2.2815e-04 gnorm: 0.71 [ 7:45:12<14:17:54] +[titan] 2025-07-09 21:20:14,729 - root - INFO - step: 35165 loss: 16.6914 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5782 global_avg_mtp_loss: 14.1132 +[titan] 2025-07-09 21:20:14,729 - root - INFO - lr: 2.2813e-04 gnorm: 0.77 [ 7:45:16<14:17:50] +[titan] 2025-07-09 21:20:18,621 - root - INFO - step: 35170 loss: 16.7739 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 21:20:18,622 - root - INFO - lr: 2.2811e-04 gnorm: 0.77 [ 7:45:20<14:17:45] +[titan] 2025-07-09 21:20:22,508 - root - INFO - step: 35175 loss: 16.7375 memory: 44.58GiB(31.99%) tps: 84,326 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.1422 +[titan] 2025-07-09 21:20:22,508 - root - INFO - lr: 2.2809e-04 gnorm: 0.76 [ 7:45:23<14:17:41] +[titan] 2025-07-09 21:20:26,415 - root - INFO - step: 35180 loss: 16.7892 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.2046 +[titan] 2025-07-09 21:20:26,415 - root - INFO - lr: 2.2807e-04 gnorm: 0.76 [ 7:45:27<14:17:37] +[titan] 2025-07-09 21:20:30,325 - root - INFO - step: 35185 loss: 16.8405 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.2387 +[titan] 2025-07-09 21:20:30,325 - root - INFO - lr: 2.2805e-04 gnorm: 0.80 [ 7:45:31<14:17:33] +[titan] 2025-07-09 21:20:34,236 - root - INFO - step: 35190 loss: 16.4393 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.8952 +[titan] 2025-07-09 21:20:34,237 - root - INFO - lr: 2.2804e-04 gnorm: 0.76 [ 7:45:35<14:17:29] +[titan] 2025-07-09 21:20:38,146 - root - INFO - step: 35195 loss: 16.7487 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.1599 +[titan] 2025-07-09 21:20:38,146 - root - INFO - lr: 2.2802e-04 gnorm: 0.72 [ 7:45:39<14:17:25] +[titan] 2025-07-09 21:20:41,285 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:20:42,079 - root - INFO - step: 35200 loss: 16.6466 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.0697 +[titan] 2025-07-09 21:20:42,079 - root - INFO - lr: 2.2800e-04 gnorm: 0.81 [ 7:45:43<14:17:21] +[titan] 2025-07-09 21:20:45,964 - root - INFO - step: 35205 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 84,338 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0219 +[titan] 2025-07-09 21:20:45,965 - root - INFO - lr: 2.2798e-04 gnorm: 0.84 [ 7:45:47<14:17:17] +[titan] 2025-07-09 21:20:49,915 - root - INFO - step: 35210 loss: 16.7623 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1828 +[titan] 2025-07-09 21:20:49,916 - root - INFO - lr: 2.2796e-04 gnorm: 0.83 [ 7:45:51<14:17:13] +[titan] 2025-07-09 21:20:53,835 - root - INFO - step: 35215 loss: 17.1167 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.6865 global_avg_mtp_loss: 14.4302 +[titan] 2025-07-09 21:20:53,835 - root - INFO - lr: 2.2794e-04 gnorm: 0.88 [ 7:45:55<14:17:09] +[titan] 2025-07-09 21:20:57,753 - root - INFO - step: 35220 loss: 16.7956 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.1933 +[titan] 2025-07-09 21:20:57,753 - root - INFO - lr: 2.2792e-04 gnorm: 0.84 [ 7:45:59<14:17:05] +[titan] 2025-07-09 21:21:01,676 - root - INFO - step: 35225 loss: 16.7625 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1782 +[titan] 2025-07-09 21:21:01,677 - root - INFO - lr: 2.2790e-04 gnorm: 0.78 [ 7:46:03<14:17:01] +[titan] 2025-07-09 21:21:05,620 - root - INFO - step: 35230 loss: 16.8301 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.2324 +[titan] 2025-07-09 21:21:05,621 - root - INFO - lr: 2.2788e-04 gnorm: 0.83 [ 7:46:07<14:16:57] +[titan] 2025-07-09 21:21:09,523 - root - INFO - step: 35235 loss: 16.8027 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.2049 +[titan] 2025-07-09 21:21:09,523 - root - INFO - lr: 2.2786e-04 gnorm: 0.75 [ 7:46:10<14:16:53] +[titan] 2025-07-09 21:21:13,468 - root - INFO - step: 35240 loss: 16.5696 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 14.0264 +[titan] 2025-07-09 21:21:13,468 - root - INFO - lr: 2.2785e-04 gnorm: 0.82 [ 7:46:14<14:16:49] +[titan] 2025-07-09 21:21:17,362 - root - INFO - step: 35245 loss: 16.9299 memory: 44.58GiB(31.99%) tps: 84,151 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.6295 global_avg_mtp_loss: 14.3003 +[titan] 2025-07-09 21:21:17,362 - root - INFO - lr: 2.2783e-04 gnorm: 0.80 [ 7:46:18<14:16:44] +[titan] 2025-07-09 21:21:20,467 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:21:21,253 - root - INFO - step: 35250 loss: 16.7031 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1308 +[titan] 2025-07-09 21:21:21,254 - root - INFO - lr: 2.2781e-04 gnorm: 0.75 [ 7:46:22<14:16:40] +[titan] 2025-07-09 21:21:25,157 - root - INFO - step: 35255 loss: 16.7897 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.1874 +[titan] 2025-07-09 21:21:25,157 - root - INFO - lr: 2.2779e-04 gnorm: 0.75 [ 7:46:26<14:16:36] +[titan] 2025-07-09 21:21:29,063 - root - INFO - step: 35260 loss: 16.7716 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1846 +[titan] 2025-07-09 21:21:29,063 - root - INFO - lr: 2.2777e-04 gnorm: 0.73 [ 7:46:30<14:16:32] +[titan] 2025-07-09 21:21:32,995 - root - INFO - step: 35265 loss: 16.7321 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1365 +[titan] 2025-07-09 21:21:32,995 - root - INFO - lr: 2.2775e-04 gnorm: 0.76 [ 7:46:34<14:16:28] +[titan] 2025-07-09 21:21:36,887 - root - INFO - step: 35270 loss: 16.5600 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.5542 global_avg_mtp_loss: 14.0058 +[titan] 2025-07-09 21:21:36,888 - root - INFO - lr: 2.2773e-04 gnorm: 0.76 [ 7:46:38<14:16:24] +[titan] 2025-07-09 21:21:40,833 - root - INFO - step: 35275 loss: 16.5979 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 14.0412 +[titan] 2025-07-09 21:21:40,833 - root - INFO - lr: 2.2771e-04 gnorm: 0.78 [ 7:46:42<14:16:20] +[titan] 2025-07-09 21:21:44,738 - root - INFO - step: 35280 loss: 16.7199 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1352 +[titan] 2025-07-09 21:21:44,739 - root - INFO - lr: 2.2769e-04 gnorm: 0.79 [ 7:46:46<14:16:16] +[titan] 2025-07-09 21:21:48,683 - root - INFO - step: 35285 loss: 16.7917 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.1883 +[titan] 2025-07-09 21:21:48,683 - root - INFO - lr: 2.2768e-04 gnorm: 0.79 [ 7:46:50<14:16:12] +[titan] 2025-07-09 21:21:52,583 - root - INFO - step: 35290 loss: 16.7217 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1453 +[titan] 2025-07-09 21:21:52,584 - root - INFO - lr: 2.2766e-04 gnorm: 0.83 [ 7:46:53<14:16:08] +[titan] 2025-07-09 21:21:56,479 - root - INFO - step: 35295 loss: 16.7054 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5886 global_avg_mtp_loss: 14.1169 +[titan] 2025-07-09 21:21:56,480 - root - INFO - lr: 2.2764e-04 gnorm: 0.72 [ 7:46:57<14:16:04] +[titan] 2025-07-09 21:21:59,633 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:22:00,422 - root - INFO - step: 35300 loss: 16.6832 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.1013 +[titan] 2025-07-09 21:22:00,422 - root - INFO - lr: 2.2762e-04 gnorm: 0.80 [ 7:47:01<14:16:00] +[titan] 2025-07-09 21:22:04,338 - root - INFO - step: 35305 loss: 16.8458 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6170 global_avg_mtp_loss: 14.2288 +[titan] 2025-07-09 21:22:04,338 - root - INFO - lr: 2.2760e-04 gnorm: 0.77 [ 7:47:05<14:15:56] +[titan] 2025-07-09 21:22:08,255 - root - INFO - step: 35310 loss: 16.4724 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.9299 +[titan] 2025-07-09 21:22:08,255 - root - INFO - lr: 2.2758e-04 gnorm: 0.79 [ 7:47:09<14:15:52] +[titan] 2025-07-09 21:22:12,171 - root - INFO - step: 35315 loss: 16.6257 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 14.0696 +[titan] 2025-07-09 21:22:12,171 - root - INFO - lr: 2.2756e-04 gnorm: 0.84 [ 7:47:13<14:15:47] +[titan] 2025-07-09 21:22:16,089 - root - INFO - step: 35320 loss: 16.5618 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0019 +[titan] 2025-07-09 21:22:16,089 - root - INFO - lr: 2.2754e-04 gnorm: 0.81 [ 7:47:17<14:15:43] +[titan] 2025-07-09 21:22:20,070 - root - INFO - step: 35325 loss: 16.7062 memory: 44.58GiB(31.99%) tps: 82,310 tflops: 284.07 mfu: 28.72% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.1234 +[titan] 2025-07-09 21:22:20,071 - root - INFO - lr: 2.2752e-04 gnorm: 0.81 [ 7:47:21<14:15:39] +[titan] 2025-07-09 21:22:22,574 - root - INFO - Dumping profiler traces at step 35328 +[titan] 2025-07-09 21:22:22,606 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:22:24,204 - root - INFO - step: 35330 loss: 16.6591 memory: 44.58GiB(31.99%) tps: 79,285 tflops: 273.62 mfu: 27.67% global_avg_ntp_loss: 2.5719 global_avg_mtp_loss: 14.0872 +[titan] 2025-07-09 21:22:24,204 - root - INFO - lr: 2.2750e-04 gnorm: 0.81 [ 7:47:25<14:15:36] +[titan] 2025-07-09 21:22:28,111 - root - INFO - step: 35335 loss: 16.5716 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0234 +[titan] 2025-07-09 21:22:28,112 - root - INFO - lr: 2.2749e-04 gnorm: 0.77 [ 7:47:29<14:15:32] +[titan] 2025-07-09 21:22:32,049 - root - INFO - step: 35340 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0963 +[titan] 2025-07-09 21:22:32,049 - root - INFO - lr: 2.2747e-04 gnorm: 0.79 [ 7:47:33<14:15:28] +[titan] 2025-07-09 21:22:35,964 - root - INFO - step: 35345 loss: 16.4264 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.8865 +[titan] 2025-07-09 21:22:35,964 - root - INFO - lr: 2.2745e-04 gnorm: 0.76 [ 7:47:37<14:15:24] +[titan] 2025-07-09 21:22:39,090 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:22:39,875 - root - INFO - step: 35350 loss: 16.8155 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.2235 +[titan] 2025-07-09 21:22:39,876 - root - INFO - lr: 2.2743e-04 gnorm: 0.77 [ 7:47:41<14:15:19] +[titan] 2025-07-09 21:22:43,789 - root - INFO - step: 35355 loss: 16.7363 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 14.1615 +[titan] 2025-07-09 21:22:43,789 - root - INFO - lr: 2.2741e-04 gnorm: 0.77 [ 7:47:45<14:15:15] +[titan] 2025-07-09 21:22:47,708 - root - INFO - step: 35360 loss: 16.8793 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 21:22:47,709 - root - INFO - lr: 2.2739e-04 gnorm: 0.78 [ 7:47:49<14:15:11] +[titan] 2025-07-09 21:22:51,605 - root - INFO - step: 35365 loss: 16.7721 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5970 global_avg_mtp_loss: 14.1751 +[titan] 2025-07-09 21:22:51,605 - root - INFO - lr: 2.2737e-04 gnorm: 0.82 [ 7:47:53<14:15:07] +[titan] 2025-07-09 21:22:55,555 - root - INFO - step: 35370 loss: 16.6041 memory: 44.58GiB(31.99%) tps: 82,953 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5583 global_avg_mtp_loss: 14.0458 +[titan] 2025-07-09 21:22:55,556 - root - INFO - lr: 2.2735e-04 gnorm: 0.69 [ 7:47:56<14:15:03] +[titan] 2025-07-09 21:22:59,514 - root - INFO - step: 35375 loss: 16.7106 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.67 mfu: 28.89% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1200 +[titan] 2025-07-09 21:22:59,515 - root - INFO - lr: 2.2733e-04 gnorm: 0.74 [ 7:48:00<14:14:59] +[titan] 2025-07-09 21:23:03,417 - root - INFO - step: 35380 loss: 16.6314 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.0542 +[titan] 2025-07-09 21:23:03,417 - root - INFO - lr: 2.2731e-04 gnorm: 0.75 [ 7:48:04<14:14:55] +[titan] 2025-07-09 21:23:07,331 - root - INFO - step: 35385 loss: 16.6691 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0952 +[titan] 2025-07-09 21:23:07,332 - root - INFO - lr: 2.2730e-04 gnorm: 0.76 [ 7:48:08<14:14:51] +[titan] 2025-07-09 21:23:11,235 - root - INFO - step: 35390 loss: 16.6381 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-09 21:23:11,235 - root - INFO - lr: 2.2728e-04 gnorm: 0.76 [ 7:48:12<14:14:47] +[titan] 2025-07-09 21:23:15,178 - root - INFO - step: 35395 loss: 16.4306 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.8998 +[titan] 2025-07-09 21:23:15,179 - root - INFO - lr: 2.2726e-04 gnorm: 0.77 [ 7:48:16<14:14:43] +[titan] 2025-07-09 21:23:18,282 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:23:19,084 - root - INFO - step: 35400 loss: 16.9208 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.2929 +[titan] 2025-07-09 21:23:19,084 - root - INFO - lr: 2.2724e-04 gnorm: 0.81 [ 7:48:20<14:14:39] +[titan] 2025-07-09 21:23:23,026 - root - INFO - step: 35405 loss: 16.6616 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-09 21:23:23,026 - root - INFO - lr: 2.2722e-04 gnorm: 0.75 [ 7:48:24<14:14:35] +[titan] 2025-07-09 21:23:26,929 - root - INFO - step: 35410 loss: 16.7600 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5955 global_avg_mtp_loss: 14.1645 +[titan] 2025-07-09 21:23:26,929 - root - INFO - lr: 2.2720e-04 gnorm: 0.79 [ 7:48:28<14:14:31] +[titan] 2025-07-09 21:23:30,843 - root - INFO - step: 35415 loss: 16.7191 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1289 +[titan] 2025-07-09 21:23:30,844 - root - INFO - lr: 2.2718e-04 gnorm: 0.73 [ 7:48:32<14:14:27] +[titan] 2025-07-09 21:23:34,795 - root - INFO - step: 35420 loss: 16.6258 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.5759 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-09 21:23:34,795 - root - INFO - lr: 2.2716e-04 gnorm: 0.75 [ 7:48:36<14:14:23] +[titan] 2025-07-09 21:23:38,703 - root - INFO - step: 35425 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0520 +[titan] 2025-07-09 21:23:38,703 - root - INFO - lr: 2.2714e-04 gnorm: 0.74 [ 7:48:40<14:14:19] +[titan] 2025-07-09 21:23:42,687 - root - INFO - step: 35430 loss: 16.3158 memory: 44.58GiB(31.99%) tps: 82,259 tflops: 283.89 mfu: 28.70% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.7984 +[titan] 2025-07-09 21:23:42,687 - root - INFO - lr: 2.2712e-04 gnorm: 0.78 [ 7:48:44<14:14:15] +[titan] 2025-07-09 21:23:46,714 - root - INFO - step: 35435 loss: 16.8448 memory: 44.58GiB(31.99%) tps: 81,372 tflops: 280.83 mfu: 28.40% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.2295 +[titan] 2025-07-09 21:23:46,714 - root - INFO - lr: 2.2711e-04 gnorm: 0.80 [ 7:48:48<14:14:11] +[titan] 2025-07-09 21:23:50,616 - root - INFO - step: 35440 loss: 16.5865 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 14.0304 +[titan] 2025-07-09 21:23:50,616 - root - INFO - lr: 2.2709e-04 gnorm: 0.80 [ 7:48:52<14:14:07] +[titan] 2025-07-09 21:23:54,566 - root - INFO - step: 35445 loss: 16.7406 memory: 44.58GiB(31.99%) tps: 82,958 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.1524 +[titan] 2025-07-09 21:23:54,567 - root - INFO - lr: 2.2707e-04 gnorm: 0.77 [ 7:48:55<14:14:03] +[titan] 2025-07-09 21:23:57,725 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:23:58,510 - root - INFO - step: 35450 loss: 16.7983 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.1965 +[titan] 2025-07-09 21:23:58,510 - root - INFO - lr: 2.2705e-04 gnorm: 0.75 [ 7:48:59<14:13:59] +[titan] 2025-07-09 21:24:02,467 - root - INFO - step: 35455 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0571 +[titan] 2025-07-09 21:24:02,467 - root - INFO - lr: 2.2703e-04 gnorm: 0.80 [ 7:49:03<14:13:55] +[titan] 2025-07-09 21:24:06,382 - root - INFO - step: 35460 loss: 16.6245 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5749 global_avg_mtp_loss: 14.0496 +[titan] 2025-07-09 21:24:06,383 - root - INFO - lr: 2.2701e-04 gnorm: 0.80 [ 7:49:07<14:13:51] +[titan] 2025-07-09 21:24:10,286 - root - INFO - step: 35465 loss: 16.5952 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0392 +[titan] 2025-07-09 21:24:10,287 - root - INFO - lr: 2.2699e-04 gnorm: 0.77 [ 7:49:11<14:13:47] +[titan] 2025-07-09 21:24:14,252 - root - INFO - step: 35470 loss: 16.7099 memory: 44.58GiB(31.99%) tps: 82,639 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.1311 +[titan] 2025-07-09 21:24:14,252 - root - INFO - lr: 2.2697e-04 gnorm: 0.82 [ 7:49:15<14:13:43] +[titan] 2025-07-09 21:24:18,148 - root - INFO - step: 35475 loss: 16.9790 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.6255 global_avg_mtp_loss: 14.3535 +[titan] 2025-07-09 21:24:18,149 - root - INFO - lr: 2.2695e-04 gnorm: 0.77 [ 7:49:19<14:13:39] +[titan] 2025-07-09 21:24:22,071 - root - INFO - step: 35480 loss: 16.7468 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1624 +[titan] 2025-07-09 21:24:22,071 - root - INFO - lr: 2.2693e-04 gnorm: 0.74 [ 7:49:23<14:13:34] +[titan] 2025-07-09 21:24:25,972 - root - INFO - step: 35485 loss: 16.8785 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.2796 +[titan] 2025-07-09 21:24:25,972 - root - INFO - lr: 2.2692e-04 gnorm: 0.76 [ 7:49:27<14:13:30] +[titan] 2025-07-09 21:24:29,903 - root - INFO - step: 35490 loss: 16.7335 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.1578 +[titan] 2025-07-09 21:24:29,903 - root - INFO - lr: 2.2690e-04 gnorm: 0.80 [ 7:49:31<14:13:26] +[titan] 2025-07-09 21:24:33,858 - root - INFO - step: 35495 loss: 16.4900 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9528 +[titan] 2025-07-09 21:24:33,858 - root - INFO - lr: 2.2688e-04 gnorm: 0.79 [ 7:49:35<14:13:22] +[titan] 2025-07-09 21:24:36,964 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:24:37,751 - root - INFO - step: 35500 loss: 16.9091 memory: 44.58GiB(31.99%) tps: 84,186 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.6222 global_avg_mtp_loss: 14.2869 +[titan] 2025-07-09 21:24:37,751 - root - INFO - lr: 2.2686e-04 gnorm: 0.79 [ 7:49:39<14:13:18] +[titan] 2025-07-09 21:24:41,664 - root - INFO - step: 35505 loss: 16.7677 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1784 +[titan] 2025-07-09 21:24:41,664 - root - INFO - lr: 2.2684e-04 gnorm: 0.77 [ 7:49:43<14:13:14] +[titan] 2025-07-09 21:24:45,578 - root - INFO - step: 35510 loss: 16.8086 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.2046 +[titan] 2025-07-09 21:24:45,578 - root - INFO - lr: 2.2682e-04 gnorm: 0.70 [ 7:49:46<14:13:10] +[titan] 2025-07-09 21:24:49,492 - root - INFO - step: 35515 loss: 16.8398 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2331 +[titan] 2025-07-09 21:24:49,492 - root - INFO - lr: 2.2680e-04 gnorm: 0.81 [ 7:49:50<14:13:06] +[titan] 2025-07-09 21:24:53,409 - root - INFO - step: 35520 loss: 17.1179 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.6866 global_avg_mtp_loss: 14.4313 +[titan] 2025-07-09 21:24:53,410 - root - INFO - lr: 2.2678e-04 gnorm: 0.80 [ 7:49:54<14:13:02] +[titan] 2025-07-09 21:24:57,338 - root - INFO - step: 35525 loss: 16.5134 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9784 +[titan] 2025-07-09 21:24:57,339 - root - INFO - lr: 2.2676e-04 gnorm: 0.83 [ 7:49:58<14:12:58] +[titan] 2025-07-09 21:25:01,248 - root - INFO - step: 35530 loss: 16.8085 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.2120 +[titan] 2025-07-09 21:25:01,248 - root - INFO - lr: 2.2674e-04 gnorm: 0.77 [ 7:50:02<14:12:54] +[titan] 2025-07-09 21:25:05,135 - root - INFO - step: 35535 loss: 16.5676 memory: 44.58GiB(31.99%) tps: 84,314 tflops: 290.98 mfu: 29.42% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0070 +[titan] 2025-07-09 21:25:05,135 - root - INFO - lr: 2.2672e-04 gnorm: 0.81 [ 7:50:06<14:12:50] +[titan] 2025-07-09 21:25:09,036 - root - INFO - step: 35540 loss: 16.8021 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.2161 +[titan] 2025-07-09 21:25:09,036 - root - INFO - lr: 2.2671e-04 gnorm: 0.74 [ 7:50:10<14:12:46] +[titan] 2025-07-09 21:25:12,952 - root - INFO - step: 35545 loss: 16.8161 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.2050 +[titan] 2025-07-09 21:25:12,953 - root - INFO - lr: 2.2669e-04 gnorm: 0.76 [ 7:50:14<14:12:42] +[titan] 2025-07-09 21:25:16,081 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:25:16,873 - root - INFO - step: 35550 loss: 16.8279 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.2310 +[titan] 2025-07-09 21:25:16,873 - root - INFO - lr: 2.2667e-04 gnorm: 0.78 [ 7:50:18<14:12:38] +[titan] 2025-07-09 21:25:20,779 - root - INFO - step: 35555 loss: 16.9634 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.6303 global_avg_mtp_loss: 14.3331 +[titan] 2025-07-09 21:25:20,779 - root - INFO - lr: 2.2665e-04 gnorm: 0.83 [ 7:50:22<14:12:33] +[titan] 2025-07-09 21:25:24,743 - root - INFO - step: 35560 loss: 16.5820 memory: 44.58GiB(31.99%) tps: 82,658 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0154 +[titan] 2025-07-09 21:25:24,744 - root - INFO - lr: 2.2663e-04 gnorm: 0.79 [ 7:50:26<14:12:29] +[titan] 2025-07-09 21:25:28,658 - root - INFO - step: 35565 loss: 16.6508 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0774 +[titan] 2025-07-09 21:25:28,658 - root - INFO - lr: 2.2661e-04 gnorm: 0.85 [ 7:50:30<14:12:25] +[titan] 2025-07-09 21:25:32,552 - root - INFO - step: 35570 loss: 16.7862 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5979 global_avg_mtp_loss: 14.1882 +[titan] 2025-07-09 21:25:32,552 - root - INFO - lr: 2.2659e-04 gnorm: 0.78 [ 7:50:33<14:12:21] +[titan] 2025-07-09 21:25:36,482 - root - INFO - step: 35575 loss: 16.8995 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6196 global_avg_mtp_loss: 14.2799 +[titan] 2025-07-09 21:25:36,482 - root - INFO - lr: 2.2657e-04 gnorm: 0.83 [ 7:50:37<14:12:17] +[titan] 2025-07-09 21:25:40,383 - root - INFO - step: 35580 loss: 16.5443 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5617 global_avg_mtp_loss: 13.9826 +[titan] 2025-07-09 21:25:40,383 - root - INFO - lr: 2.2655e-04 gnorm: 0.81 [ 7:50:41<14:12:13] +[titan] 2025-07-09 21:25:44,298 - root - INFO - step: 35585 loss: 16.7190 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5897 global_avg_mtp_loss: 14.1294 +[titan] 2025-07-09 21:25:44,298 - root - INFO - lr: 2.2653e-04 gnorm: 0.76 [ 7:50:45<14:12:09] +[titan] 2025-07-09 21:25:48,195 - root - INFO - step: 35590 loss: 16.6375 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0677 +[titan] 2025-07-09 21:25:48,196 - root - INFO - lr: 2.2652e-04 gnorm: 0.79 [ 7:50:49<14:12:05] +[titan] 2025-07-09 21:25:52,111 - root - INFO - step: 35595 loss: 16.6194 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0513 +[titan] 2025-07-09 21:25:52,111 - root - INFO - lr: 2.2650e-04 gnorm: 0.78 [ 7:50:53<14:12:01] +[titan] 2025-07-09 21:25:55,223 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:25:56,014 - root - INFO - step: 35600 loss: 16.8293 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.6105 global_avg_mtp_loss: 14.2188 +[titan] 2025-07-09 21:25:56,014 - root - INFO - lr: 2.2648e-04 gnorm: 0.79 [ 7:50:57<14:11:57] +[titan] 2025-07-09 21:25:59,947 - root - INFO - step: 35605 loss: 16.8472 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.6037 global_avg_mtp_loss: 14.2435 +[titan] 2025-07-09 21:25:59,947 - root - INFO - lr: 2.2646e-04 gnorm: 0.77 [ 7:51:01<14:11:53] +[titan] 2025-07-09 21:26:03,879 - root - INFO - step: 35610 loss: 16.9153 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6181 global_avg_mtp_loss: 14.2972 +[titan] 2025-07-09 21:26:03,879 - root - INFO - lr: 2.2644e-04 gnorm: 0.75 [ 7:51:05<14:11:49] +[titan] 2025-07-09 21:26:07,780 - root - INFO - step: 35615 loss: 16.7340 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5895 global_avg_mtp_loss: 14.1445 +[titan] 2025-07-09 21:26:07,781 - root - INFO - lr: 2.2642e-04 gnorm: 0.81 [ 7:51:09<14:11:45] +[titan] 2025-07-09 21:26:11,710 - root - INFO - step: 35620 loss: 16.4789 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9298 +[titan] 2025-07-09 21:26:11,710 - root - INFO - lr: 2.2640e-04 gnorm: 0.77 [ 7:51:13<14:11:41] +[titan] 2025-07-09 21:26:15,612 - root - INFO - step: 35625 loss: 16.6360 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.0572 +[titan] 2025-07-09 21:26:15,612 - root - INFO - lr: 2.2638e-04 gnorm: 0.77 [ 7:51:16<14:11:37] +[titan] 2025-07-09 21:26:19,513 - root - INFO - step: 35630 loss: 16.6148 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0508 +[titan] 2025-07-09 21:26:19,513 - root - INFO - lr: 2.2636e-04 gnorm: 0.75 [ 7:51:20<14:11:32] +[titan] 2025-07-09 21:26:23,432 - root - INFO - step: 35635 loss: 16.6683 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.0972 +[titan] 2025-07-09 21:26:23,432 - root - INFO - lr: 2.2634e-04 gnorm: 0.76 [ 7:51:24<14:11:28] +[titan] 2025-07-09 21:26:27,348 - root - INFO - step: 35640 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0758 +[titan] 2025-07-09 21:26:27,348 - root - INFO - lr: 2.2632e-04 gnorm: 0.85 [ 7:51:28<14:11:24] +[titan] 2025-07-09 21:26:31,286 - root - INFO - step: 35645 loss: 16.6804 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.0973 +[titan] 2025-07-09 21:26:31,286 - root - INFO - lr: 2.2631e-04 gnorm: 0.76 [ 7:51:32<14:11:20] +[titan] 2025-07-09 21:26:34,392 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:26:35,174 - root - INFO - step: 35650 loss: 16.6591 memory: 44.58GiB(31.99%) tps: 84,275 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.0877 +[titan] 2025-07-09 21:26:35,175 - root - INFO - lr: 2.2629e-04 gnorm: 0.77 [ 7:51:36<14:11:16] +[titan] 2025-07-09 21:26:39,076 - root - INFO - step: 35655 loss: 16.9433 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.3270 +[titan] 2025-07-09 21:26:39,077 - root - INFO - lr: 2.2627e-04 gnorm: 0.77 [ 7:51:40<14:11:12] +[titan] 2025-07-09 21:26:43,004 - root - INFO - step: 35660 loss: 16.8128 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2042 +[titan] 2025-07-09 21:26:43,004 - root - INFO - lr: 2.2625e-04 gnorm: 0.76 [ 7:51:44<14:11:08] +[titan] 2025-07-09 21:26:46,943 - root - INFO - step: 35665 loss: 16.7235 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1312 +[titan] 2025-07-09 21:26:46,943 - root - INFO - lr: 2.2623e-04 gnorm: 0.80 [ 7:51:48<14:11:04] +[titan] 2025-07-09 21:26:50,884 - root - INFO - step: 35670 loss: 16.7505 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5869 global_avg_mtp_loss: 14.1636 +[titan] 2025-07-09 21:26:50,885 - root - INFO - lr: 2.2621e-04 gnorm: 0.77 [ 7:51:52<14:11:00] +[titan] 2025-07-09 21:26:54,834 - root - INFO - step: 35675 loss: 16.6725 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.0883 +[titan] 2025-07-09 21:26:54,834 - root - INFO - lr: 2.2619e-04 gnorm: 0.94 [ 7:51:56<14:10:56] +[titan] 2025-07-09 21:26:58,771 - root - INFO - step: 35680 loss: 16.6787 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.1079 +[titan] 2025-07-09 21:26:58,771 - root - INFO - lr: 2.2617e-04 gnorm: 0.78 [ 7:52:00<14:10:52] +[titan] 2025-07-09 21:27:02,678 - root - INFO - step: 35685 loss: 16.6825 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.1073 +[titan] 2025-07-09 21:27:02,678 - root - INFO - lr: 2.2615e-04 gnorm: 0.76 [ 7:52:04<14:10:48] +[titan] 2025-07-09 21:27:06,583 - root - INFO - step: 35690 loss: 16.4252 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.8836 +[titan] 2025-07-09 21:27:06,583 - root - INFO - lr: 2.2613e-04 gnorm: 0.83 [ 7:52:07<14:10:44] +[titan] 2025-07-09 21:27:10,494 - root - INFO - step: 35695 loss: 16.7226 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5807 global_avg_mtp_loss: 14.1419 +[titan] 2025-07-09 21:27:10,494 - root - INFO - lr: 2.2611e-04 gnorm: 0.81 [ 7:52:11<14:10:40] +[titan] 2025-07-09 21:27:13,651 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:27:14,452 - root - INFO - step: 35700 loss: 16.6855 memory: 44.58GiB(31.99%) tps: 82,784 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1035 +[titan] 2025-07-09 21:27:14,453 - root - INFO - lr: 2.2610e-04 gnorm: 0.75 [ 7:52:15<14:10:36] +[titan] 2025-07-09 21:27:18,394 - root - INFO - step: 35705 loss: 16.5336 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 13.9799 +[titan] 2025-07-09 21:27:18,395 - root - INFO - lr: 2.2608e-04 gnorm: 0.73 [ 7:52:19<14:10:32] +[titan] 2025-07-09 21:27:22,351 - root - INFO - step: 35710 loss: 16.6945 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.1156 +[titan] 2025-07-09 21:27:22,352 - root - INFO - lr: 2.2606e-04 gnorm: 0.77 [ 7:52:23<14:10:28] +[titan] 2025-07-09 21:27:26,278 - root - INFO - step: 35715 loss: 16.6436 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0770 +[titan] 2025-07-09 21:27:26,278 - root - INFO - lr: 2.2604e-04 gnorm: 0.77 [ 7:52:27<14:10:24] +[titan] 2025-07-09 21:27:30,187 - root - INFO - step: 35720 loss: 16.6342 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.0619 +[titan] 2025-07-09 21:27:30,188 - root - INFO - lr: 2.2602e-04 gnorm: 0.75 [ 7:52:31<14:10:20] +[titan] 2025-07-09 21:27:34,086 - root - INFO - step: 35725 loss: 16.4660 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9174 +[titan] 2025-07-09 21:27:34,086 - root - INFO - lr: 2.2600e-04 gnorm: 0.77 [ 7:52:35<14:10:16] +[titan] 2025-07-09 21:27:37,983 - root - INFO - step: 35730 loss: 16.6243 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0545 +[titan] 2025-07-09 21:27:37,984 - root - INFO - lr: 2.2598e-04 gnorm: 0.74 [ 7:52:39<14:10:11] +[titan] 2025-07-09 21:27:41,907 - root - INFO - step: 35735 loss: 17.1383 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6677 global_avg_mtp_loss: 14.4706 +[titan] 2025-07-09 21:27:41,908 - root - INFO - lr: 2.2596e-04 gnorm: 0.72 [ 7:52:43<14:10:07] +[titan] 2025-07-09 21:27:45,829 - root - INFO - step: 35740 loss: 16.7906 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6105 global_avg_mtp_loss: 14.1802 +[titan] 2025-07-09 21:27:45,829 - root - INFO - lr: 2.2594e-04 gnorm: 0.76 [ 7:52:47<14:10:03] +[titan] 2025-07-09 21:27:49,778 - root - INFO - step: 35745 loss: 16.6246 memory: 44.58GiB(31.99%) tps: 82,969 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0428 +[titan] 2025-07-09 21:27:49,779 - root - INFO - lr: 2.2592e-04 gnorm: 0.78 [ 7:52:51<14:09:59] +[titan] 2025-07-09 21:27:52,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:27:53,713 - root - INFO - step: 35750 loss: 16.7176 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1367 +[titan] 2025-07-09 21:27:53,713 - root - INFO - lr: 2.2590e-04 gnorm: 0.80 [ 7:52:55<14:09:55] +[titan] 2025-07-09 21:27:57,633 - root - INFO - step: 35755 loss: 16.8816 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.2703 +[titan] 2025-07-09 21:27:57,633 - root - INFO - lr: 2.2589e-04 gnorm: 0.73 [ 7:52:59<14:09:51] +[titan] 2025-07-09 21:28:01,533 - root - INFO - step: 35760 loss: 16.6643 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0934 +[titan] 2025-07-09 21:28:01,533 - root - INFO - lr: 2.2587e-04 gnorm: 0.72 [ 7:53:02<14:09:47] +[titan] 2025-07-09 21:28:05,494 - root - INFO - step: 35765 loss: 16.8051 memory: 44.58GiB(31.99%) tps: 82,734 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.2192 +[titan] 2025-07-09 21:28:05,494 - root - INFO - lr: 2.2585e-04 gnorm: 0.76 [ 7:53:06<14:09:43] +[titan] 2025-07-09 21:28:09,426 - root - INFO - step: 35770 loss: 16.6979 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5807 global_avg_mtp_loss: 14.1172 +[titan] 2025-07-09 21:28:09,426 - root - INFO - lr: 2.2583e-04 gnorm: 0.75 [ 7:53:10<14:09:39] +[titan] 2025-07-09 21:28:13,353 - root - INFO - step: 35775 loss: 16.5667 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5559 global_avg_mtp_loss: 14.0109 +[titan] 2025-07-09 21:28:13,354 - root - INFO - lr: 2.2581e-04 gnorm: 0.85 [ 7:53:14<14:09:35] +[titan] 2025-07-09 21:28:17,278 - root - INFO - step: 35780 loss: 16.5021 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 13.9523 +[titan] 2025-07-09 21:28:17,279 - root - INFO - lr: 2.2579e-04 gnorm: 0.89 [ 7:53:18<14:09:31] +[titan] 2025-07-09 21:28:21,173 - root - INFO - step: 35785 loss: 16.5239 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9780 +[titan] 2025-07-09 21:28:21,173 - root - INFO - lr: 2.2577e-04 gnorm: 0.89 [ 7:53:22<14:09:27] +[titan] 2025-07-09 21:28:25,063 - root - INFO - step: 35790 loss: 16.9508 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.3165 +[titan] 2025-07-09 21:28:25,063 - root - INFO - lr: 2.2575e-04 gnorm: 0.84 [ 7:53:26<14:09:23] +[titan] 2025-07-09 21:28:28,998 - root - INFO - step: 35795 loss: 16.7107 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 21:28:28,998 - root - INFO - lr: 2.2573e-04 gnorm: 0.75 [ 7:53:30<14:09:19] +[titan] 2025-07-09 21:28:32,134 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:28:32,930 - root - INFO - step: 35800 loss: 16.5187 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 13.9630 +[titan] 2025-07-09 21:28:32,930 - root - INFO - lr: 2.2571e-04 gnorm: 0.82 [ 7:53:34<14:09:15] +[titan] 2025-07-09 21:28:36,858 - root - INFO - step: 35805 loss: 16.1163 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.6221 +[titan] 2025-07-09 21:28:36,858 - root - INFO - lr: 2.2569e-04 gnorm: 1.03 [ 7:53:38<14:09:11] +[titan] 2025-07-09 21:28:40,793 - root - INFO - step: 35810 loss: 16.5643 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-09 21:28:40,794 - root - INFO - lr: 2.2567e-04 gnorm: 0.74 [ 7:53:42<14:09:07] +[titan] 2025-07-09 21:28:44,700 - root - INFO - step: 35815 loss: 16.7538 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5825 global_avg_mtp_loss: 14.1712 +[titan] 2025-07-09 21:28:44,700 - root - INFO - lr: 2.2566e-04 gnorm: 0.77 [ 7:53:46<14:09:03] +[titan] 2025-07-09 21:28:48,618 - root - INFO - step: 35820 loss: 16.8342 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.2333 +[titan] 2025-07-09 21:28:48,618 - root - INFO - lr: 2.2564e-04 gnorm: 0.76 [ 7:53:49<14:08:59] +[titan] 2025-07-09 21:28:52,549 - root - INFO - step: 35825 loss: 16.8545 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2409 +[titan] 2025-07-09 21:28:52,549 - root - INFO - lr: 2.2562e-04 gnorm: 0.77 [ 7:53:53<14:08:54] +[titan] 2025-07-09 21:28:56,502 - root - INFO - step: 35830 loss: 16.7073 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5856 global_avg_mtp_loss: 14.1216 +[titan] 2025-07-09 21:28:56,502 - root - INFO - lr: 2.2560e-04 gnorm: 0.72 [ 7:53:57<14:08:51] +[titan] 2025-07-09 21:29:00,424 - root - INFO - step: 35835 loss: 16.5357 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9783 +[titan] 2025-07-09 21:29:00,424 - root - INFO - lr: 2.2558e-04 gnorm: 0.73 [ 7:54:01<14:08:46] +[titan] 2025-07-09 21:29:04,442 - root - INFO - step: 35840 loss: 16.7737 memory: 44.58GiB(31.99%) tps: 81,549 tflops: 281.44 mfu: 28.46% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.1730 +[titan] 2025-07-09 21:29:04,443 - root - INFO - lr: 2.2556e-04 gnorm: 0.89 [ 7:54:05<14:08:43] +[titan] 2025-07-09 21:29:04,598 - root - INFO - Dumping profiler traces at step 35840 +[titan] 2025-07-09 21:29:04,631 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:29:08,539 - root - INFO - step: 35845 loss: 16.5664 memory: 44.58GiB(31.99%) tps: 80,004 tflops: 276.11 mfu: 27.92% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 13.9956 +[titan] 2025-07-09 21:29:08,539 - root - INFO - lr: 2.2554e-04 gnorm: 0.73 [ 7:54:09<14:08:39] +[titan] 2025-07-09 21:29:11,648 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:29:12,455 - root - INFO - step: 35850 loss: 16.7591 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5927 global_avg_mtp_loss: 14.1663 +[titan] 2025-07-09 21:29:12,455 - root - INFO - lr: 2.2552e-04 gnorm: 0.78 [ 7:54:13<14:08:35] +[titan] 2025-07-09 21:29:16,369 - root - INFO - step: 35855 loss: 16.8052 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.2000 +[titan] 2025-07-09 21:29:16,370 - root - INFO - lr: 2.2550e-04 gnorm: 0.76 [ 7:54:17<14:08:31] +[titan] 2025-07-09 21:29:20,294 - root - INFO - step: 35860 loss: 16.6996 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1163 +[titan] 2025-07-09 21:29:20,294 - root - INFO - lr: 2.2548e-04 gnorm: 0.80 [ 7:54:21<14:08:27] +[titan] 2025-07-09 21:29:24,188 - root - INFO - step: 35865 loss: 16.5323 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5543 global_avg_mtp_loss: 13.9780 +[titan] 2025-07-09 21:29:24,188 - root - INFO - lr: 2.2546e-04 gnorm: 0.78 [ 7:54:25<14:08:23] +[titan] 2025-07-09 21:29:28,106 - root - INFO - step: 35870 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 13.9553 +[titan] 2025-07-09 21:29:28,106 - root - INFO - lr: 2.2544e-04 gnorm: 0.75 [ 7:54:29<14:08:18] +[titan] 2025-07-09 21:29:32,030 - root - INFO - step: 35875 loss: 16.8993 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.6286 global_avg_mtp_loss: 14.2707 +[titan] 2025-07-09 21:29:32,031 - root - INFO - lr: 2.2543e-04 gnorm: 0.72 [ 7:54:33<14:08:14] +[titan] 2025-07-09 21:29:35,987 - root - INFO - step: 35880 loss: 16.6482 memory: 44.58GiB(31.99%) tps: 82,828 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0770 +[titan] 2025-07-09 21:29:35,987 - root - INFO - lr: 2.2541e-04 gnorm: 0.76 [ 7:54:37<14:08:10] +[titan] 2025-07-09 21:29:39,886 - root - INFO - step: 35885 loss: 16.9070 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.6302 global_avg_mtp_loss: 14.2768 +[titan] 2025-07-09 21:29:39,886 - root - INFO - lr: 2.2539e-04 gnorm: 0.80 [ 7:54:41<14:08:06] +[titan] 2025-07-09 21:29:43,818 - root - INFO - step: 35890 loss: 16.6895 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.1132 +[titan] 2025-07-09 21:29:43,819 - root - INFO - lr: 2.2537e-04 gnorm: 0.82 [ 7:54:45<14:08:02] +[titan] 2025-07-09 21:29:47,758 - root - INFO - step: 35895 loss: 16.6432 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0702 +[titan] 2025-07-09 21:29:47,758 - root - INFO - lr: 2.2535e-04 gnorm: 0.82 [ 7:54:49<14:07:58] +[titan] 2025-07-09 21:29:50,878 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:29:51,665 - root - INFO - step: 35900 loss: 16.5467 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 13.9846 +[titan] 2025-07-09 21:29:51,665 - root - INFO - lr: 2.2533e-04 gnorm: 0.74 [ 7:54:53<14:07:54] +[titan] 2025-07-09 21:29:55,576 - root - INFO - step: 35905 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1699 +[titan] 2025-07-09 21:29:55,577 - root - INFO - lr: 2.2531e-04 gnorm: 0.75 [ 7:54:56<14:07:50] +[titan] 2025-07-09 21:29:59,520 - root - INFO - step: 35910 loss: 16.8642 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6183 global_avg_mtp_loss: 14.2459 +[titan] 2025-07-09 21:29:59,520 - root - INFO - lr: 2.2529e-04 gnorm: 0.79 [ 7:55:00<14:07:46] +[titan] 2025-07-09 21:30:03,474 - root - INFO - step: 35915 loss: 16.7475 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.1614 +[titan] 2025-07-09 21:30:03,475 - root - INFO - lr: 2.2527e-04 gnorm: 0.77 [ 7:55:04<14:07:42] +[titan] 2025-07-09 21:30:07,392 - root - INFO - step: 35920 loss: 16.9623 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6541 global_avg_mtp_loss: 14.3083 +[titan] 2025-07-09 21:30:07,393 - root - INFO - lr: 2.2525e-04 gnorm: 0.96 [ 7:55:08<14:07:38] +[titan] 2025-07-09 21:30:11,322 - root - INFO - step: 35925 loss: 16.5433 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 13.9841 +[titan] 2025-07-09 21:30:11,323 - root - INFO - lr: 2.2523e-04 gnorm: 0.75 [ 7:55:12<14:07:34] +[titan] 2025-07-09 21:30:15,266 - root - INFO - step: 35930 loss: 16.7422 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.1564 +[titan] 2025-07-09 21:30:15,267 - root - INFO - lr: 2.2522e-04 gnorm: 0.78 [ 7:55:16<14:07:30] +[titan] 2025-07-09 21:30:19,165 - root - INFO - step: 35935 loss: 16.7935 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5932 global_avg_mtp_loss: 14.2003 +[titan] 2025-07-09 21:30:19,166 - root - INFO - lr: 2.2520e-04 gnorm: 0.80 [ 7:55:20<14:07:26] +[titan] 2025-07-09 21:30:23,066 - root - INFO - step: 35940 loss: 16.7521 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.1583 +[titan] 2025-07-09 21:30:23,066 - root - INFO - lr: 2.2518e-04 gnorm: 1.09 [ 7:55:24<14:07:22] +[titan] 2025-07-09 21:30:26,993 - root - INFO - step: 35945 loss: 16.6542 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0870 +[titan] 2025-07-09 21:30:26,993 - root - INFO - lr: 2.2516e-04 gnorm: 0.75 [ 7:55:28<14:07:18] +[titan] 2025-07-09 21:30:30,128 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:30:30,918 - root - INFO - step: 35950 loss: 16.4066 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.8803 +[titan] 2025-07-09 21:30:30,919 - root - INFO - lr: 2.2514e-04 gnorm: 0.76 [ 7:55:32<14:07:14] +[titan] 2025-07-09 21:30:34,851 - root - INFO - step: 35955 loss: 16.4187 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.8908 +[titan] 2025-07-09 21:30:34,851 - root - INFO - lr: 2.2512e-04 gnorm: 0.73 [ 7:55:36<14:07:10] +[titan] 2025-07-09 21:30:38,787 - root - INFO - step: 35960 loss: 16.7368 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5909 global_avg_mtp_loss: 14.1459 +[titan] 2025-07-09 21:30:38,787 - root - INFO - lr: 2.2510e-04 gnorm: 0.75 [ 7:55:40<14:07:06] +[titan] 2025-07-09 21:30:42,687 - root - INFO - step: 35965 loss: 16.3944 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8723 +[titan] 2025-07-09 21:30:42,687 - root - INFO - lr: 2.2508e-04 gnorm: 0.75 [ 7:55:44<14:07:02] +[titan] 2025-07-09 21:30:46,609 - root - INFO - step: 35970 loss: 16.6222 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0538 +[titan] 2025-07-09 21:30:46,610 - root - INFO - lr: 2.2506e-04 gnorm: 0.78 [ 7:55:47<14:06:58] +[titan] 2025-07-09 21:30:50,508 - root - INFO - step: 35975 loss: 16.7413 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1450 +[titan] 2025-07-09 21:30:50,509 - root - INFO - lr: 2.2504e-04 gnorm: 0.76 [ 7:55:51<14:06:53] +[titan] 2025-07-09 21:30:54,440 - root - INFO - step: 35980 loss: 16.7788 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5841 global_avg_mtp_loss: 14.1947 +[titan] 2025-07-09 21:30:54,440 - root - INFO - lr: 2.2502e-04 gnorm: 0.75 [ 7:55:55<14:06:49] +[titan] 2025-07-09 21:30:58,417 - root - INFO - step: 35985 loss: 16.6405 memory: 44.58GiB(31.99%) tps: 82,390 tflops: 284.34 mfu: 28.75% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0710 +[titan] 2025-07-09 21:30:58,418 - root - INFO - lr: 2.2500e-04 gnorm: 0.83 [ 7:55:59<14:06:45] +[titan] 2025-07-09 21:31:02,340 - root - INFO - step: 35990 loss: 16.7814 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5981 global_avg_mtp_loss: 14.1833 +[titan] 2025-07-09 21:31:02,341 - root - INFO - lr: 2.2498e-04 gnorm: 0.75 [ 7:56:03<14:06:41] +[titan] 2025-07-09 21:31:06,258 - root - INFO - step: 35995 loss: 16.2375 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-09 21:31:06,259 - root - INFO - lr: 2.2497e-04 gnorm: 0.76 [ 7:56:07<14:06:37] +[titan] 2025-07-09 21:31:09,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:31:10,156 - root - INFO - step: 36000 loss: 16.6963 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.1165 +[titan] 2025-07-09 21:31:10,156 - root - INFO - lr: 2.2495e-04 gnorm: 0.78 [ 7:56:11<14:06:33] +[titan] 2025-07-09 21:31:14,087 - root - INFO - step: 36005 loss: 16.5593 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5631 global_avg_mtp_loss: 13.9961 +[titan] 2025-07-09 21:31:14,087 - root - INFO - lr: 2.2493e-04 gnorm: 0.71 [ 7:56:15<14:06:29] +[titan] 2025-07-09 21:31:18,003 - root - INFO - step: 36010 loss: 16.2355 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.7244 +[titan] 2025-07-09 21:31:18,003 - root - INFO - lr: 2.2491e-04 gnorm: 0.74 [ 7:56:19<14:06:25] +[titan] 2025-07-09 21:31:21,926 - root - INFO - step: 36015 loss: 16.6725 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5829 global_avg_mtp_loss: 14.0897 +[titan] 2025-07-09 21:31:21,926 - root - INFO - lr: 2.2489e-04 gnorm: 0.74 [ 7:56:23<14:06:21] +[titan] 2025-07-09 21:31:25,830 - root - INFO - step: 36020 loss: 16.7708 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5868 global_avg_mtp_loss: 14.1840 +[titan] 2025-07-09 21:31:25,831 - root - INFO - lr: 2.2487e-04 gnorm: 0.75 [ 7:56:27<14:06:17] +[titan] 2025-07-09 21:31:29,756 - root - INFO - step: 36025 loss: 16.8515 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.6129 global_avg_mtp_loss: 14.2386 +[titan] 2025-07-09 21:31:29,756 - root - INFO - lr: 2.2485e-04 gnorm: 0.89 [ 7:56:31<14:06:13] +[titan] 2025-07-09 21:31:33,671 - root - INFO - step: 36030 loss: 16.6873 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.1057 +[titan] 2025-07-09 21:31:33,671 - root - INFO - lr: 2.2483e-04 gnorm: 0.80 [ 7:56:35<14:06:09] +[titan] 2025-07-09 21:31:37,584 - root - INFO - step: 36035 loss: 16.4956 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9438 +[titan] 2025-07-09 21:31:37,584 - root - INFO - lr: 2.2481e-04 gnorm: 0.76 [ 7:56:38<14:06:05] +[titan] 2025-07-09 21:31:41,481 - root - INFO - step: 36040 loss: 16.7478 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5823 global_avg_mtp_loss: 14.1655 +[titan] 2025-07-09 21:31:41,482 - root - INFO - lr: 2.2479e-04 gnorm: 0.77 [ 7:56:42<14:06:01] +[titan] 2025-07-09 21:31:45,396 - root - INFO - step: 36045 loss: 16.6499 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0787 +[titan] 2025-07-09 21:31:45,397 - root - INFO - lr: 2.2477e-04 gnorm: 0.80 [ 7:56:46<14:05:57] +[titan] 2025-07-09 21:31:48,569 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:31:49,356 - root - INFO - step: 36050 loss: 16.5589 memory: 44.58GiB(31.99%) tps: 82,772 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 13.9913 +[titan] 2025-07-09 21:31:49,356 - root - INFO - lr: 2.2475e-04 gnorm: 0.79 [ 7:56:50<14:05:53] +[titan] 2025-07-09 21:31:53,270 - root - INFO - step: 36055 loss: 16.6764 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5808 global_avg_mtp_loss: 14.0956 +[titan] 2025-07-09 21:31:53,270 - root - INFO - lr: 2.2474e-04 gnorm: 0.82 [ 7:56:54<14:05:49] +[titan] 2025-07-09 21:31:57,177 - root - INFO - step: 36060 loss: 16.8613 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2335 +[titan] 2025-07-09 21:31:57,177 - root - INFO - lr: 2.2472e-04 gnorm: 0.77 [ 7:56:58<14:05:45] +[titan] 2025-07-09 21:32:01,128 - root - INFO - step: 36065 loss: 16.5571 memory: 44.58GiB(31.99%) tps: 82,941 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0024 +[titan] 2025-07-09 21:32:01,128 - root - INFO - lr: 2.2470e-04 gnorm: 0.79 [ 7:57:02<14:05:41] +[titan] 2025-07-09 21:32:05,079 - root - INFO - step: 36070 loss: 16.8803 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.6384 global_avg_mtp_loss: 14.2419 +[titan] 2025-07-09 21:32:05,079 - root - INFO - lr: 2.2468e-04 gnorm: 0.79 [ 7:57:06<14:05:37] +[titan] 2025-07-09 21:32:09,011 - root - INFO - step: 36075 loss: 16.8554 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.6132 global_avg_mtp_loss: 14.2422 +[titan] 2025-07-09 21:32:09,011 - root - INFO - lr: 2.2466e-04 gnorm: 0.78 [ 7:57:10<14:05:33] +[titan] 2025-07-09 21:32:12,966 - root - INFO - step: 36080 loss: 16.7191 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1316 +[titan] 2025-07-09 21:32:12,966 - root - INFO - lr: 2.2464e-04 gnorm: 0.78 [ 7:57:14<14:05:29] +[titan] 2025-07-09 21:32:16,884 - root - INFO - step: 36085 loss: 16.9554 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6361 global_avg_mtp_loss: 14.3193 +[titan] 2025-07-09 21:32:16,884 - root - INFO - lr: 2.2462e-04 gnorm: 0.76 [ 7:57:18<14:05:25] +[titan] 2025-07-09 21:32:20,850 - root - INFO - step: 36090 loss: 16.6092 memory: 44.58GiB(31.99%) tps: 82,641 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.5701 global_avg_mtp_loss: 14.0391 +[titan] 2025-07-09 21:32:20,850 - root - INFO - lr: 2.2460e-04 gnorm: 0.77 [ 7:57:22<14:05:21] +[titan] 2025-07-09 21:32:24,769 - root - INFO - step: 36095 loss: 16.4755 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-09 21:32:24,769 - root - INFO - lr: 2.2458e-04 gnorm: 0.74 [ 7:57:26<14:05:16] +[titan] 2025-07-09 21:32:27,907 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:32:28,697 - root - INFO - step: 36100 loss: 16.6988 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1098 +[titan] 2025-07-09 21:32:28,698 - root - INFO - lr: 2.2456e-04 gnorm: 0.75 [ 7:57:30<14:05:12] +[titan] 2025-07-09 21:32:32,641 - root - INFO - step: 36105 loss: 16.6110 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0483 +[titan] 2025-07-09 21:32:32,641 - root - INFO - lr: 2.2454e-04 gnorm: 0.77 [ 7:57:33<14:05:08] +[titan] 2025-07-09 21:32:36,561 - root - INFO - step: 36110 loss: 16.7866 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.1681 +[titan] 2025-07-09 21:32:36,561 - root - INFO - lr: 2.2452e-04 gnorm: 0.78 [ 7:57:37<14:05:04] +[titan] 2025-07-09 21:32:40,504 - root - INFO - step: 36115 loss: 16.8917 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6086 global_avg_mtp_loss: 14.2831 +[titan] 2025-07-09 21:32:40,504 - root - INFO - lr: 2.2450e-04 gnorm: 0.79 [ 7:57:41<14:05:00] +[titan] 2025-07-09 21:32:44,430 - root - INFO - step: 36120 loss: 16.7806 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6072 global_avg_mtp_loss: 14.1734 +[titan] 2025-07-09 21:32:44,431 - root - INFO - lr: 2.2449e-04 gnorm: 0.74 [ 7:57:45<14:04:56] +[titan] 2025-07-09 21:32:48,327 - root - INFO - step: 36125 loss: 16.8416 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.2336 +[titan] 2025-07-09 21:32:48,328 - root - INFO - lr: 2.2447e-04 gnorm: 0.75 [ 7:57:49<14:04:52] +[titan] 2025-07-09 21:32:52,240 - root - INFO - step: 36130 loss: 16.7954 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.1906 +[titan] 2025-07-09 21:32:52,240 - root - INFO - lr: 2.2445e-04 gnorm: 0.72 [ 7:57:53<14:04:48] +[titan] 2025-07-09 21:32:56,171 - root - INFO - step: 36135 loss: 16.6525 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.0940 +[titan] 2025-07-09 21:32:56,171 - root - INFO - lr: 2.2443e-04 gnorm: 0.79 [ 7:57:57<14:04:44] +[titan] 2025-07-09 21:33:00,086 - root - INFO - step: 36140 loss: 16.8000 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.2052 +[titan] 2025-07-09 21:33:00,086 - root - INFO - lr: 2.2441e-04 gnorm: 1.68 [ 7:58:01<14:04:40] +[titan] 2025-07-09 21:33:04,032 - root - INFO - step: 36145 loss: 16.4017 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5342 global_avg_mtp_loss: 13.8675 +[titan] 2025-07-09 21:33:04,032 - root - INFO - lr: 2.2439e-04 gnorm: 0.75 [ 7:58:05<14:04:36] +[titan] 2025-07-09 21:33:07,195 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:33:07,990 - root - INFO - step: 36150 loss: 16.5497 memory: 44.58GiB(31.99%) tps: 82,790 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 13.9705 +[titan] 2025-07-09 21:33:07,990 - root - INFO - lr: 2.2437e-04 gnorm: 0.78 [ 7:58:09<14:04:32] +[titan] 2025-07-09 21:33:11,891 - root - INFO - step: 36155 loss: 16.6631 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.0842 +[titan] 2025-07-09 21:33:11,891 - root - INFO - lr: 2.2435e-04 gnorm: 0.85 [ 7:58:13<14:04:28] +[titan] 2025-07-09 21:33:15,851 - root - INFO - step: 36160 loss: 16.5457 memory: 44.58GiB(31.99%) tps: 82,750 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 14.0084 +[titan] 2025-07-09 21:33:15,851 - root - INFO - lr: 2.2433e-04 gnorm: 0.74 [ 7:58:17<14:04:24] +[titan] 2025-07-09 21:33:19,776 - root - INFO - step: 36165 loss: 16.4225 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.8960 +[titan] 2025-07-09 21:33:19,776 - root - INFO - lr: 2.2431e-04 gnorm: 0.73 [ 7:58:21<14:04:20] +[titan] 2025-07-09 21:33:23,740 - root - INFO - step: 36170 loss: 16.5705 memory: 44.58GiB(31.99%) tps: 82,674 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0184 +[titan] 2025-07-09 21:33:23,740 - root - INFO - lr: 2.2429e-04 gnorm: 0.80 [ 7:58:25<14:04:16] +[titan] 2025-07-09 21:33:27,696 - root - INFO - step: 36175 loss: 16.5885 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.0150 +[titan] 2025-07-09 21:33:27,697 - root - INFO - lr: 2.2427e-04 gnorm: 0.79 [ 7:58:29<14:04:12] +[titan] 2025-07-09 21:33:31,676 - root - INFO - step: 36180 loss: 16.7295 memory: 44.58GiB(31.99%) tps: 82,349 tflops: 284.20 mfu: 28.74% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.1471 +[titan] 2025-07-09 21:33:31,676 - root - INFO - lr: 2.2425e-04 gnorm: 0.78 [ 7:58:33<14:04:08] +[titan] 2025-07-09 21:33:35,594 - root - INFO - step: 36185 loss: 16.6488 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0867 +[titan] 2025-07-09 21:33:35,595 - root - INFO - lr: 2.2424e-04 gnorm: 0.72 [ 7:58:36<14:04:04] +[titan] 2025-07-09 21:33:39,543 - root - INFO - step: 36190 loss: 16.6311 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5785 global_avg_mtp_loss: 14.0526 +[titan] 2025-07-09 21:33:39,544 - root - INFO - lr: 2.2422e-04 gnorm: 0.77 [ 7:58:40<14:04:00] +[titan] 2025-07-09 21:33:43,498 - root - INFO - step: 36195 loss: 16.4183 memory: 44.58GiB(31.99%) tps: 82,870 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.8934 +[titan] 2025-07-09 21:33:43,498 - root - INFO - lr: 2.2420e-04 gnorm: 0.78 [ 7:58:44<14:03:56] +[titan] 2025-07-09 21:33:46,628 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:33:47,423 - root - INFO - step: 36200 loss: 16.7446 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.6036 global_avg_mtp_loss: 14.1409 +[titan] 2025-07-09 21:33:47,423 - root - INFO - lr: 2.2418e-04 gnorm: 0.79 [ 7:58:48<14:03:52] +[titan] 2025-07-09 21:33:51,329 - root - INFO - step: 36205 loss: 16.7411 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1464 +[titan] 2025-07-09 21:33:51,329 - root - INFO - lr: 2.2416e-04 gnorm: 0.81 [ 7:58:52<14:03:48] +[titan] 2025-07-09 21:33:55,257 - root - INFO - step: 36210 loss: 16.9119 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.2994 +[titan] 2025-07-09 21:33:55,258 - root - INFO - lr: 2.2414e-04 gnorm: 0.73 [ 7:58:56<14:03:44] +[titan] 2025-07-09 21:33:59,204 - root - INFO - step: 36215 loss: 16.7372 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.1622 +[titan] 2025-07-09 21:33:59,204 - root - INFO - lr: 2.2412e-04 gnorm: 0.78 [ 7:59:00<14:03:40] +[titan] 2025-07-09 21:34:03,143 - root - INFO - step: 36220 loss: 16.4162 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.8885 +[titan] 2025-07-09 21:34:03,143 - root - INFO - lr: 2.2410e-04 gnorm: 0.77 [ 7:59:04<14:03:36] +[titan] 2025-07-09 21:34:07,092 - root - INFO - step: 36225 loss: 16.7647 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.6135 global_avg_mtp_loss: 14.1512 +[titan] 2025-07-09 21:34:07,092 - root - INFO - lr: 2.2408e-04 gnorm: 0.79 [ 7:59:08<14:03:32] +[titan] 2025-07-09 21:34:11,012 - root - INFO - step: 36230 loss: 16.7168 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 21:34:11,013 - root - INFO - lr: 2.2406e-04 gnorm: 0.75 [ 7:59:12<14:03:28] +[titan] 2025-07-09 21:34:14,949 - root - INFO - step: 36235 loss: 16.6645 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5696 global_avg_mtp_loss: 14.0949 +[titan] 2025-07-09 21:34:14,950 - root - INFO - lr: 2.2404e-04 gnorm: 0.80 [ 7:59:16<14:03:24] +[titan] 2025-07-09 21:34:18,899 - root - INFO - step: 36240 loss: 16.9082 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.6231 global_avg_mtp_loss: 14.2851 +[titan] 2025-07-09 21:34:18,899 - root - INFO - lr: 2.2402e-04 gnorm: 0.78 [ 7:59:20<14:03:20] +[titan] 2025-07-09 21:34:22,794 - root - INFO - step: 36245 loss: 16.7771 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.1727 +[titan] 2025-07-09 21:34:22,794 - root - INFO - lr: 2.2400e-04 gnorm: 0.84 [ 7:59:24<14:03:16] +[titan] 2025-07-09 21:34:25,912 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:34:26,712 - root - INFO - step: 36250 loss: 16.7821 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.1760 +[titan] 2025-07-09 21:34:26,712 - root - INFO - lr: 2.2399e-04 gnorm: 0.77 [ 7:59:28<14:03:12] +[titan] 2025-07-09 21:34:30,625 - root - INFO - step: 36255 loss: 16.8798 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.2522 +[titan] 2025-07-09 21:34:30,626 - root - INFO - lr: 2.2397e-04 gnorm: 0.75 [ 7:59:31<14:03:07] +[titan] 2025-07-09 21:34:34,597 - root - INFO - step: 36260 loss: 16.6046 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0364 +[titan] 2025-07-09 21:34:34,598 - root - INFO - lr: 2.2395e-04 gnorm: 0.73 [ 7:59:35<14:03:04] +[titan] 2025-07-09 21:34:38,580 - root - INFO - step: 36265 loss: 16.6893 memory: 44.58GiB(31.99%) tps: 82,294 tflops: 284.01 mfu: 28.72% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.1092 +[titan] 2025-07-09 21:34:38,580 - root - INFO - lr: 2.2393e-04 gnorm: 0.80 [ 7:59:39<14:03:00] +[titan] 2025-07-09 21:34:42,567 - root - INFO - step: 36270 loss: 16.7919 memory: 44.58GiB(31.99%) tps: 82,182 tflops: 283.62 mfu: 28.68% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.1896 +[titan] 2025-07-09 21:34:42,567 - root - INFO - lr: 2.2391e-04 gnorm: 0.81 [ 7:59:43<14:02:56] +[titan] 2025-07-09 21:34:46,568 - root - INFO - step: 36275 loss: 16.6405 memory: 44.58GiB(31.99%) tps: 81,909 tflops: 282.68 mfu: 28.58% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0840 +[titan] 2025-07-09 21:34:46,568 - root - INFO - lr: 2.2389e-04 gnorm: 0.75 [ 7:59:47<14:02:52] +[titan] 2025-07-09 21:34:50,497 - root - INFO - step: 36280 loss: 16.3916 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.8505 +[titan] 2025-07-09 21:34:50,498 - root - INFO - lr: 2.2387e-04 gnorm: 0.78 [ 7:59:51<14:02:48] +[titan] 2025-07-09 21:34:54,400 - root - INFO - step: 36285 loss: 16.6705 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.1049 +[titan] 2025-07-09 21:34:54,400 - root - INFO - lr: 2.2385e-04 gnorm: 0.76 [ 7:59:55<14:02:44] +[titan] 2025-07-09 21:34:58,301 - root - INFO - step: 36290 loss: 16.8556 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.2404 +[titan] 2025-07-09 21:34:58,301 - root - INFO - lr: 2.2383e-04 gnorm: 0.80 [ 7:59:59<14:02:40] +[titan] 2025-07-09 21:35:02,197 - root - INFO - step: 36295 loss: 16.3815 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.8510 +[titan] 2025-07-09 21:35:02,197 - root - INFO - lr: 2.2381e-04 gnorm: 0.76 [ 8:00:03<14:02:35] +[titan] 2025-07-09 21:35:05,333 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:35:06,118 - root - INFO - step: 36300 loss: 16.6335 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0693 +[titan] 2025-07-09 21:35:06,118 - root - INFO - lr: 2.2379e-04 gnorm: 0.78 [ 8:00:07<14:02:31] +[titan] 2025-07-09 21:35:10,042 - root - INFO - step: 36305 loss: 16.6755 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.0867 +[titan] 2025-07-09 21:35:10,043 - root - INFO - lr: 2.2377e-04 gnorm: 0.76 [ 8:00:11<14:02:27] +[titan] 2025-07-09 21:35:13,956 - root - INFO - step: 36310 loss: 16.7434 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5973 global_avg_mtp_loss: 14.1461 +[titan] 2025-07-09 21:35:13,957 - root - INFO - lr: 2.2375e-04 gnorm: 0.77 [ 8:00:15<14:02:23] +[titan] 2025-07-09 21:35:17,887 - root - INFO - step: 36315 loss: 16.7693 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5976 global_avg_mtp_loss: 14.1717 +[titan] 2025-07-09 21:35:17,888 - root - INFO - lr: 2.2373e-04 gnorm: 0.78 [ 8:00:19<14:02:19] +[titan] 2025-07-09 21:35:21,861 - root - INFO - step: 36320 loss: 16.5818 memory: 44.58GiB(31.99%) tps: 82,472 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.5657 global_avg_mtp_loss: 14.0161 +[titan] 2025-07-09 21:35:21,861 - root - INFO - lr: 2.2372e-04 gnorm: 0.85 [ 8:00:23<14:02:15] +[titan] 2025-07-09 21:35:25,776 - root - INFO - step: 36325 loss: 16.6877 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1115 +[titan] 2025-07-09 21:35:25,776 - root - INFO - lr: 2.2370e-04 gnorm: 0.81 [ 8:00:27<14:02:11] +[titan] 2025-07-09 21:35:29,693 - root - INFO - step: 36330 loss: 16.6633 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.0934 +[titan] 2025-07-09 21:35:29,694 - root - INFO - lr: 2.2368e-04 gnorm: 0.79 [ 8:00:31<14:02:07] +[titan] 2025-07-09 21:35:33,629 - root - INFO - step: 36335 loss: 16.6590 memory: 44.58GiB(31.99%) tps: 83,277 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0925 +[titan] 2025-07-09 21:35:33,629 - root - INFO - lr: 2.2366e-04 gnorm: 0.83 [ 8:00:34<14:02:03] +[titan] 2025-07-09 21:35:37,535 - root - INFO - step: 36340 loss: 16.7303 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1397 +[titan] 2025-07-09 21:35:37,535 - root - INFO - lr: 2.2364e-04 gnorm: 0.73 [ 8:00:38<14:01:59] +[titan] 2025-07-09 21:35:41,458 - root - INFO - step: 36345 loss: 16.4886 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9387 +[titan] 2025-07-09 21:35:41,458 - root - INFO - lr: 2.2362e-04 gnorm: 0.72 [ 8:00:42<14:01:55] +[titan] 2025-07-09 21:35:44,680 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:35:45,463 - root - INFO - step: 36350 loss: 16.7008 memory: 44.58GiB(31.99%) tps: 81,810 tflops: 282.34 mfu: 28.55% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.1062 +[titan] 2025-07-09 21:35:45,464 - root - INFO - lr: 2.2360e-04 gnorm: 0.79 [ 8:00:46<14:01:51] +[titan] 2025-07-09 21:35:47,177 - root - INFO - Dumping profiler traces at step 36352 +[titan] 2025-07-09 21:35:47,209 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:35:49,554 - root - INFO - step: 36355 loss: 16.9140 memory: 44.58GiB(31.99%) tps: 80,111 tflops: 276.48 mfu: 27.96% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.2863 +[titan] 2025-07-09 21:35:49,554 - root - INFO - lr: 2.2358e-04 gnorm: 0.80 [ 8:00:50<14:01:47] +[titan] 2025-07-09 21:35:53,458 - root - INFO - step: 36360 loss: 16.7815 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.1797 +[titan] 2025-07-09 21:35:53,458 - root - INFO - lr: 2.2356e-04 gnorm: 0.72 [ 8:00:54<14:01:43] +[titan] 2025-07-09 21:35:57,415 - root - INFO - step: 36365 loss: 16.7396 memory: 44.58GiB(31.99%) tps: 82,818 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1491 +[titan] 2025-07-09 21:35:57,415 - root - INFO - lr: 2.2354e-04 gnorm: 0.77 [ 8:00:58<14:01:39] +[titan] 2025-07-09 21:36:01,338 - root - INFO - step: 36370 loss: 16.6088 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0392 +[titan] 2025-07-09 21:36:01,338 - root - INFO - lr: 2.2352e-04 gnorm: 0.74 [ 8:01:02<14:01:35] +[titan] 2025-07-09 21:36:05,258 - root - INFO - step: 36375 loss: 16.7060 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1170 +[titan] 2025-07-09 21:36:05,258 - root - INFO - lr: 2.2350e-04 gnorm: 0.77 [ 8:01:06<14:01:31] +[titan] 2025-07-09 21:36:09,200 - root - INFO - step: 36380 loss: 16.8102 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.2008 +[titan] 2025-07-09 21:36:09,200 - root - INFO - lr: 2.2348e-04 gnorm: 0.80 [ 8:01:10<14:01:27] +[titan] 2025-07-09 21:36:13,111 - root - INFO - step: 36385 loss: 16.7806 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.6062 global_avg_mtp_loss: 14.1744 +[titan] 2025-07-09 21:36:13,112 - root - INFO - lr: 2.2346e-04 gnorm: 0.77 [ 8:01:14<14:01:23] +[titan] 2025-07-09 21:36:17,019 - root - INFO - step: 36390 loss: 16.7862 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1982 +[titan] 2025-07-09 21:36:17,019 - root - INFO - lr: 2.2344e-04 gnorm: 0.76 [ 8:01:18<14:01:19] +[titan] 2025-07-09 21:36:20,941 - root - INFO - step: 36395 loss: 16.6386 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0749 +[titan] 2025-07-09 21:36:20,941 - root - INFO - lr: 2.2343e-04 gnorm: 0.75 [ 8:01:22<14:01:15] +[titan] 2025-07-09 21:36:24,058 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:36:24,856 - root - INFO - step: 36400 loss: 16.5691 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 21:36:24,856 - root - INFO - lr: 2.2341e-04 gnorm: 0.76 [ 8:01:26<14:01:11] +[titan] 2025-07-09 21:36:28,784 - root - INFO - step: 36405 loss: 16.7771 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.1859 +[titan] 2025-07-09 21:36:28,785 - root - INFO - lr: 2.2339e-04 gnorm: 0.79 [ 8:01:30<14:01:07] +[titan] 2025-07-09 21:36:32,732 - root - INFO - step: 36410 loss: 16.7230 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1340 +[titan] 2025-07-09 21:36:32,732 - root - INFO - lr: 2.2337e-04 gnorm: 0.79 [ 8:01:34<14:01:03] +[titan] 2025-07-09 21:36:36,663 - root - INFO - step: 36415 loss: 16.6356 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 14.0701 +[titan] 2025-07-09 21:36:36,663 - root - INFO - lr: 2.2335e-04 gnorm: 0.73 [ 8:01:37<14:00:59] +[titan] 2025-07-09 21:36:40,572 - root - INFO - step: 36420 loss: 16.6375 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0693 +[titan] 2025-07-09 21:36:40,572 - root - INFO - lr: 2.2333e-04 gnorm: 0.79 [ 8:01:41<14:00:55] +[titan] 2025-07-09 21:36:44,470 - root - INFO - step: 36425 loss: 16.8669 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.6242 global_avg_mtp_loss: 14.2426 +[titan] 2025-07-09 21:36:44,471 - root - INFO - lr: 2.2331e-04 gnorm: 0.75 [ 8:01:45<14:00:51] +[titan] 2025-07-09 21:36:48,384 - root - INFO - step: 36430 loss: 16.8248 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6127 global_avg_mtp_loss: 14.2121 +[titan] 2025-07-09 21:36:48,384 - root - INFO - lr: 2.2329e-04 gnorm: 0.78 [ 8:01:49<14:00:47] +[titan] 2025-07-09 21:36:52,321 - root - INFO - step: 36435 loss: 16.6915 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.1143 +[titan] 2025-07-09 21:36:52,321 - root - INFO - lr: 2.2327e-04 gnorm: 0.76 [ 8:01:53<14:00:43] +[titan] 2025-07-09 21:36:56,278 - root - INFO - step: 36440 loss: 16.8062 memory: 44.58GiB(31.99%) tps: 82,810 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.5978 global_avg_mtp_loss: 14.2084 +[titan] 2025-07-09 21:36:56,278 - root - INFO - lr: 2.2325e-04 gnorm: 0.75 [ 8:01:57<14:00:39] +[titan] 2025-07-09 21:37:00,223 - root - INFO - step: 36445 loss: 16.6968 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.1234 +[titan] 2025-07-09 21:37:00,223 - root - INFO - lr: 2.2323e-04 gnorm: 0.77 [ 8:02:01<14:00:35] +[titan] 2025-07-09 21:37:03,348 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:37:04,165 - root - INFO - step: 36450 loss: 16.6545 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.5806 global_avg_mtp_loss: 14.0739 +[titan] 2025-07-09 21:37:04,165 - root - INFO - lr: 2.2321e-04 gnorm: 0.75 [ 8:02:05<14:00:31] +[titan] 2025-07-09 21:37:08,066 - root - INFO - step: 36455 loss: 16.6433 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0677 +[titan] 2025-07-09 21:37:08,066 - root - INFO - lr: 2.2319e-04 gnorm: 0.70 [ 8:02:09<14:00:27] +[titan] 2025-07-09 21:37:12,025 - root - INFO - step: 36460 loss: 16.6895 memory: 44.58GiB(31.99%) tps: 82,788 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1073 +[titan] 2025-07-09 21:37:12,025 - root - INFO - lr: 2.2317e-04 gnorm: 0.82 [ 8:02:13<14:00:23] +[titan] 2025-07-09 21:37:15,950 - root - INFO - step: 36465 loss: 16.9575 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.3243 +[titan] 2025-07-09 21:37:15,950 - root - INFO - lr: 2.2316e-04 gnorm: 0.81 [ 8:02:17<14:00:18] +[titan] 2025-07-09 21:37:19,887 - root - INFO - step: 36470 loss: 16.5997 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 14.0249 +[titan] 2025-07-09 21:37:19,887 - root - INFO - lr: 2.2314e-04 gnorm: 0.81 [ 8:02:21<14:00:14] +[titan] 2025-07-09 21:37:23,796 - root - INFO - step: 36475 loss: 16.9596 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.6243 global_avg_mtp_loss: 14.3353 +[titan] 2025-07-09 21:37:23,797 - root - INFO - lr: 2.2312e-04 gnorm: 0.86 [ 8:02:25<14:00:10] +[titan] 2025-07-09 21:37:27,708 - root - INFO - step: 36480 loss: 16.7558 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.1676 +[titan] 2025-07-09 21:37:27,708 - root - INFO - lr: 2.2310e-04 gnorm: 0.77 [ 8:02:29<14:00:06] +[titan] 2025-07-09 21:37:31,614 - root - INFO - step: 36485 loss: 16.5677 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0003 +[titan] 2025-07-09 21:37:31,614 - root - INFO - lr: 2.2308e-04 gnorm: 0.86 [ 8:02:32<14:00:02] +[titan] 2025-07-09 21:37:35,553 - root - INFO - step: 36490 loss: 16.6120 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.0402 +[titan] 2025-07-09 21:37:35,553 - root - INFO - lr: 2.2306e-04 gnorm: 0.76 [ 8:02:36<13:59:58] +[titan] 2025-07-09 21:37:39,482 - root - INFO - step: 36495 loss: 16.4809 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9366 +[titan] 2025-07-09 21:37:39,482 - root - INFO - lr: 2.2304e-04 gnorm: 0.76 [ 8:02:40<13:59:54] +[titan] 2025-07-09 21:37:42,609 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:37:43,394 - root - INFO - step: 36500 loss: 16.7099 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.1155 +[titan] 2025-07-09 21:37:43,394 - root - INFO - lr: 2.2302e-04 gnorm: 0.71 [ 8:02:44<13:59:50] +[titan] 2025-07-09 21:37:47,336 - root - INFO - step: 36505 loss: 16.9463 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.6298 global_avg_mtp_loss: 14.3165 +[titan] 2025-07-09 21:37:47,337 - root - INFO - lr: 2.2300e-04 gnorm: 0.73 [ 8:02:48<13:59:46] +[titan] 2025-07-09 21:37:51,258 - root - INFO - step: 36510 loss: 16.7247 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1291 +[titan] 2025-07-09 21:37:51,259 - root - INFO - lr: 2.2298e-04 gnorm: 0.79 [ 8:02:52<13:59:42] +[titan] 2025-07-09 21:37:55,203 - root - INFO - step: 36515 loss: 16.6308 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0762 +[titan] 2025-07-09 21:37:55,204 - root - INFO - lr: 2.2296e-04 gnorm: 0.75 [ 8:02:56<13:59:38] +[titan] 2025-07-09 21:37:59,121 - root - INFO - step: 36520 loss: 16.7103 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.1352 +[titan] 2025-07-09 21:37:59,121 - root - INFO - lr: 2.2294e-04 gnorm: 0.86 [ 8:03:00<13:59:34] +[titan] 2025-07-09 21:38:03,014 - root - INFO - step: 36525 loss: 16.4065 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.8772 +[titan] 2025-07-09 21:38:03,014 - root - INFO - lr: 2.2292e-04 gnorm: 0.84 [ 8:03:04<13:59:30] +[titan] 2025-07-09 21:38:06,910 - root - INFO - step: 36530 loss: 16.9182 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2998 +[titan] 2025-07-09 21:38:06,911 - root - INFO - lr: 2.2290e-04 gnorm: 0.79 [ 8:03:08<13:59:26] +[titan] 2025-07-09 21:38:10,856 - root - INFO - step: 36535 loss: 16.8387 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.2328 +[titan] 2025-07-09 21:38:10,857 - root - INFO - lr: 2.2288e-04 gnorm: 0.78 [ 8:03:12<13:59:22] +[titan] 2025-07-09 21:38:14,758 - root - INFO - step: 36540 loss: 16.6089 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0458 +[titan] 2025-07-09 21:38:14,758 - root - INFO - lr: 2.2287e-04 gnorm: 0.78 [ 8:03:16<13:59:18] +[titan] 2025-07-09 21:38:18,681 - root - INFO - step: 36545 loss: 17.0190 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.6584 global_avg_mtp_loss: 14.3605 +[titan] 2025-07-09 21:38:18,681 - root - INFO - lr: 2.2285e-04 gnorm: 0.75 [ 8:03:19<13:59:14] +[titan] 2025-07-09 21:38:21,792 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:38:22,576 - root - INFO - step: 36550 loss: 16.8512 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.2443 +[titan] 2025-07-09 21:38:22,576 - root - INFO - lr: 2.2283e-04 gnorm: 0.76 [ 8:03:23<13:59:10] +[titan] 2025-07-09 21:38:26,515 - root - INFO - step: 36555 loss: 16.7877 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.6001 global_avg_mtp_loss: 14.1876 +[titan] 2025-07-09 21:38:26,515 - root - INFO - lr: 2.2281e-04 gnorm: 0.76 [ 8:03:27<13:59:06] +[titan] 2025-07-09 21:38:30,429 - root - INFO - step: 36560 loss: 16.7781 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5982 global_avg_mtp_loss: 14.1799 +[titan] 2025-07-09 21:38:30,430 - root - INFO - lr: 2.2279e-04 gnorm: 0.77 [ 8:03:31<13:59:02] +[titan] 2025-07-09 21:38:34,356 - root - INFO - step: 36565 loss: 16.4992 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 13.9471 +[titan] 2025-07-09 21:38:34,357 - root - INFO - lr: 2.2277e-04 gnorm: 0.81 [ 8:03:35<13:58:57] +[titan] 2025-07-09 21:38:38,257 - root - INFO - step: 36570 loss: 16.4820 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9370 +[titan] 2025-07-09 21:38:38,258 - root - INFO - lr: 2.2275e-04 gnorm: 0.80 [ 8:03:39<13:58:53] +[titan] 2025-07-09 21:38:42,165 - root - INFO - step: 36575 loss: 16.5512 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 13.9864 +[titan] 2025-07-09 21:38:42,166 - root - INFO - lr: 2.2273e-04 gnorm: 0.76 [ 8:03:43<13:58:49] +[titan] 2025-07-09 21:38:46,073 - root - INFO - step: 36580 loss: 16.5216 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 13.9753 +[titan] 2025-07-09 21:38:46,073 - root - INFO - lr: 2.2271e-04 gnorm: 0.80 [ 8:03:47<13:58:45] +[titan] 2025-07-09 21:38:49,989 - root - INFO - step: 36585 loss: 16.5581 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5541 global_avg_mtp_loss: 14.0040 +[titan] 2025-07-09 21:38:49,989 - root - INFO - lr: 2.2269e-04 gnorm: 0.81 [ 8:03:51<13:58:41] +[titan] 2025-07-09 21:38:53,907 - root - INFO - step: 36590 loss: 16.4577 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-09 21:38:53,908 - root - INFO - lr: 2.2267e-04 gnorm: 0.77 [ 8:03:55<13:58:37] +[titan] 2025-07-09 21:38:57,818 - root - INFO - step: 36595 loss: 16.4754 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 13.9337 +[titan] 2025-07-09 21:38:57,818 - root - INFO - lr: 2.2265e-04 gnorm: 0.76 [ 8:03:59<13:58:33] +[titan] 2025-07-09 21:39:00,954 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:39:01,748 - root - INFO - step: 36600 loss: 16.7109 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1241 +[titan] 2025-07-09 21:39:01,749 - root - INFO - lr: 2.2263e-04 gnorm: 0.75 [ 8:04:03<13:58:29] +[titan] 2025-07-09 21:39:05,655 - root - INFO - step: 36605 loss: 16.7396 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5887 global_avg_mtp_loss: 14.1509 +[titan] 2025-07-09 21:39:05,655 - root - INFO - lr: 2.2261e-04 gnorm: 0.76 [ 8:04:06<13:58:25] +[titan] 2025-07-09 21:39:09,579 - root - INFO - step: 36610 loss: 16.7854 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.1828 +[titan] 2025-07-09 21:39:09,579 - root - INFO - lr: 2.2259e-04 gnorm: 0.87 [ 8:04:10<13:58:21] +[titan] 2025-07-09 21:39:13,492 - root - INFO - step: 36615 loss: 16.7782 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.6033 global_avg_mtp_loss: 14.1749 +[titan] 2025-07-09 21:39:13,492 - root - INFO - lr: 2.2257e-04 gnorm: 0.74 [ 8:04:14<13:58:17] +[titan] 2025-07-09 21:39:17,412 - root - INFO - step: 36620 loss: 16.7874 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.1886 +[titan] 2025-07-09 21:39:17,412 - root - INFO - lr: 2.2256e-04 gnorm: 0.77 [ 8:04:18<13:58:13] +[titan] 2025-07-09 21:39:21,342 - root - INFO - step: 36625 loss: 16.3217 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8046 +[titan] 2025-07-09 21:39:21,342 - root - INFO - lr: 2.2254e-04 gnorm: 0.85 [ 8:04:22<13:58:09] +[titan] 2025-07-09 21:39:25,246 - root - INFO - step: 36630 loss: 16.5938 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5583 global_avg_mtp_loss: 14.0356 +[titan] 2025-07-09 21:39:25,247 - root - INFO - lr: 2.2252e-04 gnorm: 0.74 [ 8:04:26<13:58:05] +[titan] 2025-07-09 21:39:29,149 - root - INFO - step: 36635 loss: 16.8795 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.6200 global_avg_mtp_loss: 14.2595 +[titan] 2025-07-09 21:39:29,149 - root - INFO - lr: 2.2250e-04 gnorm: 0.74 [ 8:04:30<13:58:01] +[titan] 2025-07-09 21:39:33,056 - root - INFO - step: 36640 loss: 16.8564 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.6069 global_avg_mtp_loss: 14.2494 +[titan] 2025-07-09 21:39:33,057 - root - INFO - lr: 2.2248e-04 gnorm: 0.80 [ 8:04:34<13:57:57] +[titan] 2025-07-09 21:39:36,968 - root - INFO - step: 36645 loss: 16.7965 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6026 global_avg_mtp_loss: 14.1939 +[titan] 2025-07-09 21:39:36,969 - root - INFO - lr: 2.2246e-04 gnorm: 0.82 [ 8:04:38<13:57:52] +[titan] 2025-07-09 21:39:40,086 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:39:40,874 - root - INFO - step: 36650 loss: 16.8363 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.6084 global_avg_mtp_loss: 14.2279 +[titan] 2025-07-09 21:39:40,874 - root - INFO - lr: 2.2244e-04 gnorm: 0.76 [ 8:04:42<13:57:48] +[titan] 2025-07-09 21:39:44,807 - root - INFO - step: 36655 loss: 16.8667 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2584 +[titan] 2025-07-09 21:39:44,807 - root - INFO - lr: 2.2242e-04 gnorm: 0.82 [ 8:04:46<13:57:44] +[titan] 2025-07-09 21:39:48,714 - root - INFO - step: 36660 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5604 global_avg_mtp_loss: 14.0555 +[titan] 2025-07-09 21:39:48,715 - root - INFO - lr: 2.2240e-04 gnorm: 0.81 [ 8:04:50<13:57:40] +[titan] 2025-07-09 21:39:52,612 - root - INFO - step: 36665 loss: 16.8126 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.2237 +[titan] 2025-07-09 21:39:52,612 - root - INFO - lr: 2.2238e-04 gnorm: 0.77 [ 8:04:53<13:57:36] +[titan] 2025-07-09 21:39:56,540 - root - INFO - step: 36670 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.8421 +[titan] 2025-07-09 21:39:56,540 - root - INFO - lr: 2.2236e-04 gnorm: 0.72 [ 8:04:57<13:57:32] +[titan] 2025-07-09 21:40:00,458 - root - INFO - step: 36675 loss: 16.7188 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1291 +[titan] 2025-07-09 21:40:00,458 - root - INFO - lr: 2.2234e-04 gnorm: 0.77 [ 8:05:01<13:57:28] +[titan] 2025-07-09 21:40:04,369 - root - INFO - step: 36680 loss: 16.7277 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5922 global_avg_mtp_loss: 14.1356 +[titan] 2025-07-09 21:40:04,369 - root - INFO - lr: 2.2232e-04 gnorm: 0.76 [ 8:05:05<13:57:24] +[titan] 2025-07-09 21:40:08,287 - root - INFO - step: 36685 loss: 16.6596 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.0817 +[titan] 2025-07-09 21:40:08,288 - root - INFO - lr: 2.2230e-04 gnorm: 0.76 [ 8:05:09<13:57:20] +[titan] 2025-07-09 21:40:12,209 - root - INFO - step: 36690 loss: 16.7597 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1728 +[titan] 2025-07-09 21:40:12,209 - root - INFO - lr: 2.2228e-04 gnorm: 0.71 [ 8:05:13<13:57:16] +[titan] 2025-07-09 21:40:16,121 - root - INFO - step: 36695 loss: 16.5108 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9590 +[titan] 2025-07-09 21:40:16,122 - root - INFO - lr: 2.2226e-04 gnorm: 0.78 [ 8:05:17<13:57:12] +[titan] 2025-07-09 21:40:19,250 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:40:20,036 - root - INFO - step: 36700 loss: 16.8200 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.6042 global_avg_mtp_loss: 14.2158 +[titan] 2025-07-09 21:40:20,036 - root - INFO - lr: 2.2225e-04 gnorm: 0.72 [ 8:05:21<13:57:08] +[titan] 2025-07-09 21:40:23,965 - root - INFO - step: 36705 loss: 16.5186 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 13.9620 +[titan] 2025-07-09 21:40:23,965 - root - INFO - lr: 2.2223e-04 gnorm: 0.70 [ 8:05:25<13:57:04] +[titan] 2025-07-09 21:40:27,893 - root - INFO - step: 36710 loss: 16.7948 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.2011 +[titan] 2025-07-09 21:40:27,894 - root - INFO - lr: 2.2221e-04 gnorm: 0.80 [ 8:05:29<13:57:00] +[titan] 2025-07-09 21:40:31,806 - root - INFO - step: 36715 loss: 16.7562 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1673 +[titan] 2025-07-09 21:40:31,806 - root - INFO - lr: 2.2219e-04 gnorm: 0.81 [ 8:05:33<13:56:56] +[titan] 2025-07-09 21:40:35,735 - root - INFO - step: 36720 loss: 16.4428 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 13.9058 +[titan] 2025-07-09 21:40:35,735 - root - INFO - lr: 2.2217e-04 gnorm: 0.86 [ 8:05:37<13:56:52] +[titan] 2025-07-09 21:40:39,705 - root - INFO - step: 36725 loss: 16.7056 memory: 44.58GiB(31.99%) tps: 82,549 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.1314 +[titan] 2025-07-09 21:40:39,705 - root - INFO - lr: 2.2215e-04 gnorm: 0.74 [ 8:05:40<13:56:48] +[titan] 2025-07-09 21:40:43,652 - root - INFO - step: 36730 loss: 16.7974 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.1909 +[titan] 2025-07-09 21:40:43,653 - root - INFO - lr: 2.2213e-04 gnorm: 0.72 [ 8:05:44<13:56:44] +[titan] 2025-07-09 21:40:47,589 - root - INFO - step: 36735 loss: 16.4531 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9078 +[titan] 2025-07-09 21:40:47,589 - root - INFO - lr: 2.2211e-04 gnorm: 0.77 [ 8:05:48<13:56:40] +[titan] 2025-07-09 21:40:51,552 - root - INFO - step: 36740 loss: 16.7590 memory: 44.58GiB(31.99%) tps: 82,676 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.1591 +[titan] 2025-07-09 21:40:51,553 - root - INFO - lr: 2.2209e-04 gnorm: 0.83 [ 8:05:52<13:56:36] +[titan] 2025-07-09 21:40:55,515 - root - INFO - step: 36745 loss: 16.5907 memory: 44.58GiB(31.99%) tps: 82,713 tflops: 285.46 mfu: 28.86% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0420 +[titan] 2025-07-09 21:40:55,515 - root - INFO - lr: 2.2207e-04 gnorm: 0.77 [ 8:05:56<13:56:32] +[titan] 2025-07-09 21:40:58,664 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:40:59,457 - root - INFO - step: 36750 loss: 16.7184 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 21:40:59,458 - root - INFO - lr: 2.2205e-04 gnorm: 0.83 [ 8:06:00<13:56:28] +[titan] 2025-07-09 21:41:03,357 - root - INFO - step: 36755 loss: 16.7466 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.1579 +[titan] 2025-07-09 21:41:03,357 - root - INFO - lr: 2.2203e-04 gnorm: 0.93 [ 8:06:04<13:56:24] +[titan] 2025-07-09 21:41:07,282 - root - INFO - step: 36760 loss: 16.7243 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1379 +[titan] 2025-07-09 21:41:07,282 - root - INFO - lr: 2.2201e-04 gnorm: 0.87 [ 8:06:08<13:56:20] +[titan] 2025-07-09 21:41:11,190 - root - INFO - step: 36765 loss: 16.6090 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0406 +[titan] 2025-07-09 21:41:11,190 - root - INFO - lr: 2.2199e-04 gnorm: 0.83 [ 8:06:12<13:56:16] +[titan] 2025-07-09 21:41:15,083 - root - INFO - step: 36770 loss: 16.7021 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1199 +[titan] 2025-07-09 21:41:15,084 - root - INFO - lr: 2.2197e-04 gnorm: 0.75 [ 8:06:16<13:56:11] +[titan] 2025-07-09 21:41:18,974 - root - INFO - step: 36775 loss: 16.5787 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0208 +[titan] 2025-07-09 21:41:18,974 - root - INFO - lr: 2.2195e-04 gnorm: 0.77 [ 8:06:20<13:56:07] +[titan] 2025-07-09 21:41:22,864 - root - INFO - step: 36780 loss: 16.6040 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0343 +[titan] 2025-07-09 21:41:22,865 - root - INFO - lr: 2.2193e-04 gnorm: 0.80 [ 8:06:24<13:56:03] +[titan] 2025-07-09 21:41:26,765 - root - INFO - step: 36785 loss: 16.6338 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0608 +[titan] 2025-07-09 21:41:26,765 - root - INFO - lr: 2.2192e-04 gnorm: 0.81 [ 8:06:28<13:55:59] +[titan] 2025-07-09 21:41:30,679 - root - INFO - step: 36790 loss: 16.4197 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.8755 +[titan] 2025-07-09 21:41:30,679 - root - INFO - lr: 2.2190e-04 gnorm: 0.78 [ 8:06:31<13:55:55] +[titan] 2025-07-09 21:41:34,566 - root - INFO - step: 36795 loss: 16.6675 memory: 44.58GiB(31.99%) tps: 84,301 tflops: 290.94 mfu: 29.42% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0967 +[titan] 2025-07-09 21:41:34,566 - root - INFO - lr: 2.2188e-04 gnorm: 0.76 [ 8:06:35<13:55:51] +[titan] 2025-07-09 21:41:37,675 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:41:38,468 - root - INFO - step: 36800 loss: 16.6965 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5827 global_avg_mtp_loss: 14.1138 +[titan] 2025-07-09 21:41:38,469 - root - INFO - lr: 2.2186e-04 gnorm: 0.78 [ 8:06:39<13:55:47] +[titan] 2025-07-09 21:41:42,391 - root - INFO - step: 36805 loss: 16.9277 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6144 global_avg_mtp_loss: 14.3132 +[titan] 2025-07-09 21:41:42,391 - root - INFO - lr: 2.2184e-04 gnorm: 0.78 [ 8:06:43<13:55:43] +[titan] 2025-07-09 21:41:46,325 - root - INFO - step: 36810 loss: 16.5637 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 13.9946 +[titan] 2025-07-09 21:41:46,325 - root - INFO - lr: 2.2182e-04 gnorm: 0.73 [ 8:06:47<13:55:39] +[titan] 2025-07-09 21:41:50,240 - root - INFO - step: 36815 loss: 16.6068 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.0335 +[titan] 2025-07-09 21:41:50,241 - root - INFO - lr: 2.2180e-04 gnorm: 0.78 [ 8:06:51<13:55:35] +[titan] 2025-07-09 21:41:54,152 - root - INFO - step: 36820 loss: 16.8047 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.2135 +[titan] 2025-07-09 21:41:54,152 - root - INFO - lr: 2.2178e-04 gnorm: 0.73 [ 8:06:55<13:55:31] +[titan] 2025-07-09 21:41:58,063 - root - INFO - step: 36825 loss: 16.8404 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.2216 +[titan] 2025-07-09 21:41:58,063 - root - INFO - lr: 2.2176e-04 gnorm: 0.74 [ 8:06:59<13:55:27] +[titan] 2025-07-09 21:42:01,974 - root - INFO - step: 36830 loss: 16.7747 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1853 +[titan] 2025-07-09 21:42:01,974 - root - INFO - lr: 2.2174e-04 gnorm: 0.74 [ 8:07:03<13:55:23] +[titan] 2025-07-09 21:42:05,902 - root - INFO - step: 36835 loss: 17.0750 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.6555 global_avg_mtp_loss: 14.4195 +[titan] 2025-07-09 21:42:05,902 - root - INFO - lr: 2.2172e-04 gnorm: 0.76 [ 8:07:07<13:55:19] +[titan] 2025-07-09 21:42:09,834 - root - INFO - step: 36840 loss: 16.5377 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 13.9844 +[titan] 2025-07-09 21:42:09,834 - root - INFO - lr: 2.2170e-04 gnorm: 0.80 [ 8:07:11<13:55:14] +[titan] 2025-07-09 21:42:13,750 - root - INFO - step: 36845 loss: 16.7192 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5994 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 21:42:13,750 - root - INFO - lr: 2.2168e-04 gnorm: 0.78 [ 8:07:15<13:55:10] +[titan] 2025-07-09 21:42:16,863 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:42:17,659 - root - INFO - step: 36850 loss: 16.4043 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8903 +[titan] 2025-07-09 21:42:17,660 - root - INFO - lr: 2.2166e-04 gnorm: 0.79 [ 8:07:18<13:55:06] +[titan] 2025-07-09 21:42:21,629 - root - INFO - step: 36855 loss: 16.5054 memory: 44.58GiB(31.99%) tps: 82,555 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 13.9533 +[titan] 2025-07-09 21:42:21,629 - root - INFO - lr: 2.2164e-04 gnorm: 1.10 [ 8:07:22<13:55:02] +[titan] 2025-07-09 21:42:25,543 - root - INFO - step: 36860 loss: 16.5681 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 14.0180 +[titan] 2025-07-09 21:42:25,543 - root - INFO - lr: 2.2162e-04 gnorm: 0.81 [ 8:07:26<13:54:58] +[titan] 2025-07-09 21:42:28,906 - root - INFO - Dumping profiler traces at step 36864 +[titan] 2025-07-09 21:42:28,937 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:42:29,730 - root - INFO - step: 36865 loss: 16.6489 memory: 44.58GiB(31.99%) tps: 78,259 tflops: 270.08 mfu: 27.31% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0861 +[titan] 2025-07-09 21:42:29,730 - root - INFO - lr: 2.2160e-04 gnorm: 0.76 [ 8:07:30<13:54:55] +[titan] 2025-07-09 21:42:33,664 - root - INFO - step: 36870 loss: 16.6078 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-09 21:42:33,664 - root - INFO - lr: 2.2158e-04 gnorm: 0.81 [ 8:07:34<13:54:51] +[titan] 2025-07-09 21:42:37,557 - root - INFO - step: 36875 loss: 16.6614 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0841 +[titan] 2025-07-09 21:42:37,557 - root - INFO - lr: 2.2157e-04 gnorm: 0.77 [ 8:07:38<13:54:47] +[titan] 2025-07-09 21:42:41,511 - root - INFO - step: 36880 loss: 16.7622 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5922 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 21:42:41,511 - root - INFO - lr: 2.2155e-04 gnorm: 0.77 [ 8:07:42<13:54:43] +[titan] 2025-07-09 21:42:45,420 - root - INFO - step: 36885 loss: 16.3812 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.8502 +[titan] 2025-07-09 21:42:45,420 - root - INFO - lr: 2.2153e-04 gnorm: 0.71 [ 8:07:46<13:54:39] +[titan] 2025-07-09 21:42:49,334 - root - INFO - step: 36890 loss: 16.6953 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.1151 +[titan] 2025-07-09 21:42:49,335 - root - INFO - lr: 2.2151e-04 gnorm: 0.74 [ 8:07:50<13:54:35] +[titan] 2025-07-09 21:42:53,233 - root - INFO - step: 36895 loss: 16.7863 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.1611 +[titan] 2025-07-09 21:42:53,234 - root - INFO - lr: 2.2149e-04 gnorm: 0.79 [ 8:07:54<13:54:30] +[titan] 2025-07-09 21:42:56,387 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:42:57,184 - root - INFO - step: 36900 loss: 16.7245 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5805 global_avg_mtp_loss: 14.1440 +[titan] 2025-07-09 21:42:57,184 - root - INFO - lr: 2.2147e-04 gnorm: 0.80 [ 8:07:58<13:54:26] +[titan] 2025-07-09 21:43:01,085 - root - INFO - step: 36905 loss: 16.7332 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1428 +[titan] 2025-07-09 21:43:01,085 - root - INFO - lr: 2.2145e-04 gnorm: 0.78 [ 8:08:02<13:54:22] +[titan] 2025-07-09 21:43:05,005 - root - INFO - step: 36910 loss: 16.6699 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0992 +[titan] 2025-07-09 21:43:05,005 - root - INFO - lr: 2.2143e-04 gnorm: 0.76 [ 8:08:06<13:54:18] +[titan] 2025-07-09 21:43:08,946 - root - INFO - step: 36915 loss: 16.5349 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 13.9735 +[titan] 2025-07-09 21:43:08,947 - root - INFO - lr: 2.2141e-04 gnorm: 0.73 [ 8:08:10<13:54:14] +[titan] 2025-07-09 21:43:12,877 - root - INFO - step: 36920 loss: 16.5829 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0241 +[titan] 2025-07-09 21:43:12,877 - root - INFO - lr: 2.2139e-04 gnorm: 0.80 [ 8:08:14<13:54:10] +[titan] 2025-07-09 21:43:16,795 - root - INFO - step: 36925 loss: 16.6593 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0837 +[titan] 2025-07-09 21:43:16,795 - root - INFO - lr: 2.2137e-04 gnorm: 1.89 [ 8:08:18<13:54:06] +[titan] 2025-07-09 21:43:20,696 - root - INFO - step: 36930 loss: 16.7945 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.1956 +[titan] 2025-07-09 21:43:20,697 - root - INFO - lr: 2.2135e-04 gnorm: 0.81 [ 8:08:21<13:54:02] +[titan] 2025-07-09 21:43:24,671 - root - INFO - step: 36935 loss: 16.6989 memory: 44.58GiB(31.99%) tps: 82,458 tflops: 284.58 mfu: 28.77% global_avg_ntp_loss: 2.5788 global_avg_mtp_loss: 14.1201 +[titan] 2025-07-09 21:43:24,671 - root - INFO - lr: 2.2133e-04 gnorm: 0.77 [ 8:08:25<13:53:58] +[titan] 2025-07-09 21:43:28,624 - root - INFO - step: 36940 loss: 16.5576 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0000 +[titan] 2025-07-09 21:43:28,625 - root - INFO - lr: 2.2131e-04 gnorm: 0.80 [ 8:08:29<13:53:54] +[titan] 2025-07-09 21:43:32,516 - root - INFO - step: 36945 loss: 16.8185 memory: 44.58GiB(31.99%) tps: 84,211 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.2208 +[titan] 2025-07-09 21:43:32,516 - root - INFO - lr: 2.2129e-04 gnorm: 0.75 [ 8:08:33<13:53:50] +[titan] 2025-07-09 21:43:35,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:43:36,433 - root - INFO - step: 36950 loss: 16.4653 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9427 +[titan] 2025-07-09 21:43:36,433 - root - INFO - lr: 2.2127e-04 gnorm: 0.79 [ 8:08:37<13:53:46] +[titan] 2025-07-09 21:43:40,320 - root - INFO - step: 36955 loss: 16.5571 memory: 44.58GiB(31.99%) tps: 84,294 tflops: 290.91 mfu: 29.41% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0074 +[titan] 2025-07-09 21:43:40,321 - root - INFO - lr: 2.2125e-04 gnorm: 0.77 [ 8:08:41<13:53:42] +[titan] 2025-07-09 21:43:44,215 - root - INFO - step: 36960 loss: 16.6077 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5607 global_avg_mtp_loss: 14.0470 +[titan] 2025-07-09 21:43:44,215 - root - INFO - lr: 2.2123e-04 gnorm: 0.76 [ 8:08:45<13:53:38] +[titan] 2025-07-09 21:43:48,142 - root - INFO - step: 36965 loss: 16.7516 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.1606 +[titan] 2025-07-09 21:43:48,142 - root - INFO - lr: 2.2121e-04 gnorm: 0.78 [ 8:08:49<13:53:34] +[titan] 2025-07-09 21:43:52,061 - root - INFO - step: 36970 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.7633 +[titan] 2025-07-09 21:43:52,061 - root - INFO - lr: 2.2120e-04 gnorm: 0.78 [ 8:08:53<13:53:30] +[titan] 2025-07-09 21:43:55,981 - root - INFO - step: 36975 loss: 16.4829 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-09 21:43:55,981 - root - INFO - lr: 2.2118e-04 gnorm: 0.77 [ 8:08:57<13:53:26] +[titan] 2025-07-09 21:43:59,889 - root - INFO - step: 36980 loss: 16.6487 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.0785 +[titan] 2025-07-09 21:43:59,890 - root - INFO - lr: 2.2116e-04 gnorm: 0.77 [ 8:09:01<13:53:22] +[titan] 2025-07-09 21:44:03,797 - root - INFO - step: 36985 loss: 16.9722 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.6427 global_avg_mtp_loss: 14.3295 +[titan] 2025-07-09 21:44:03,797 - root - INFO - lr: 2.2114e-04 gnorm: 0.85 [ 8:09:05<13:53:18] +[titan] 2025-07-09 21:44:07,721 - root - INFO - step: 36990 loss: 16.7303 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5903 global_avg_mtp_loss: 14.1399 +[titan] 2025-07-09 21:44:07,721 - root - INFO - lr: 2.2112e-04 gnorm: 0.75 [ 8:09:08<13:53:14] +[titan] 2025-07-09 21:44:11,635 - root - INFO - step: 36995 loss: 16.5418 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 13.9851 +[titan] 2025-07-09 21:44:11,635 - root - INFO - lr: 2.2110e-04 gnorm: 0.74 [ 8:09:12<13:53:09] +[titan] 2025-07-09 21:44:14,741 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:44:15,532 - root - INFO - step: 37000 loss: 16.8088 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.2010 +[titan] 2025-07-09 21:44:15,532 - root - INFO - lr: 2.2108e-04 gnorm: 1.04 [ 8:09:16<13:53:05] +[titan] 2025-07-09 21:44:19,422 - root - INFO - step: 37005 loss: 16.6825 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.0905 +[titan] 2025-07-09 21:44:19,423 - root - INFO - lr: 2.2106e-04 gnorm: 0.73 [ 8:09:20<13:53:01] +[titan] 2025-07-09 21:44:23,357 - root - INFO - step: 37010 loss: 16.4963 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 13.9381 +[titan] 2025-07-09 21:44:23,357 - root - INFO - lr: 2.2104e-04 gnorm: 0.76 [ 8:09:24<13:52:57] +[titan] 2025-07-09 21:44:27,267 - root - INFO - step: 37015 loss: 16.5284 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 13.9811 +[titan] 2025-07-09 21:44:27,267 - root - INFO - lr: 2.2102e-04 gnorm: 0.80 [ 8:09:28<13:52:53] +[titan] 2025-07-09 21:44:31,173 - root - INFO - step: 37020 loss: 16.6207 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0607 +[titan] 2025-07-09 21:44:31,173 - root - INFO - lr: 2.2100e-04 gnorm: 0.79 [ 8:09:32<13:52:49] +[titan] 2025-07-09 21:44:35,075 - root - INFO - step: 37025 loss: 16.6918 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1154 +[titan] 2025-07-09 21:44:35,075 - root - INFO - lr: 2.2098e-04 gnorm: 0.81 [ 8:09:36<13:52:45] +[titan] 2025-07-09 21:44:39,030 - root - INFO - step: 37030 loss: 16.5060 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9599 +[titan] 2025-07-09 21:44:39,030 - root - INFO - lr: 2.2096e-04 gnorm: 0.80 [ 8:09:40<13:52:41] +[titan] 2025-07-09 21:44:42,922 - root - INFO - step: 37035 loss: 16.8509 memory: 44.58GiB(31.99%) tps: 84,195 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.6039 global_avg_mtp_loss: 14.2470 +[titan] 2025-07-09 21:44:42,923 - root - INFO - lr: 2.2094e-04 gnorm: 0.75 [ 8:09:44<13:52:37] +[titan] 2025-07-09 21:44:46,850 - root - INFO - step: 37040 loss: 16.7109 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1220 +[titan] 2025-07-09 21:44:46,850 - root - INFO - lr: 2.2092e-04 gnorm: 0.81 [ 8:09:48<13:52:33] +[titan] 2025-07-09 21:44:50,753 - root - INFO - step: 37045 loss: 16.6630 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0899 +[titan] 2025-07-09 21:44:50,753 - root - INFO - lr: 2.2090e-04 gnorm: 0.75 [ 8:09:52<13:52:29] +[titan] 2025-07-09 21:44:53,896 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:44:54,684 - root - INFO - step: 37050 loss: 16.9121 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6195 global_avg_mtp_loss: 14.2926 +[titan] 2025-07-09 21:44:54,684 - root - INFO - lr: 2.2088e-04 gnorm: 0.78 [ 8:09:55<13:52:25] +[titan] 2025-07-09 21:44:58,604 - root - INFO - step: 37055 loss: 16.5751 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0110 +[titan] 2025-07-09 21:44:58,604 - root - INFO - lr: 2.2086e-04 gnorm: 0.82 [ 8:09:59<13:52:21] +[titan] 2025-07-09 21:45:02,505 - root - INFO - step: 37060 loss: 16.4797 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9355 +[titan] 2025-07-09 21:45:02,505 - root - INFO - lr: 2.2084e-04 gnorm: 0.79 [ 8:10:03<13:52:17] +[titan] 2025-07-09 21:45:06,405 - root - INFO - step: 37065 loss: 16.5476 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 13.9954 +[titan] 2025-07-09 21:45:06,405 - root - INFO - lr: 2.2083e-04 gnorm: 0.73 [ 8:10:07<13:52:13] +[titan] 2025-07-09 21:45:10,321 - root - INFO - step: 37070 loss: 16.7244 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.1509 +[titan] 2025-07-09 21:45:10,321 - root - INFO - lr: 2.2081e-04 gnorm: 0.75 [ 8:10:11<13:52:09] +[titan] 2025-07-09 21:45:14,269 - root - INFO - step: 37075 loss: 16.7605 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5905 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 21:45:14,269 - root - INFO - lr: 2.2079e-04 gnorm: 0.77 [ 8:10:15<13:52:05] +[titan] 2025-07-09 21:45:18,168 - root - INFO - step: 37080 loss: 16.9291 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.6172 global_avg_mtp_loss: 14.3119 +[titan] 2025-07-09 21:45:18,168 - root - INFO - lr: 2.2077e-04 gnorm: 0.79 [ 8:10:19<13:52:00] +[titan] 2025-07-09 21:45:22,099 - root - INFO - step: 37085 loss: 16.8048 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.2182 +[titan] 2025-07-09 21:45:22,100 - root - INFO - lr: 2.2075e-04 gnorm: 0.83 [ 8:10:23<13:51:56] +[titan] 2025-07-09 21:45:26,007 - root - INFO - step: 37090 loss: 16.5394 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 13.9614 +[titan] 2025-07-09 21:45:26,007 - root - INFO - lr: 2.2073e-04 gnorm: 0.84 [ 8:10:27<13:51:52] +[titan] 2025-07-09 21:45:29,936 - root - INFO - step: 37095 loss: 16.4809 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9430 +[titan] 2025-07-09 21:45:29,936 - root - INFO - lr: 2.2071e-04 gnorm: 1.44 [ 8:10:31<13:51:48] +[titan] 2025-07-09 21:45:33,063 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:45:33,891 - root - INFO - step: 37100 loss: 16.6189 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.0472 +[titan] 2025-07-09 21:45:33,892 - root - INFO - lr: 2.2069e-04 gnorm: 0.73 [ 8:10:35<13:51:44] +[titan] 2025-07-09 21:45:37,805 - root - INFO - step: 37105 loss: 16.5595 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-09 21:45:37,805 - root - INFO - lr: 2.2067e-04 gnorm: 0.77 [ 8:10:39<13:51:40] +[titan] 2025-07-09 21:45:41,718 - root - INFO - step: 37110 loss: 16.7776 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.1836 +[titan] 2025-07-09 21:45:41,718 - root - INFO - lr: 2.2065e-04 gnorm: 0.75 [ 8:10:42<13:51:36] +[titan] 2025-07-09 21:45:45,656 - root - INFO - step: 37115 loss: 16.8780 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.2772 +[titan] 2025-07-09 21:45:45,657 - root - INFO - lr: 2.2063e-04 gnorm: 0.80 [ 8:10:46<13:51:32] +[titan] 2025-07-09 21:45:49,566 - root - INFO - step: 37120 loss: 16.4259 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 13.8814 +[titan] 2025-07-09 21:45:49,566 - root - INFO - lr: 2.2061e-04 gnorm: 0.80 [ 8:10:50<13:51:28] +[titan] 2025-07-09 21:45:53,462 - root - INFO - step: 37125 loss: 16.8267 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.2249 +[titan] 2025-07-09 21:45:53,462 - root - INFO - lr: 2.2059e-04 gnorm: 0.79 [ 8:10:54<13:51:24] +[titan] 2025-07-09 21:45:57,385 - root - INFO - step: 37130 loss: 16.5376 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 13.9728 +[titan] 2025-07-09 21:45:57,386 - root - INFO - lr: 2.2057e-04 gnorm: 0.82 [ 8:10:58<13:51:20] +[titan] 2025-07-09 21:46:01,281 - root - INFO - step: 37135 loss: 16.7234 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.1455 +[titan] 2025-07-09 21:46:01,281 - root - INFO - lr: 2.2055e-04 gnorm: 0.80 [ 8:11:02<13:51:16] +[titan] 2025-07-09 21:46:05,171 - root - INFO - step: 37140 loss: 16.6515 memory: 44.58GiB(31.99%) tps: 84,245 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0742 +[titan] 2025-07-09 21:46:05,171 - root - INFO - lr: 2.2053e-04 gnorm: 0.75 [ 8:11:06<13:51:12] +[titan] 2025-07-09 21:46:09,067 - root - INFO - step: 37145 loss: 16.7399 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.1539 +[titan] 2025-07-09 21:46:09,067 - root - INFO - lr: 2.2051e-04 gnorm: 0.82 [ 8:11:10<13:51:08] +[titan] 2025-07-09 21:46:12,198 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:46:12,987 - root - INFO - step: 37150 loss: 16.6793 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 14.1147 +[titan] 2025-07-09 21:46:12,987 - root - INFO - lr: 2.2049e-04 gnorm: 0.83 [ 8:11:14<13:51:04] +[titan] 2025-07-09 21:46:16,882 - root - INFO - step: 37155 loss: 16.7298 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5900 global_avg_mtp_loss: 14.1397 +[titan] 2025-07-09 21:46:16,882 - root - INFO - lr: 2.2047e-04 gnorm: 0.76 [ 8:11:18<13:51:00] +[titan] 2025-07-09 21:46:20,790 - root - INFO - step: 37160 loss: 16.6252 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0509 +[titan] 2025-07-09 21:46:20,790 - root - INFO - lr: 2.2045e-04 gnorm: 0.77 [ 8:11:22<13:50:56] +[titan] 2025-07-09 21:46:24,701 - root - INFO - step: 37165 loss: 16.1812 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.6985 +[titan] 2025-07-09 21:46:24,701 - root - INFO - lr: 2.2043e-04 gnorm: 0.74 [ 8:11:25<13:50:51] +[titan] 2025-07-09 21:46:28,626 - root - INFO - step: 37170 loss: 16.8252 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.2141 +[titan] 2025-07-09 21:46:28,626 - root - INFO - lr: 2.2042e-04 gnorm: 0.80 [ 8:11:29<13:50:47] +[titan] 2025-07-09 21:46:32,566 - root - INFO - step: 37175 loss: 16.9524 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.6390 global_avg_mtp_loss: 14.3134 +[titan] 2025-07-09 21:46:32,567 - root - INFO - lr: 2.2040e-04 gnorm: 0.79 [ 8:11:33<13:50:43] +[titan] 2025-07-09 21:46:36,563 - root - INFO - step: 37180 loss: 16.8450 memory: 44.58GiB(31.99%) tps: 81,987 tflops: 282.95 mfu: 28.61% global_avg_ntp_loss: 2.6062 global_avg_mtp_loss: 14.2388 +[titan] 2025-07-09 21:46:36,564 - root - INFO - lr: 2.2038e-04 gnorm: 0.77 [ 8:11:37<13:50:40] +[titan] 2025-07-09 21:46:40,506 - root - INFO - step: 37185 loss: 16.6401 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5636 global_avg_mtp_loss: 14.0765 +[titan] 2025-07-09 21:46:40,506 - root - INFO - lr: 2.2036e-04 gnorm: 0.74 [ 8:11:41<13:50:36] +[titan] 2025-07-09 21:46:44,449 - root - INFO - step: 37190 loss: 16.8958 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6179 global_avg_mtp_loss: 14.2779 +[titan] 2025-07-09 21:46:44,450 - root - INFO - lr: 2.2034e-04 gnorm: 0.81 [ 8:11:45<13:50:32] +[titan] 2025-07-09 21:46:48,387 - root - INFO - step: 37195 loss: 16.5092 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9654 +[titan] 2025-07-09 21:46:48,387 - root - INFO - lr: 2.2032e-04 gnorm: 0.73 [ 8:11:49<13:50:27] +[titan] 2025-07-09 21:46:51,496 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:46:52,281 - root - INFO - step: 37200 loss: 16.6600 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0835 +[titan] 2025-07-09 21:46:52,281 - root - INFO - lr: 2.2030e-04 gnorm: 0.76 [ 8:11:53<13:50:23] +[titan] 2025-07-09 21:46:56,196 - root - INFO - step: 37205 loss: 16.5848 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0182 +[titan] 2025-07-09 21:46:56,196 - root - INFO - lr: 2.2028e-04 gnorm: 0.78 [ 8:11:57<13:50:19] +[titan] 2025-07-09 21:47:00,108 - root - INFO - step: 37210 loss: 16.4209 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.8802 +[titan] 2025-07-09 21:47:00,109 - root - INFO - lr: 2.2026e-04 gnorm: 0.74 [ 8:12:01<13:50:15] +[titan] 2025-07-09 21:47:04,013 - root - INFO - step: 37215 loss: 16.4563 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9144 +[titan] 2025-07-09 21:47:04,014 - root - INFO - lr: 2.2024e-04 gnorm: 0.78 [ 8:12:05<13:50:11] +[titan] 2025-07-09 21:47:07,979 - root - INFO - step: 37220 loss: 16.4672 memory: 44.58GiB(31.99%) tps: 82,644 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 13.9189 +[titan] 2025-07-09 21:47:07,979 - root - INFO - lr: 2.2022e-04 gnorm: 0.78 [ 8:12:09<13:50:07] +[titan] 2025-07-09 21:47:11,907 - root - INFO - step: 37225 loss: 16.8326 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6163 global_avg_mtp_loss: 14.2163 +[titan] 2025-07-09 21:47:11,908 - root - INFO - lr: 2.2020e-04 gnorm: 0.74 [ 8:12:13<13:50:03] +[titan] 2025-07-09 21:47:15,811 - root - INFO - step: 37230 loss: 16.8415 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.2390 +[titan] 2025-07-09 21:47:15,811 - root - INFO - lr: 2.2018e-04 gnorm: 0.75 [ 8:12:17<13:49:59] +[titan] 2025-07-09 21:47:19,707 - root - INFO - step: 37235 loss: 16.6546 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0914 +[titan] 2025-07-09 21:47:19,707 - root - INFO - lr: 2.2016e-04 gnorm: 0.73 [ 8:12:20<13:49:55] +[titan] 2025-07-09 21:47:23,618 - root - INFO - step: 37240 loss: 16.4949 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9657 +[titan] 2025-07-09 21:47:23,619 - root - INFO - lr: 2.2014e-04 gnorm: 0.72 [ 8:12:24<13:49:51] +[titan] 2025-07-09 21:47:27,549 - root - INFO - step: 37245 loss: 16.4448 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.9166 +[titan] 2025-07-09 21:47:27,549 - root - INFO - lr: 2.2012e-04 gnorm: 0.79 [ 8:12:28<13:49:47] +[titan] 2025-07-09 21:47:30,676 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:47:31,469 - root - INFO - step: 37250 loss: 16.7673 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1748 +[titan] 2025-07-09 21:47:31,469 - root - INFO - lr: 2.2010e-04 gnorm: 0.75 [ 8:12:32<13:49:43] +[titan] 2025-07-09 21:47:35,419 - root - INFO - step: 37255 loss: 16.5391 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9926 +[titan] 2025-07-09 21:47:35,419 - root - INFO - lr: 2.2008e-04 gnorm: 1.05 [ 8:12:36<13:49:39] +[titan] 2025-07-09 21:47:39,306 - root - INFO - step: 37260 loss: 16.6979 memory: 44.58GiB(31.99%) tps: 84,319 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-09 21:47:39,306 - root - INFO - lr: 2.2006e-04 gnorm: 0.75 [ 8:12:40<13:49:35] +[titan] 2025-07-09 21:47:43,235 - root - INFO - step: 37265 loss: 16.8977 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.6141 global_avg_mtp_loss: 14.2836 +[titan] 2025-07-09 21:47:43,236 - root - INFO - lr: 2.2004e-04 gnorm: 0.90 [ 8:12:44<13:49:31] +[titan] 2025-07-09 21:47:47,137 - root - INFO - step: 37270 loss: 16.8154 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.6065 global_avg_mtp_loss: 14.2088 +[titan] 2025-07-09 21:47:47,137 - root - INFO - lr: 2.2002e-04 gnorm: 0.84 [ 8:12:48<13:49:27] +[titan] 2025-07-09 21:47:51,046 - root - INFO - step: 37275 loss: 16.6641 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0886 +[titan] 2025-07-09 21:47:51,046 - root - INFO - lr: 2.2000e-04 gnorm: 0.86 [ 8:12:52<13:49:23] +[titan] 2025-07-09 21:47:54,947 - root - INFO - step: 37280 loss: 16.8015 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6203 global_avg_mtp_loss: 14.1812 +[titan] 2025-07-09 21:47:54,948 - root - INFO - lr: 2.1999e-04 gnorm: 0.87 [ 8:12:56<13:49:19] +[titan] 2025-07-09 21:47:58,851 - root - INFO - step: 37285 loss: 16.6324 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5766 global_avg_mtp_loss: 14.0557 +[titan] 2025-07-09 21:47:58,852 - root - INFO - lr: 2.1997e-04 gnorm: 0.73 [ 8:13:00<13:49:14] +[titan] 2025-07-09 21:48:02,775 - root - INFO - step: 37290 loss: 16.5924 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0266 +[titan] 2025-07-09 21:48:02,775 - root - INFO - lr: 2.1995e-04 gnorm: 0.72 [ 8:13:04<13:49:10] +[titan] 2025-07-09 21:48:06,705 - root - INFO - step: 37295 loss: 16.8297 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6157 global_avg_mtp_loss: 14.2140 +[titan] 2025-07-09 21:48:06,705 - root - INFO - lr: 2.1993e-04 gnorm: 0.73 [ 8:13:07<13:49:06] +[titan] 2025-07-09 21:48:09,839 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:48:10,626 - root - INFO - step: 37300 loss: 17.0472 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6539 global_avg_mtp_loss: 14.3932 +[titan] 2025-07-09 21:48:10,626 - root - INFO - lr: 2.1991e-04 gnorm: 0.78 [ 8:13:11<13:49:02] +[titan] 2025-07-09 21:48:14,577 - root - INFO - step: 37305 loss: 16.7063 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1359 +[titan] 2025-07-09 21:48:14,578 - root - INFO - lr: 2.1989e-04 gnorm: 0.88 [ 8:13:15<13:48:58] +[titan] 2025-07-09 21:48:18,485 - root - INFO - step: 37310 loss: 16.8250 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.2206 +[titan] 2025-07-09 21:48:18,485 - root - INFO - lr: 2.1987e-04 gnorm: 0.79 [ 8:13:19<13:48:54] +[titan] 2025-07-09 21:48:22,439 - root - INFO - step: 37315 loss: 16.2746 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7784 +[titan] 2025-07-09 21:48:22,439 - root - INFO - lr: 2.1985e-04 gnorm: 0.80 [ 8:13:23<13:48:50] +[titan] 2025-07-09 21:48:26,373 - root - INFO - step: 37320 loss: 16.6021 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0476 +[titan] 2025-07-09 21:48:26,374 - root - INFO - lr: 2.1983e-04 gnorm: 0.80 [ 8:13:27<13:48:46] +[titan] 2025-07-09 21:48:30,361 - root - INFO - step: 37325 loss: 16.8400 memory: 44.58GiB(31.99%) tps: 82,180 tflops: 283.62 mfu: 28.68% global_avg_ntp_loss: 2.6155 global_avg_mtp_loss: 14.2245 +[titan] 2025-07-09 21:48:30,361 - root - INFO - lr: 2.1981e-04 gnorm: 0.79 [ 8:13:31<13:48:42] +[titan] 2025-07-09 21:48:34,330 - root - INFO - step: 37330 loss: 16.4562 memory: 44.58GiB(31.99%) tps: 82,561 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 13.9046 +[titan] 2025-07-09 21:48:34,331 - root - INFO - lr: 2.1979e-04 gnorm: 0.78 [ 8:13:35<13:48:38] +[titan] 2025-07-09 21:48:38,277 - root - INFO - step: 37335 loss: 16.3135 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5184 global_avg_mtp_loss: 13.7951 +[titan] 2025-07-09 21:48:38,277 - root - INFO - lr: 2.1977e-04 gnorm: 0.79 [ 8:13:39<13:48:34] +[titan] 2025-07-09 21:48:42,179 - root - INFO - step: 37340 loss: 16.7797 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.1753 +[titan] 2025-07-09 21:48:42,180 - root - INFO - lr: 2.1975e-04 gnorm: 0.79 [ 8:13:43<13:48:30] +[titan] 2025-07-09 21:48:46,110 - root - INFO - step: 37345 loss: 16.5510 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5545 global_avg_mtp_loss: 13.9965 +[titan] 2025-07-09 21:48:46,110 - root - INFO - lr: 2.1973e-04 gnorm: 0.77 [ 8:13:47<13:48:26] +[titan] 2025-07-09 21:48:49,280 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:48:50,069 - root - INFO - step: 37350 loss: 16.7006 memory: 44.58GiB(31.99%) tps: 82,777 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1176 +[titan] 2025-07-09 21:48:50,069 - root - INFO - lr: 2.1971e-04 gnorm: 0.85 [ 8:13:51<13:48:22] +[titan] 2025-07-09 21:48:54,012 - root - INFO - step: 37355 loss: 16.4379 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9130 +[titan] 2025-07-09 21:48:54,012 - root - INFO - lr: 2.1969e-04 gnorm: 0.84 [ 8:13:55<13:48:18] +[titan] 2025-07-09 21:48:57,939 - root - INFO - step: 37360 loss: 16.4825 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9406 +[titan] 2025-07-09 21:48:57,939 - root - INFO - lr: 2.1967e-04 gnorm: 0.72 [ 8:13:59<13:48:14] +[titan] 2025-07-09 21:49:01,859 - root - INFO - step: 37365 loss: 16.5867 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0217 +[titan] 2025-07-09 21:49:01,859 - root - INFO - lr: 2.1965e-04 gnorm: 0.87 [ 8:14:03<13:48:10] +[titan] 2025-07-09 21:49:05,815 - root - INFO - step: 37370 loss: 16.7655 memory: 44.58GiB(31.99%) tps: 82,841 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.1774 +[titan] 2025-07-09 21:49:05,815 - root - INFO - lr: 2.1963e-04 gnorm: 0.73 [ 8:14:07<13:48:06] +[titan] 2025-07-09 21:49:09,816 - root - INFO - step: 37375 loss: 16.6046 memory: 44.58GiB(31.99%) tps: 81,913 tflops: 282.69 mfu: 28.58% global_avg_ntp_loss: 2.5615 global_avg_mtp_loss: 14.0431 +[titan] 2025-07-09 21:49:09,816 - root - INFO - lr: 2.1961e-04 gnorm: 0.78 [ 8:14:11<13:48:02] +[titan] 2025-07-09 21:49:10,758 - root - INFO - Dumping profiler traces at step 37376 +[titan] 2025-07-09 21:49:10,790 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:49:13,919 - root - INFO - step: 37380 loss: 16.4800 memory: 44.58GiB(31.99%) tps: 79,864 tflops: 275.63 mfu: 27.87% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 13.9364 +[titan] 2025-07-09 21:49:13,920 - root - INFO - lr: 2.1959e-04 gnorm: 0.75 [ 8:14:15<13:47:59] +[titan] 2025-07-09 21:49:17,899 - root - INFO - step: 37385 loss: 16.4337 memory: 44.58GiB(31.99%) tps: 82,345 tflops: 284.19 mfu: 28.73% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-09 21:49:17,899 - root - INFO - lr: 2.1957e-04 gnorm: 0.79 [ 8:14:19<13:47:55] +[titan] 2025-07-09 21:49:21,853 - root - INFO - step: 37390 loss: 16.7612 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.1561 +[titan] 2025-07-09 21:49:21,853 - root - INFO - lr: 2.1955e-04 gnorm: 0.79 [ 8:14:23<13:47:51] +[titan] 2025-07-09 21:49:25,772 - root - INFO - step: 37395 loss: 16.7701 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.1669 +[titan] 2025-07-09 21:49:25,772 - root - INFO - lr: 2.1953e-04 gnorm: 0.80 [ 8:14:26<13:47:47] +[titan] 2025-07-09 21:49:28,926 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:49:29,728 - root - INFO - step: 37400 loss: 16.7591 memory: 44.58GiB(31.99%) tps: 82,843 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.1586 +[titan] 2025-07-09 21:49:29,728 - root - INFO - lr: 2.1952e-04 gnorm: 0.75 [ 8:14:30<13:47:43] +[titan] 2025-07-09 21:49:33,686 - root - INFO - step: 37405 loss: 16.9271 memory: 44.58GiB(31.99%) tps: 82,802 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.6098 global_avg_mtp_loss: 14.3173 +[titan] 2025-07-09 21:49:33,686 - root - INFO - lr: 2.1950e-04 gnorm: 0.75 [ 8:14:34<13:47:39] +[titan] 2025-07-09 21:49:37,617 - root - INFO - step: 37410 loss: 16.7910 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5964 global_avg_mtp_loss: 14.1946 +[titan] 2025-07-09 21:49:37,617 - root - INFO - lr: 2.1948e-04 gnorm: 0.80 [ 8:14:38<13:47:35] +[titan] 2025-07-09 21:49:41,563 - root - INFO - step: 37415 loss: 16.7669 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 21:49:41,564 - root - INFO - lr: 2.1946e-04 gnorm: 0.78 [ 8:14:42<13:47:31] +[titan] 2025-07-09 21:49:45,497 - root - INFO - step: 37420 loss: 16.6307 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 14.0687 +[titan] 2025-07-09 21:49:45,497 - root - INFO - lr: 2.1944e-04 gnorm: 0.76 [ 8:14:46<13:47:27] +[titan] 2025-07-09 21:49:49,421 - root - INFO - step: 37425 loss: 16.6954 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5771 global_avg_mtp_loss: 14.1183 +[titan] 2025-07-09 21:49:49,422 - root - INFO - lr: 2.1942e-04 gnorm: 0.78 [ 8:14:50<13:47:23] +[titan] 2025-07-09 21:49:53,372 - root - INFO - step: 37430 loss: 16.6551 memory: 44.58GiB(31.99%) tps: 82,941 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5770 global_avg_mtp_loss: 14.0781 +[titan] 2025-07-09 21:49:53,373 - root - INFO - lr: 2.1940e-04 gnorm: 0.74 [ 8:14:54<13:47:19] +[titan] 2025-07-09 21:49:57,287 - root - INFO - step: 37435 loss: 16.8711 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.2552 +[titan] 2025-07-09 21:49:57,288 - root - INFO - lr: 2.1938e-04 gnorm: 0.78 [ 8:14:58<13:47:15] +[titan] 2025-07-09 21:50:01,189 - root - INFO - step: 37440 loss: 16.6367 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0725 +[titan] 2025-07-09 21:50:01,189 - root - INFO - lr: 2.1936e-04 gnorm: 0.82 [ 8:15:02<13:47:10] +[titan] 2025-07-09 21:50:05,099 - root - INFO - step: 37445 loss: 16.8124 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5981 global_avg_mtp_loss: 14.2143 +[titan] 2025-07-09 21:50:05,100 - root - INFO - lr: 2.1934e-04 gnorm: 0.77 [ 8:15:06<13:47:06] +[titan] 2025-07-09 21:50:08,232 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:50:09,027 - root - INFO - step: 37450 loss: 16.3464 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-09 21:50:09,027 - root - INFO - lr: 2.1932e-04 gnorm: 0.78 [ 8:15:10<13:47:02] +[titan] 2025-07-09 21:50:12,953 - root - INFO - step: 37455 loss: 16.7182 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5745 global_avg_mtp_loss: 14.1437 +[titan] 2025-07-09 21:50:12,953 - root - INFO - lr: 2.1930e-04 gnorm: 0.75 [ 8:15:14<13:46:58] +[titan] 2025-07-09 21:50:16,860 - root - INFO - step: 37460 loss: 16.6622 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0940 +[titan] 2025-07-09 21:50:16,860 - root - INFO - lr: 2.1928e-04 gnorm: 0.76 [ 8:15:18<13:46:54] +[titan] 2025-07-09 21:50:20,788 - root - INFO - step: 37465 loss: 16.4310 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.9025 +[titan] 2025-07-09 21:50:20,788 - root - INFO - lr: 2.1926e-04 gnorm: 0.78 [ 8:15:21<13:46:50] +[titan] 2025-07-09 21:50:24,716 - root - INFO - step: 37470 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 14.0583 +[titan] 2025-07-09 21:50:24,716 - root - INFO - lr: 2.1924e-04 gnorm: 0.73 [ 8:15:25<13:46:46] +[titan] 2025-07-09 21:50:28,632 - root - INFO - step: 37475 loss: 16.7040 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5954 global_avg_mtp_loss: 14.1087 +[titan] 2025-07-09 21:50:28,633 - root - INFO - lr: 2.1922e-04 gnorm: 0.75 [ 8:15:29<13:46:42] +[titan] 2025-07-09 21:50:32,580 - root - INFO - step: 37480 loss: 16.6197 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 14.0505 +[titan] 2025-07-09 21:50:32,580 - root - INFO - lr: 2.1920e-04 gnorm: 0.77 [ 8:15:33<13:46:38] +[titan] 2025-07-09 21:50:36,473 - root - INFO - step: 37485 loss: 16.3657 memory: 44.58GiB(31.99%) tps: 84,190 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.8466 +[titan] 2025-07-09 21:50:36,473 - root - INFO - lr: 2.1918e-04 gnorm: 0.76 [ 8:15:37<13:46:34] +[titan] 2025-07-09 21:50:40,445 - root - INFO - step: 37490 loss: 16.7796 memory: 44.58GiB(31.99%) tps: 82,493 tflops: 284.70 mfu: 28.79% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1904 +[titan] 2025-07-09 21:50:40,446 - root - INFO - lr: 2.1916e-04 gnorm: 0.77 [ 8:15:41<13:46:30] +[titan] 2025-07-09 21:50:44,351 - root - INFO - step: 37495 loss: 16.5634 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 14.0167 +[titan] 2025-07-09 21:50:44,352 - root - INFO - lr: 2.1914e-04 gnorm: 0.76 [ 8:15:45<13:46:26] +[titan] 2025-07-09 21:50:47,465 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:50:48,258 - root - INFO - step: 37500 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 14.0296 +[titan] 2025-07-09 21:50:48,259 - root - INFO - lr: 2.1912e-04 gnorm: 0.83 [ 8:15:49<13:46:22] +[titan] 2025-07-09 21:50:52,223 - root - INFO - step: 37505 loss: 16.8258 memory: 44.58GiB(31.99%) tps: 82,657 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.2232 +[titan] 2025-07-09 21:50:52,223 - root - INFO - lr: 2.1910e-04 gnorm: 0.86 [ 8:15:53<13:46:18] +[titan] 2025-07-09 21:50:56,122 - root - INFO - step: 37510 loss: 16.4028 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.8695 +[titan] 2025-07-09 21:50:56,122 - root - INFO - lr: 2.1908e-04 gnorm: 0.87 [ 8:15:57<13:46:14] +[titan] 2025-07-09 21:51:00,035 - root - INFO - step: 37515 loss: 16.6972 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5936 global_avg_mtp_loss: 14.1036 +[titan] 2025-07-09 21:51:00,035 - root - INFO - lr: 2.1906e-04 gnorm: 0.83 [ 8:16:01<13:46:10] +[titan] 2025-07-09 21:51:03,955 - root - INFO - step: 37520 loss: 17.0063 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3644 +[titan] 2025-07-09 21:51:03,956 - root - INFO - lr: 2.1904e-04 gnorm: 0.84 [ 8:16:05<13:46:06] +[titan] 2025-07-09 21:51:07,907 - root - INFO - step: 37525 loss: 16.3233 memory: 44.58GiB(31.99%) tps: 82,937 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8079 +[titan] 2025-07-09 21:51:07,907 - root - INFO - lr: 2.1903e-04 gnorm: 0.73 [ 8:16:09<13:46:02] +[titan] 2025-07-09 21:51:11,906 - root - INFO - step: 37530 loss: 16.7617 memory: 44.58GiB(31.99%) tps: 81,936 tflops: 282.77 mfu: 28.59% global_avg_ntp_loss: 2.6018 global_avg_mtp_loss: 14.1599 +[titan] 2025-07-09 21:51:11,907 - root - INFO - lr: 2.1901e-04 gnorm: 0.73 [ 8:16:13<13:45:58] +[titan] 2025-07-09 21:51:15,807 - root - INFO - step: 37535 loss: 16.8078 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.6052 global_avg_mtp_loss: 14.2026 +[titan] 2025-07-09 21:51:15,808 - root - INFO - lr: 2.1899e-04 gnorm: 0.78 [ 8:16:17<13:45:54] +[titan] 2025-07-09 21:51:19,727 - root - INFO - step: 37540 loss: 16.9280 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6328 global_avg_mtp_loss: 14.2952 +[titan] 2025-07-09 21:51:19,727 - root - INFO - lr: 2.1897e-04 gnorm: 0.80 [ 8:16:20<13:45:50] +[titan] 2025-07-09 21:51:23,666 - root - INFO - step: 37545 loss: 16.6914 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.1141 +[titan] 2025-07-09 21:51:23,667 - root - INFO - lr: 2.1895e-04 gnorm: 0.81 [ 8:16:24<13:45:46] +[titan] 2025-07-09 21:51:26,779 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:51:27,583 - root - INFO - step: 37550 loss: 16.6401 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0736 +[titan] 2025-07-09 21:51:27,583 - root - INFO - lr: 2.1893e-04 gnorm: 0.76 [ 8:16:28<13:45:42] +[titan] 2025-07-09 21:51:31,489 - root - INFO - step: 37555 loss: 16.7091 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5961 global_avg_mtp_loss: 14.1130 +[titan] 2025-07-09 21:51:31,490 - root - INFO - lr: 2.1891e-04 gnorm: 0.77 [ 8:16:32<13:45:38] +[titan] 2025-07-09 21:51:35,408 - root - INFO - step: 37560 loss: 16.7462 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1618 +[titan] 2025-07-09 21:51:35,408 - root - INFO - lr: 2.1889e-04 gnorm: 0.72 [ 8:16:36<13:45:34] +[titan] 2025-07-09 21:51:39,330 - root - INFO - step: 37565 loss: 17.0143 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6385 global_avg_mtp_loss: 14.3758 +[titan] 2025-07-09 21:51:39,330 - root - INFO - lr: 2.1887e-04 gnorm: 0.74 [ 8:16:40<13:45:30] +[titan] 2025-07-09 21:51:43,277 - root - INFO - step: 37570 loss: 16.7986 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.2047 +[titan] 2025-07-09 21:51:43,278 - root - INFO - lr: 2.1885e-04 gnorm: 0.75 [ 8:16:44<13:45:26] +[titan] 2025-07-09 21:51:47,224 - root - INFO - step: 37575 loss: 16.6068 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-09 21:51:47,225 - root - INFO - lr: 2.1883e-04 gnorm: 0.78 [ 8:16:48<13:45:22] +[titan] 2025-07-09 21:51:51,159 - root - INFO - step: 37580 loss: 16.6567 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0824 +[titan] 2025-07-09 21:51:51,159 - root - INFO - lr: 2.1881e-04 gnorm: 0.77 [ 8:16:52<13:45:18] +[titan] 2025-07-09 21:51:55,055 - root - INFO - step: 37585 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9974 +[titan] 2025-07-09 21:51:55,055 - root - INFO - lr: 2.1879e-04 gnorm: 0.79 [ 8:16:56<13:45:13] +[titan] 2025-07-09 21:51:58,955 - root - INFO - step: 37590 loss: 16.6359 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0633 +[titan] 2025-07-09 21:51:58,955 - root - INFO - lr: 2.1877e-04 gnorm: 0.74 [ 8:17:00<13:45:09] +[titan] 2025-07-09 21:52:02,879 - root - INFO - step: 37595 loss: 16.5038 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9596 +[titan] 2025-07-09 21:52:02,879 - root - INFO - lr: 2.1875e-04 gnorm: 0.84 [ 8:17:04<13:45:05] +[titan] 2025-07-09 21:52:05,993 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:52:06,779 - root - INFO - step: 37600 loss: 16.4796 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9380 +[titan] 2025-07-09 21:52:06,779 - root - INFO - lr: 2.1873e-04 gnorm: 0.83 [ 8:17:07<13:45:01] +[titan] 2025-07-09 21:52:10,676 - root - INFO - step: 37605 loss: 16.5985 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0353 +[titan] 2025-07-09 21:52:10,677 - root - INFO - lr: 2.1871e-04 gnorm: 0.80 [ 8:17:11<13:44:57] +[titan] 2025-07-09 21:52:14,572 - root - INFO - step: 37610 loss: 16.6381 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0768 +[titan] 2025-07-09 21:52:14,573 - root - INFO - lr: 2.1869e-04 gnorm: 0.79 [ 8:17:15<13:44:53] +[titan] 2025-07-09 21:52:18,493 - root - INFO - step: 37615 loss: 16.7349 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.1401 +[titan] 2025-07-09 21:52:18,493 - root - INFO - lr: 2.1867e-04 gnorm: 0.79 [ 8:17:19<13:44:49] +[titan] 2025-07-09 21:52:22,405 - root - INFO - step: 37620 loss: 16.7640 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5868 global_avg_mtp_loss: 14.1772 +[titan] 2025-07-09 21:52:22,406 - root - INFO - lr: 2.1865e-04 gnorm: 0.77 [ 8:17:23<13:44:45] +[titan] 2025-07-09 21:52:26,309 - root - INFO - step: 37625 loss: 16.7851 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5922 global_avg_mtp_loss: 14.1929 +[titan] 2025-07-09 21:52:26,310 - root - INFO - lr: 2.1863e-04 gnorm: 0.75 [ 8:17:27<13:44:41] +[titan] 2025-07-09 21:52:30,212 - root - INFO - step: 37630 loss: 16.7025 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.1232 +[titan] 2025-07-09 21:52:30,212 - root - INFO - lr: 2.1861e-04 gnorm: 0.83 [ 8:17:31<13:44:37] +[titan] 2025-07-09 21:52:34,127 - root - INFO - step: 37635 loss: 16.4622 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.9340 +[titan] 2025-07-09 21:52:34,128 - root - INFO - lr: 2.1859e-04 gnorm: 0.82 [ 8:17:35<13:44:33] +[titan] 2025-07-09 21:52:38,050 - root - INFO - step: 37640 loss: 16.6131 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0529 +[titan] 2025-07-09 21:52:38,050 - root - INFO - lr: 2.1857e-04 gnorm: 0.74 [ 8:17:39<13:44:29] +[titan] 2025-07-09 21:52:41,947 - root - INFO - step: 37645 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.8589 +[titan] 2025-07-09 21:52:41,947 - root - INFO - lr: 2.1855e-04 gnorm: 0.79 [ 8:17:43<13:44:25] +[titan] 2025-07-09 21:52:45,071 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:52:45,863 - root - INFO - step: 37650 loss: 16.7891 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.1979 +[titan] 2025-07-09 21:52:45,863 - root - INFO - lr: 2.1853e-04 gnorm: 0.78 [ 8:17:47<13:44:21] +[titan] 2025-07-09 21:52:49,842 - root - INFO - step: 37655 loss: 16.5599 memory: 44.58GiB(31.99%) tps: 82,357 tflops: 284.23 mfu: 28.74% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0111 +[titan] 2025-07-09 21:52:49,842 - root - INFO - lr: 2.1851e-04 gnorm: 0.73 [ 8:17:51<13:44:17] +[titan] 2025-07-09 21:52:53,754 - root - INFO - step: 37660 loss: 16.6190 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0522 +[titan] 2025-07-09 21:52:53,755 - root - INFO - lr: 2.1849e-04 gnorm: 0.71 [ 8:17:54<13:44:13] +[titan] 2025-07-09 21:52:57,658 - root - INFO - step: 37665 loss: 16.6221 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0630 +[titan] 2025-07-09 21:52:57,658 - root - INFO - lr: 2.1847e-04 gnorm: 0.84 [ 8:17:58<13:44:09] +[titan] 2025-07-09 21:53:01,565 - root - INFO - step: 37670 loss: 16.8425 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6164 global_avg_mtp_loss: 14.2262 +[titan] 2025-07-09 21:53:01,565 - root - INFO - lr: 2.1846e-04 gnorm: 0.74 [ 8:18:02<13:44:04] +[titan] 2025-07-09 21:53:05,501 - root - INFO - step: 37675 loss: 16.4195 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.8827 +[titan] 2025-07-09 21:53:05,501 - root - INFO - lr: 2.1844e-04 gnorm: 0.74 [ 8:18:06<13:44:00] +[titan] 2025-07-09 21:53:09,451 - root - INFO - step: 37680 loss: 16.7394 memory: 44.58GiB(31.99%) tps: 82,969 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1500 +[titan] 2025-07-09 21:53:09,451 - root - INFO - lr: 2.1842e-04 gnorm: 0.76 [ 8:18:10<13:43:56] +[titan] 2025-07-09 21:53:13,387 - root - INFO - step: 37685 loss: 16.6275 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0682 +[titan] 2025-07-09 21:53:13,387 - root - INFO - lr: 2.1840e-04 gnorm: 0.82 [ 8:18:14<13:43:52] +[titan] 2025-07-09 21:53:17,281 - root - INFO - step: 37690 loss: 16.5728 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0174 +[titan] 2025-07-09 21:53:17,281 - root - INFO - lr: 2.1838e-04 gnorm: 0.80 [ 8:18:18<13:43:48] +[titan] 2025-07-09 21:53:21,186 - root - INFO - step: 37695 loss: 16.6364 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.0576 +[titan] 2025-07-09 21:53:21,187 - root - INFO - lr: 2.1836e-04 gnorm: 0.75 [ 8:18:22<13:43:44] +[titan] 2025-07-09 21:53:24,310 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:53:25,110 - root - INFO - step: 37700 loss: 16.7955 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.2073 +[titan] 2025-07-09 21:53:25,110 - root - INFO - lr: 2.1834e-04 gnorm: 0.75 [ 8:18:26<13:43:40] +[titan] 2025-07-09 21:53:29,021 - root - INFO - step: 37705 loss: 16.5333 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 13.9752 +[titan] 2025-07-09 21:53:29,021 - root - INFO - lr: 2.1832e-04 gnorm: 0.85 [ 8:18:30<13:43:36] +[titan] 2025-07-09 21:53:32,943 - root - INFO - step: 37710 loss: 16.6373 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0580 +[titan] 2025-07-09 21:53:32,943 - root - INFO - lr: 2.1830e-04 gnorm: 0.84 [ 8:18:34<13:43:32] +[titan] 2025-07-09 21:53:36,873 - root - INFO - step: 37715 loss: 16.6781 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5927 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-09 21:53:36,874 - root - INFO - lr: 2.1828e-04 gnorm: 0.79 [ 8:18:38<13:43:28] +[titan] 2025-07-09 21:53:40,805 - root - INFO - step: 37720 loss: 16.5630 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 13.9992 +[titan] 2025-07-09 21:53:40,805 - root - INFO - lr: 2.1826e-04 gnorm: 0.75 [ 8:18:41<13:43:24] +[titan] 2025-07-09 21:53:44,714 - root - INFO - step: 37725 loss: 16.7051 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.1272 +[titan] 2025-07-09 21:53:44,714 - root - INFO - lr: 2.1824e-04 gnorm: 0.73 [ 8:18:45<13:43:20] +[titan] 2025-07-09 21:53:48,629 - root - INFO - step: 37730 loss: 16.4552 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9201 +[titan] 2025-07-09 21:53:48,629 - root - INFO - lr: 2.1822e-04 gnorm: 0.74 [ 8:18:49<13:43:16] +[titan] 2025-07-09 21:53:52,590 - root - INFO - step: 37735 loss: 16.7199 memory: 44.58GiB(31.99%) tps: 82,720 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.1312 +[titan] 2025-07-09 21:53:52,591 - root - INFO - lr: 2.1820e-04 gnorm: 0.81 [ 8:18:53<13:43:12] +[titan] 2025-07-09 21:53:56,485 - root - INFO - step: 37740 loss: 16.5218 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9732 +[titan] 2025-07-09 21:53:56,486 - root - INFO - lr: 2.1818e-04 gnorm: 0.77 [ 8:18:57<13:43:08] +[titan] 2025-07-09 21:54:00,401 - root - INFO - step: 37745 loss: 16.6136 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.0480 +[titan] 2025-07-09 21:54:00,402 - root - INFO - lr: 2.1816e-04 gnorm: 0.75 [ 8:19:01<13:43:04] +[titan] 2025-07-09 21:54:03,574 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:54:04,365 - root - INFO - step: 37750 loss: 16.6774 memory: 44.58GiB(31.99%) tps: 82,692 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-09 21:54:04,365 - root - INFO - lr: 2.1814e-04 gnorm: 0.78 [ 8:19:05<13:43:00] +[titan] 2025-07-09 21:54:08,267 - root - INFO - step: 37755 loss: 16.6469 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5727 global_avg_mtp_loss: 14.0742 +[titan] 2025-07-09 21:54:08,267 - root - INFO - lr: 2.1812e-04 gnorm: 0.81 [ 8:19:09<13:42:56] +[titan] 2025-07-09 21:54:12,172 - root - INFO - step: 37760 loss: 16.8049 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6177 global_avg_mtp_loss: 14.1872 +[titan] 2025-07-09 21:54:12,172 - root - INFO - lr: 2.1810e-04 gnorm: 0.76 [ 8:19:13<13:42:52] +[titan] 2025-07-09 21:54:16,076 - root - INFO - step: 37765 loss: 16.4427 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9017 +[titan] 2025-07-09 21:54:16,076 - root - INFO - lr: 2.1808e-04 gnorm: 0.77 [ 8:19:17<13:42:48] +[titan] 2025-07-09 21:54:19,970 - root - INFO - step: 37770 loss: 16.8137 memory: 44.58GiB(31.99%) tps: 84,151 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5988 global_avg_mtp_loss: 14.2149 +[titan] 2025-07-09 21:54:19,971 - root - INFO - lr: 2.1806e-04 gnorm: 0.84 [ 8:19:21<13:42:44] +[titan] 2025-07-09 21:54:23,887 - root - INFO - step: 37775 loss: 16.6800 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5701 global_avg_mtp_loss: 14.1099 +[titan] 2025-07-09 21:54:23,887 - root - INFO - lr: 2.1804e-04 gnorm: 0.75 [ 8:19:25<13:42:40] +[titan] 2025-07-09 21:54:27,797 - root - INFO - step: 37780 loss: 16.6717 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5766 global_avg_mtp_loss: 14.0951 +[titan] 2025-07-09 21:54:27,797 - root - INFO - lr: 2.1802e-04 gnorm: 0.75 [ 8:19:28<13:42:36] +[titan] 2025-07-09 21:54:31,713 - root - INFO - step: 37785 loss: 16.7427 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1520 +[titan] 2025-07-09 21:54:31,713 - root - INFO - lr: 2.1800e-04 gnorm: 0.78 [ 8:19:32<13:42:31] +[titan] 2025-07-09 21:54:35,623 - root - INFO - step: 37790 loss: 16.4985 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5381 global_avg_mtp_loss: 13.9604 +[titan] 2025-07-09 21:54:35,623 - root - INFO - lr: 2.1798e-04 gnorm: 0.81 [ 8:19:36<13:42:27] +[titan] 2025-07-09 21:54:39,525 - root - INFO - step: 37795 loss: 16.4071 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-09 21:54:39,525 - root - INFO - lr: 2.1796e-04 gnorm: 0.76 [ 8:19:40<13:42:23] +[titan] 2025-07-09 21:54:42,639 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:54:43,430 - root - INFO - step: 37800 loss: 16.8672 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.6240 global_avg_mtp_loss: 14.2431 +[titan] 2025-07-09 21:54:43,430 - root - INFO - lr: 2.1794e-04 gnorm: 0.85 [ 8:19:44<13:42:19] +[titan] 2025-07-09 21:54:47,361 - root - INFO - step: 37805 loss: 16.7446 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1508 +[titan] 2025-07-09 21:54:47,362 - root - INFO - lr: 2.1792e-04 gnorm: 0.76 [ 8:19:48<13:42:15] +[titan] 2025-07-09 21:54:51,258 - root - INFO - step: 37810 loss: 16.5694 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 14.0002 +[titan] 2025-07-09 21:54:51,258 - root - INFO - lr: 2.1790e-04 gnorm: 0.78 [ 8:19:52<13:42:11] +[titan] 2025-07-09 21:54:55,180 - root - INFO - step: 37815 loss: 16.7245 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1290 +[titan] 2025-07-09 21:54:55,181 - root - INFO - lr: 2.1788e-04 gnorm: 0.76 [ 8:19:56<13:42:07] +[titan] 2025-07-09 21:54:59,101 - root - INFO - step: 37820 loss: 16.6413 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0775 +[titan] 2025-07-09 21:54:59,101 - root - INFO - lr: 2.1786e-04 gnorm: 0.76 [ 8:20:00<13:42:03] +[titan] 2025-07-09 21:55:03,006 - root - INFO - step: 37825 loss: 16.8024 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6015 global_avg_mtp_loss: 14.2009 +[titan] 2025-07-09 21:55:03,006 - root - INFO - lr: 2.1784e-04 gnorm: 0.78 [ 8:20:04<13:41:59] +[titan] 2025-07-09 21:55:06,901 - root - INFO - step: 37830 loss: 16.5658 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-09 21:55:06,902 - root - INFO - lr: 2.1783e-04 gnorm: 0.84 [ 8:20:08<13:41:55] +[titan] 2025-07-09 21:55:10,799 - root - INFO - step: 37835 loss: 16.7086 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.1212 +[titan] 2025-07-09 21:55:10,799 - root - INFO - lr: 2.1781e-04 gnorm: 0.77 [ 8:20:11<13:41:51] +[titan] 2025-07-09 21:55:14,696 - root - INFO - step: 37840 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.8800 +[titan] 2025-07-09 21:55:14,697 - root - INFO - lr: 2.1779e-04 gnorm: 0.81 [ 8:20:15<13:41:47] +[titan] 2025-07-09 21:55:18,612 - root - INFO - step: 37845 loss: 16.8896 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.2809 +[titan] 2025-07-09 21:55:18,612 - root - INFO - lr: 2.1777e-04 gnorm: 0.76 [ 8:20:19<13:41:43] +[titan] 2025-07-09 21:55:21,725 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:55:22,514 - root - INFO - step: 37850 loss: 16.9522 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.6305 global_avg_mtp_loss: 14.3217 +[titan] 2025-07-09 21:55:22,514 - root - INFO - lr: 2.1775e-04 gnorm: 0.78 [ 8:20:23<13:41:39] +[titan] 2025-07-09 21:55:26,429 - root - INFO - step: 37855 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6455 global_avg_mtp_loss: 14.2945 +[titan] 2025-07-09 21:55:26,429 - root - INFO - lr: 2.1773e-04 gnorm: 0.78 [ 8:20:27<13:41:35] +[titan] 2025-07-09 21:55:30,355 - root - INFO - step: 37860 loss: 16.3866 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.8583 +[titan] 2025-07-09 21:55:30,355 - root - INFO - lr: 2.1771e-04 gnorm: 0.79 [ 8:20:31<13:41:31] +[titan] 2025-07-09 21:55:34,269 - root - INFO - step: 37865 loss: 16.6647 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.0905 +[titan] 2025-07-09 21:55:34,269 - root - INFO - lr: 2.1769e-04 gnorm: 0.78 [ 8:20:35<13:41:27] +[titan] 2025-07-09 21:55:38,223 - root - INFO - step: 37870 loss: 16.3829 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.8578 +[titan] 2025-07-09 21:55:38,223 - root - INFO - lr: 2.1767e-04 gnorm: 0.78 [ 8:20:39<13:41:23] +[titan] 2025-07-09 21:55:42,143 - root - INFO - step: 37875 loss: 16.6988 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1135 +[titan] 2025-07-09 21:55:42,144 - root - INFO - lr: 2.1765e-04 gnorm: 0.77 [ 8:20:43<13:41:18] +[titan] 2025-07-09 21:55:46,076 - root - INFO - step: 37880 loss: 16.9227 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6142 global_avg_mtp_loss: 14.3085 +[titan] 2025-07-09 21:55:46,077 - root - INFO - lr: 2.1763e-04 gnorm: 0.81 [ 8:20:47<13:41:14] +[titan] 2025-07-09 21:55:50,063 - root - INFO - step: 37885 loss: 16.8681 memory: 44.58GiB(31.99%) tps: 82,206 tflops: 283.71 mfu: 28.69% global_avg_ntp_loss: 2.6223 global_avg_mtp_loss: 14.2459 +[titan] 2025-07-09 21:55:50,063 - root - INFO - lr: 2.1761e-04 gnorm: 0.78 [ 8:20:51<13:41:11] +[titan] 2025-07-09 21:55:52,570 - root - INFO - Dumping profiler traces at step 37888 +[titan] 2025-07-09 21:55:52,604 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 21:55:54,203 - root - INFO - step: 37890 loss: 16.6618 memory: 44.58GiB(31.99%) tps: 79,153 tflops: 273.17 mfu: 27.62% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.0779 +[titan] 2025-07-09 21:55:54,203 - root - INFO - lr: 2.1759e-04 gnorm: 0.81 [ 8:20:55<13:41:07] +[titan] 2025-07-09 21:55:58,128 - root - INFO - step: 37895 loss: 16.5322 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 13.9645 +[titan] 2025-07-09 21:55:58,128 - root - INFO - lr: 2.1757e-04 gnorm: 0.78 [ 8:20:59<13:41:03] +[titan] 2025-07-09 21:56:01,316 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:56:02,111 - root - INFO - step: 37900 loss: 16.4514 memory: 44.58GiB(31.99%) tps: 82,277 tflops: 283.95 mfu: 28.71% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9134 +[titan] 2025-07-09 21:56:02,111 - root - INFO - lr: 2.1755e-04 gnorm: 0.75 [ 8:21:03<13:40:59] +[titan] 2025-07-09 21:56:06,033 - root - INFO - step: 37905 loss: 16.6109 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5680 global_avg_mtp_loss: 14.0429 +[titan] 2025-07-09 21:56:06,033 - root - INFO - lr: 2.1753e-04 gnorm: 0.77 [ 8:21:07<13:40:55] +[titan] 2025-07-09 21:56:09,951 - root - INFO - step: 37910 loss: 16.9451 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6278 global_avg_mtp_loss: 14.3173 +[titan] 2025-07-09 21:56:09,952 - root - INFO - lr: 2.1751e-04 gnorm: 0.82 [ 8:21:11<13:40:51] +[titan] 2025-07-09 21:56:13,891 - root - INFO - step: 37915 loss: 16.6604 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0890 +[titan] 2025-07-09 21:56:13,891 - root - INFO - lr: 2.1749e-04 gnorm: 0.76 [ 8:21:15<13:40:47] +[titan] 2025-07-09 21:56:17,836 - root - INFO - step: 37920 loss: 16.6225 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0567 +[titan] 2025-07-09 21:56:17,837 - root - INFO - lr: 2.1747e-04 gnorm: 0.76 [ 8:21:19<13:40:43] +[titan] 2025-07-09 21:56:21,757 - root - INFO - step: 37925 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6085 global_avg_mtp_loss: 14.2592 +[titan] 2025-07-09 21:56:21,757 - root - INFO - lr: 2.1745e-04 gnorm: 0.73 [ 8:21:22<13:40:39] +[titan] 2025-07-09 21:56:25,670 - root - INFO - step: 37930 loss: 16.4819 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 13.9348 +[titan] 2025-07-09 21:56:25,670 - root - INFO - lr: 2.1743e-04 gnorm: 0.78 [ 8:21:26<13:40:35] +[titan] 2025-07-09 21:56:29,582 - root - INFO - step: 37935 loss: 16.5247 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9820 +[titan] 2025-07-09 21:56:29,582 - root - INFO - lr: 2.1741e-04 gnorm: 0.82 [ 8:21:30<13:40:31] +[titan] 2025-07-09 21:56:33,477 - root - INFO - step: 37940 loss: 16.6876 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.1147 +[titan] 2025-07-09 21:56:33,477 - root - INFO - lr: 2.1739e-04 gnorm: 0.78 [ 8:21:34<13:40:27] +[titan] 2025-07-09 21:56:37,385 - root - INFO - step: 37945 loss: 16.6819 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.0991 +[titan] 2025-07-09 21:56:37,385 - root - INFO - lr: 2.1737e-04 gnorm: 0.76 [ 8:21:38<13:40:22] +[titan] 2025-07-09 21:56:40,494 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:56:41,284 - root - INFO - step: 37950 loss: 16.8414 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.2448 +[titan] 2025-07-09 21:56:41,284 - root - INFO - lr: 2.1735e-04 gnorm: 0.74 [ 8:21:42<13:40:18] +[titan] 2025-07-09 21:56:45,183 - root - INFO - step: 37955 loss: 16.5694 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-09 21:56:45,183 - root - INFO - lr: 2.1733e-04 gnorm: 0.73 [ 8:21:46<13:40:14] +[titan] 2025-07-09 21:56:49,087 - root - INFO - step: 37960 loss: 16.5732 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0200 +[titan] 2025-07-09 21:56:49,087 - root - INFO - lr: 2.1731e-04 gnorm: 0.78 [ 8:21:50<13:40:10] +[titan] 2025-07-09 21:56:52,986 - root - INFO - step: 37965 loss: 16.5841 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0264 +[titan] 2025-07-09 21:56:52,986 - root - INFO - lr: 2.1729e-04 gnorm: 0.75 [ 8:21:54<13:40:06] +[titan] 2025-07-09 21:56:56,912 - root - INFO - step: 37970 loss: 16.7503 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.1651 +[titan] 2025-07-09 21:56:56,912 - root - INFO - lr: 2.1727e-04 gnorm: 0.75 [ 8:21:58<13:40:02] +[titan] 2025-07-09 21:57:00,848 - root - INFO - step: 37975 loss: 16.6262 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.0485 +[titan] 2025-07-09 21:57:00,848 - root - INFO - lr: 2.1725e-04 gnorm: 0.83 [ 8:22:02<13:39:58] +[titan] 2025-07-09 21:57:04,782 - root - INFO - step: 37980 loss: 16.6467 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0817 +[titan] 2025-07-09 21:57:04,782 - root - INFO - lr: 2.1723e-04 gnorm: 0.76 [ 8:22:05<13:39:54] +[titan] 2025-07-09 21:57:08,704 - root - INFO - step: 37985 loss: 16.8880 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.6137 global_avg_mtp_loss: 14.2743 +[titan] 2025-07-09 21:57:08,704 - root - INFO - lr: 2.1721e-04 gnorm: 0.77 [ 8:22:09<13:39:50] +[titan] 2025-07-09 21:57:12,660 - root - INFO - step: 37990 loss: 16.8575 memory: 44.58GiB(31.99%) tps: 82,834 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.6102 global_avg_mtp_loss: 14.2473 +[titan] 2025-07-09 21:57:12,661 - root - INFO - lr: 2.1719e-04 gnorm: 0.80 [ 8:22:13<13:39:46] +[titan] 2025-07-09 21:57:16,584 - root - INFO - step: 37995 loss: 16.5841 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0190 +[titan] 2025-07-09 21:57:16,584 - root - INFO - lr: 2.1717e-04 gnorm: 0.76 [ 8:22:17<13:39:42] +[titan] 2025-07-09 21:57:19,696 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:57:20,485 - root - INFO - step: 38000 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9509 +[titan] 2025-07-09 21:57:20,485 - root - INFO - lr: 2.1715e-04 gnorm: 0.81 [ 8:22:21<13:39:38] +[titan] 2025-07-09 21:57:24,391 - root - INFO - step: 38005 loss: 16.9686 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.6241 global_avg_mtp_loss: 14.3446 +[titan] 2025-07-09 21:57:24,391 - root - INFO - lr: 2.1713e-04 gnorm: 0.77 [ 8:22:25<13:39:34] +[titan] 2025-07-09 21:57:28,305 - root - INFO - step: 38010 loss: 16.7596 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1678 +[titan] 2025-07-09 21:57:28,305 - root - INFO - lr: 2.1711e-04 gnorm: 0.76 [ 8:22:29<13:39:30] +[titan] 2025-07-09 21:57:32,203 - root - INFO - step: 38015 loss: 16.6772 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.0942 +[titan] 2025-07-09 21:57:32,203 - root - INFO - lr: 2.1709e-04 gnorm: 0.73 [ 8:22:33<13:39:26] +[titan] 2025-07-09 21:57:36,104 - root - INFO - step: 38020 loss: 16.5610 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5470 global_avg_mtp_loss: 14.0140 +[titan] 2025-07-09 21:57:36,104 - root - INFO - lr: 2.1708e-04 gnorm: 0.78 [ 8:22:37<13:39:22] +[titan] 2025-07-09 21:57:40,022 - root - INFO - step: 38025 loss: 16.9423 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6320 global_avg_mtp_loss: 14.3103 +[titan] 2025-07-09 21:57:40,023 - root - INFO - lr: 2.1706e-04 gnorm: 0.79 [ 8:22:41<13:39:18] +[titan] 2025-07-09 21:57:43,934 - root - INFO - step: 38030 loss: 16.8477 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.6153 global_avg_mtp_loss: 14.2324 +[titan] 2025-07-09 21:57:43,934 - root - INFO - lr: 2.1704e-04 gnorm: 1.25 [ 8:22:45<13:39:14] +[titan] 2025-07-09 21:57:47,856 - root - INFO - step: 38035 loss: 16.4438 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9029 +[titan] 2025-07-09 21:57:47,857 - root - INFO - lr: 2.1702e-04 gnorm: 0.77 [ 8:22:49<13:39:10] +[titan] 2025-07-09 21:57:51,770 - root - INFO - step: 38040 loss: 16.7621 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5951 global_avg_mtp_loss: 14.1670 +[titan] 2025-07-09 21:57:51,770 - root - INFO - lr: 2.1700e-04 gnorm: 0.80 [ 8:22:52<13:39:06] +[titan] 2025-07-09 21:57:55,700 - root - INFO - step: 38045 loss: 16.8855 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6262 global_avg_mtp_loss: 14.2592 +[titan] 2025-07-09 21:57:55,700 - root - INFO - lr: 2.1698e-04 gnorm: 0.75 [ 8:22:56<13:39:01] +[titan] 2025-07-09 21:57:58,818 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:57:59,608 - root - INFO - step: 38050 loss: 16.5824 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5548 global_avg_mtp_loss: 14.0275 +[titan] 2025-07-09 21:57:59,608 - root - INFO - lr: 2.1696e-04 gnorm: 0.75 [ 8:23:00<13:38:57] +[titan] 2025-07-09 21:58:03,511 - root - INFO - step: 38055 loss: 16.4800 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9385 +[titan] 2025-07-09 21:58:03,511 - root - INFO - lr: 2.1694e-04 gnorm: 0.79 [ 8:23:04<13:38:53] +[titan] 2025-07-09 21:58:07,429 - root - INFO - step: 38060 loss: 16.7997 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5962 global_avg_mtp_loss: 14.2036 +[titan] 2025-07-09 21:58:07,429 - root - INFO - lr: 2.1692e-04 gnorm: 0.78 [ 8:23:08<13:38:49] +[titan] 2025-07-09 21:58:11,367 - root - INFO - step: 38065 loss: 16.4712 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 13.9245 +[titan] 2025-07-09 21:58:11,367 - root - INFO - lr: 2.1690e-04 gnorm: 0.73 [ 8:23:12<13:38:45] +[titan] 2025-07-09 21:58:15,268 - root - INFO - step: 38070 loss: 16.5033 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9591 +[titan] 2025-07-09 21:58:15,269 - root - INFO - lr: 2.1688e-04 gnorm: 0.76 [ 8:23:16<13:38:41] +[titan] 2025-07-09 21:58:19,177 - root - INFO - step: 38075 loss: 16.6409 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0747 +[titan] 2025-07-09 21:58:19,177 - root - INFO - lr: 2.1686e-04 gnorm: 0.75 [ 8:23:20<13:38:37] +[titan] 2025-07-09 21:58:23,114 - root - INFO - step: 38080 loss: 16.8839 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.6254 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 21:58:23,114 - root - INFO - lr: 2.1684e-04 gnorm: 0.81 [ 8:23:24<13:38:33] +[titan] 2025-07-09 21:58:27,015 - root - INFO - step: 38085 loss: 16.7660 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5932 global_avg_mtp_loss: 14.1728 +[titan] 2025-07-09 21:58:27,015 - root - INFO - lr: 2.1682e-04 gnorm: 0.77 [ 8:23:28<13:38:29] +[titan] 2025-07-09 21:58:30,929 - root - INFO - step: 38090 loss: 16.9006 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.2742 +[titan] 2025-07-09 21:58:30,929 - root - INFO - lr: 2.1680e-04 gnorm: 0.86 [ 8:23:32<13:38:25] +[titan] 2025-07-09 21:58:34,839 - root - INFO - step: 38095 loss: 16.3982 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.8783 +[titan] 2025-07-09 21:58:34,839 - root - INFO - lr: 2.1678e-04 gnorm: 0.74 [ 8:23:35<13:38:21] +[titan] 2025-07-09 21:58:37,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:58:38,737 - root - INFO - step: 38100 loss: 16.8856 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6067 global_avg_mtp_loss: 14.2788 +[titan] 2025-07-09 21:58:38,737 - root - INFO - lr: 2.1676e-04 gnorm: 0.76 [ 8:23:39<13:38:17] +[titan] 2025-07-09 21:58:42,676 - root - INFO - step: 38105 loss: 16.4925 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 13.9429 +[titan] 2025-07-09 21:58:42,676 - root - INFO - lr: 2.1674e-04 gnorm: 0.83 [ 8:23:43<13:38:13] +[titan] 2025-07-09 21:58:46,574 - root - INFO - step: 38110 loss: 16.7380 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.1446 +[titan] 2025-07-09 21:58:46,574 - root - INFO - lr: 2.1672e-04 gnorm: 0.80 [ 8:23:47<13:38:09] +[titan] 2025-07-09 21:58:50,516 - root - INFO - step: 38115 loss: 16.5330 memory: 44.58GiB(31.99%) tps: 83,127 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 13.9862 +[titan] 2025-07-09 21:58:50,516 - root - INFO - lr: 2.1670e-04 gnorm: 0.82 [ 8:23:51<13:38:05] +[titan] 2025-07-09 21:58:54,444 - root - INFO - step: 38120 loss: 16.6992 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.1289 +[titan] 2025-07-09 21:58:54,444 - root - INFO - lr: 2.1668e-04 gnorm: 0.77 [ 8:23:55<13:38:01] +[titan] 2025-07-09 21:58:58,343 - root - INFO - step: 38125 loss: 16.8576 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6070 global_avg_mtp_loss: 14.2506 +[titan] 2025-07-09 21:58:58,344 - root - INFO - lr: 2.1666e-04 gnorm: 0.80 [ 8:23:59<13:37:57] +[titan] 2025-07-09 21:59:02,229 - root - INFO - step: 38130 loss: 16.5163 memory: 44.58GiB(31.99%) tps: 84,348 tflops: 291.10 mfu: 29.43% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 13.9593 +[titan] 2025-07-09 21:59:02,229 - root - INFO - lr: 2.1664e-04 gnorm: 0.78 [ 8:24:03<13:37:53] +[titan] 2025-07-09 21:59:06,140 - root - INFO - step: 38135 loss: 16.4335 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.8922 +[titan] 2025-07-09 21:59:06,140 - root - INFO - lr: 2.1662e-04 gnorm: 0.73 [ 8:24:07<13:37:49] +[titan] 2025-07-09 21:59:10,039 - root - INFO - step: 38140 loss: 16.5868 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 21:59:10,039 - root - INFO - lr: 2.1660e-04 gnorm: 0.77 [ 8:24:11<13:37:44] +[titan] 2025-07-09 21:59:13,947 - root - INFO - step: 38145 loss: 16.6919 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1052 +[titan] 2025-07-09 21:59:13,947 - root - INFO - lr: 2.1658e-04 gnorm: 0.77 [ 8:24:15<13:37:40] +[titan] 2025-07-09 21:59:17,065 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:59:17,848 - root - INFO - step: 38150 loss: 16.6815 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5690 global_avg_mtp_loss: 14.1124 +[titan] 2025-07-09 21:59:17,848 - root - INFO - lr: 2.1656e-04 gnorm: 0.74 [ 8:24:18<13:37:36] +[titan] 2025-07-09 21:59:21,727 - root - INFO - step: 38155 loss: 16.8020 memory: 44.58GiB(31.99%) tps: 84,485 tflops: 291.57 mfu: 29.48% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.2074 +[titan] 2025-07-09 21:59:21,727 - root - INFO - lr: 2.1654e-04 gnorm: 0.76 [ 8:24:22<13:37:32] +[titan] 2025-07-09 21:59:25,648 - root - INFO - step: 38160 loss: 16.3908 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 13.8596 +[titan] 2025-07-09 21:59:25,648 - root - INFO - lr: 2.1652e-04 gnorm: 0.87 [ 8:24:26<13:37:28] +[titan] 2025-07-09 21:59:29,561 - root - INFO - step: 38165 loss: 16.7871 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1977 +[titan] 2025-07-09 21:59:29,561 - root - INFO - lr: 2.1650e-04 gnorm: 0.75 [ 8:24:30<13:37:24] +[titan] 2025-07-09 21:59:33,465 - root - INFO - step: 38170 loss: 16.8711 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.2432 +[titan] 2025-07-09 21:59:33,465 - root - INFO - lr: 2.1648e-04 gnorm: 0.79 [ 8:24:34<13:37:20] +[titan] 2025-07-09 21:59:37,382 - root - INFO - step: 38175 loss: 16.7425 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1456 +[titan] 2025-07-09 21:59:37,382 - root - INFO - lr: 2.1646e-04 gnorm: 0.79 [ 8:24:38<13:37:16] +[titan] 2025-07-09 21:59:41,277 - root - INFO - step: 38180 loss: 16.5835 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0282 +[titan] 2025-07-09 21:59:41,277 - root - INFO - lr: 2.1644e-04 gnorm: 0.79 [ 8:24:42<13:37:12] +[titan] 2025-07-09 21:59:45,193 - root - INFO - step: 38185 loss: 16.4714 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.9289 +[titan] 2025-07-09 21:59:45,193 - root - INFO - lr: 2.1642e-04 gnorm: 0.89 [ 8:24:46<13:37:08] +[titan] 2025-07-09 21:59:49,118 - root - INFO - step: 38190 loss: 16.6054 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 14.0498 +[titan] 2025-07-09 21:59:49,118 - root - INFO - lr: 2.1640e-04 gnorm: 0.77 [ 8:24:50<13:37:04] +[titan] 2025-07-09 21:59:53,028 - root - INFO - step: 38195 loss: 16.3865 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.8540 +[titan] 2025-07-09 21:59:53,028 - root - INFO - lr: 2.1638e-04 gnorm: 0.73 [ 8:24:54<13:37:00] +[titan] 2025-07-09 21:59:56,160 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 21:59:56,963 - root - INFO - step: 38200 loss: 16.5271 memory: 44.58GiB(31.99%) tps: 83,277 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9807 +[titan] 2025-07-09 21:59:56,963 - root - INFO - lr: 2.1636e-04 gnorm: 0.83 [ 8:24:58<13:36:56] +[titan] 2025-07-09 22:00:00,875 - root - INFO - step: 38205 loss: 16.6026 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0316 +[titan] 2025-07-09 22:00:00,875 - root - INFO - lr: 2.1634e-04 gnorm: 0.75 [ 8:25:02<13:36:52] +[titan] 2025-07-09 22:00:04,773 - root - INFO - step: 38210 loss: 16.6762 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5810 global_avg_mtp_loss: 14.0952 +[titan] 2025-07-09 22:00:04,773 - root - INFO - lr: 2.1632e-04 gnorm: 0.77 [ 8:25:05<13:36:48] +[titan] 2025-07-09 22:00:08,722 - root - INFO - step: 38215 loss: 16.5057 memory: 44.58GiB(31.99%) tps: 82,991 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 13.9488 +[titan] 2025-07-09 22:00:08,722 - root - INFO - lr: 2.1630e-04 gnorm: 0.77 [ 8:25:09<13:36:44] +[titan] 2025-07-09 22:00:12,634 - root - INFO - step: 38220 loss: 16.7199 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.1246 +[titan] 2025-07-09 22:00:12,635 - root - INFO - lr: 2.1628e-04 gnorm: 0.79 [ 8:25:13<13:36:40] +[titan] 2025-07-09 22:00:16,531 - root - INFO - step: 38225 loss: 16.9673 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6646 global_avg_mtp_loss: 14.3028 +[titan] 2025-07-09 22:00:16,531 - root - INFO - lr: 2.1626e-04 gnorm: 0.86 [ 8:25:17<13:36:36] +[titan] 2025-07-09 22:00:20,437 - root - INFO - step: 38230 loss: 16.5921 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.0216 +[titan] 2025-07-09 22:00:20,437 - root - INFO - lr: 2.1624e-04 gnorm: 0.81 [ 8:25:21<13:36:31] +[titan] 2025-07-09 22:00:24,339 - root - INFO - step: 38235 loss: 16.5780 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0129 +[titan] 2025-07-09 22:00:24,339 - root - INFO - lr: 2.1622e-04 gnorm: 0.81 [ 8:25:25<13:36:27] +[titan] 2025-07-09 22:00:28,269 - root - INFO - step: 38240 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0824 +[titan] 2025-07-09 22:00:28,269 - root - INFO - lr: 2.1620e-04 gnorm: 0.80 [ 8:25:29<13:36:23] +[titan] 2025-07-09 22:00:32,178 - root - INFO - step: 38245 loss: 16.7703 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1779 +[titan] 2025-07-09 22:00:32,178 - root - INFO - lr: 2.1618e-04 gnorm: 0.73 [ 8:25:33<13:36:19] +[titan] 2025-07-09 22:00:35,303 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:00:36,092 - root - INFO - step: 38250 loss: 16.3870 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.8504 +[titan] 2025-07-09 22:00:36,092 - root - INFO - lr: 2.1616e-04 gnorm: 0.85 [ 8:25:37<13:36:15] +[titan] 2025-07-09 22:00:39,988 - root - INFO - step: 38255 loss: 16.8443 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.6197 global_avg_mtp_loss: 14.2246 +[titan] 2025-07-09 22:00:39,988 - root - INFO - lr: 2.1614e-04 gnorm: 0.77 [ 8:25:41<13:36:11] +[titan] 2025-07-09 22:00:43,891 - root - INFO - step: 38260 loss: 16.6110 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0428 +[titan] 2025-07-09 22:00:43,891 - root - INFO - lr: 2.1613e-04 gnorm: 0.78 [ 8:25:45<13:36:07] +[titan] 2025-07-09 22:00:47,841 - root - INFO - step: 38265 loss: 16.8071 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.2048 +[titan] 2025-07-09 22:00:47,841 - root - INFO - lr: 2.1611e-04 gnorm: 0.75 [ 8:25:48<13:36:03] +[titan] 2025-07-09 22:00:51,779 - root - INFO - step: 38270 loss: 16.9704 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6345 global_avg_mtp_loss: 14.3359 +[titan] 2025-07-09 22:00:51,779 - root - INFO - lr: 2.1609e-04 gnorm: 0.79 [ 8:25:52<13:35:59] +[titan] 2025-07-09 22:00:55,711 - root - INFO - step: 38275 loss: 16.7056 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1192 +[titan] 2025-07-09 22:00:55,711 - root - INFO - lr: 2.1607e-04 gnorm: 0.77 [ 8:25:56<13:35:55] +[titan] 2025-07-09 22:00:59,650 - root - INFO - step: 38280 loss: 16.6575 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0845 +[titan] 2025-07-09 22:00:59,650 - root - INFO - lr: 2.1605e-04 gnorm: 0.70 [ 8:26:00<13:35:51] +[titan] 2025-07-09 22:01:03,625 - root - INFO - step: 38285 loss: 16.8349 memory: 44.58GiB(31.99%) tps: 82,432 tflops: 284.49 mfu: 28.77% global_avg_ntp_loss: 2.6194 global_avg_mtp_loss: 14.2155 +[titan] 2025-07-09 22:01:03,626 - root - INFO - lr: 2.1603e-04 gnorm: 0.77 [ 8:26:04<13:35:47] +[titan] 2025-07-09 22:01:07,539 - root - INFO - step: 38290 loss: 16.7520 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5926 global_avg_mtp_loss: 14.1594 +[titan] 2025-07-09 22:01:07,539 - root - INFO - lr: 2.1601e-04 gnorm: 0.76 [ 8:26:08<13:35:43] +[titan] 2025-07-09 22:01:11,463 - root - INFO - step: 38295 loss: 16.5014 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9523 +[titan] 2025-07-09 22:01:11,463 - root - INFO - lr: 2.1599e-04 gnorm: 0.76 [ 8:26:12<13:35:39] +[titan] 2025-07-09 22:01:14,598 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:01:15,401 - root - INFO - step: 38300 loss: 16.5910 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 22:01:15,401 - root - INFO - lr: 2.1597e-04 gnorm: 0.76 [ 8:26:16<13:35:35] +[titan] 2025-07-09 22:01:19,322 - root - INFO - step: 38305 loss: 16.6134 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0461 +[titan] 2025-07-09 22:01:19,322 - root - INFO - lr: 2.1595e-04 gnorm: 0.78 [ 8:26:20<13:35:31] +[titan] 2025-07-09 22:01:23,263 - root - INFO - step: 38310 loss: 16.7805 memory: 44.58GiB(31.99%) tps: 83,147 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5980 global_avg_mtp_loss: 14.1825 +[titan] 2025-07-09 22:01:23,264 - root - INFO - lr: 2.1593e-04 gnorm: 0.78 [ 8:26:24<13:35:27] +[titan] 2025-07-09 22:01:27,204 - root - INFO - step: 38315 loss: 16.7741 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.6078 global_avg_mtp_loss: 14.1663 +[titan] 2025-07-09 22:01:27,204 - root - INFO - lr: 2.1591e-04 gnorm: 0.80 [ 8:26:28<13:35:23] +[titan] 2025-07-09 22:01:31,129 - root - INFO - step: 38320 loss: 16.7158 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.1285 +[titan] 2025-07-09 22:01:31,130 - root - INFO - lr: 2.1589e-04 gnorm: 0.75 [ 8:26:32<13:35:19] +[titan] 2025-07-09 22:01:35,064 - root - INFO - step: 38325 loss: 16.4751 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9354 +[titan] 2025-07-09 22:01:35,064 - root - INFO - lr: 2.1587e-04 gnorm: 0.78 [ 8:26:36<13:35:15] +[titan] 2025-07-09 22:01:38,993 - root - INFO - step: 38330 loss: 16.3701 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-09 22:01:38,993 - root - INFO - lr: 2.1585e-04 gnorm: 0.76 [ 8:26:40<13:35:11] +[titan] 2025-07-09 22:01:42,926 - root - INFO - step: 38335 loss: 16.8555 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.6101 global_avg_mtp_loss: 14.2455 +[titan] 2025-07-09 22:01:42,926 - root - INFO - lr: 2.1583e-04 gnorm: 0.80 [ 8:26:44<13:35:07] +[titan] 2025-07-09 22:01:46,840 - root - INFO - step: 38340 loss: 16.7671 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1867 +[titan] 2025-07-09 22:01:46,840 - root - INFO - lr: 2.1581e-04 gnorm: 0.77 [ 8:26:47<13:35:03] +[titan] 2025-07-09 22:01:50,746 - root - INFO - step: 38345 loss: 16.7259 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.1515 +[titan] 2025-07-09 22:01:50,746 - root - INFO - lr: 2.1579e-04 gnorm: 0.77 [ 8:26:51<13:34:59] +[titan] 2025-07-09 22:01:53,852 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:01:54,645 - root - INFO - step: 38350 loss: 16.1456 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.6430 +[titan] 2025-07-09 22:01:54,645 - root - INFO - lr: 2.1577e-04 gnorm: 0.87 [ 8:26:55<13:34:55] +[titan] 2025-07-09 22:01:58,566 - root - INFO - step: 38355 loss: 16.6903 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5785 global_avg_mtp_loss: 14.1118 +[titan] 2025-07-09 22:01:58,567 - root - INFO - lr: 2.1575e-04 gnorm: 0.72 [ 8:26:59<13:34:51] +[titan] 2025-07-09 22:02:02,466 - root - INFO - step: 38360 loss: 16.8055 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.2039 +[titan] 2025-07-09 22:02:02,466 - root - INFO - lr: 2.1573e-04 gnorm: 0.78 [ 8:27:03<13:34:47] +[titan] 2025-07-09 22:02:06,415 - root - INFO - step: 38365 loss: 16.7230 memory: 44.58GiB(31.99%) tps: 82,986 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5812 global_avg_mtp_loss: 14.1418 +[titan] 2025-07-09 22:02:06,415 - root - INFO - lr: 2.1571e-04 gnorm: 0.78 [ 8:27:07<13:34:43] +[titan] 2025-07-09 22:02:10,327 - root - INFO - step: 38370 loss: 16.5026 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9574 +[titan] 2025-07-09 22:02:10,328 - root - INFO - lr: 2.1569e-04 gnorm: 0.78 [ 8:27:11<13:34:39] +[titan] 2025-07-09 22:02:14,271 - root - INFO - step: 38375 loss: 16.4626 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9283 +[titan] 2025-07-09 22:02:14,271 - root - INFO - lr: 2.1567e-04 gnorm: 0.75 [ 8:27:15<13:34:35] +[titan] 2025-07-09 22:02:18,187 - root - INFO - step: 38380 loss: 16.8501 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2365 +[titan] 2025-07-09 22:02:18,187 - root - INFO - lr: 2.1565e-04 gnorm: 0.74 [ 8:27:19<13:34:31] +[titan] 2025-07-09 22:02:22,133 - root - INFO - step: 38385 loss: 16.8362 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2213 +[titan] 2025-07-09 22:02:22,133 - root - INFO - lr: 2.1563e-04 gnorm: 0.74 [ 8:27:23<13:34:27] +[titan] 2025-07-09 22:02:26,060 - root - INFO - step: 38390 loss: 16.7861 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.6106 global_avg_mtp_loss: 14.1755 +[titan] 2025-07-09 22:02:26,061 - root - INFO - lr: 2.1561e-04 gnorm: 0.72 [ 8:27:27<13:34:23] +[titan] 2025-07-09 22:02:30,014 - root - INFO - step: 38395 loss: 16.7857 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.1923 +[titan] 2025-07-09 22:02:30,014 - root - INFO - lr: 2.1559e-04 gnorm: 0.74 [ 8:27:31<13:34:19] +[titan] 2025-07-09 22:02:33,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:02:34,039 - root - INFO - step: 38400 loss: 16.4617 memory: 44.58GiB(31.99%) tps: 81,414 tflops: 280.97 mfu: 28.41% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9274 +[titan] 2025-07-09 22:02:34,039 - root - INFO - lr: 2.1557e-04 gnorm: 0.73 [ 8:27:35<13:34:15] +[titan] 2025-07-09 22:02:34,196 - root - INFO - Dumping profiler traces at step 38400 +[titan] 2025-07-09 22:02:34,229 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:02:38,161 - root - INFO - step: 38405 loss: 16.7022 memory: 44.58GiB(31.99%) tps: 79,497 tflops: 274.36 mfu: 27.74% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1192 +[titan] 2025-07-09 22:02:38,162 - root - INFO - lr: 2.1555e-04 gnorm: 0.72 [ 8:27:39<13:34:11] +[titan] 2025-07-09 22:02:42,073 - root - INFO - step: 38410 loss: 16.6628 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.1018 +[titan] 2025-07-09 22:02:42,073 - root - INFO - lr: 2.1553e-04 gnorm: 0.73 [ 8:27:43<13:34:07] +[titan] 2025-07-09 22:02:45,978 - root - INFO - step: 38415 loss: 16.3937 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.8589 +[titan] 2025-07-09 22:02:45,978 - root - INFO - lr: 2.1551e-04 gnorm: 0.79 [ 8:27:47<13:34:03] +[titan] 2025-07-09 22:02:49,932 - root - INFO - step: 38420 loss: 16.7404 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.1553 +[titan] 2025-07-09 22:02:49,932 - root - INFO - lr: 2.1549e-04 gnorm: 0.81 [ 8:27:51<13:33:59] +[titan] 2025-07-09 22:02:53,836 - root - INFO - step: 38425 loss: 16.6023 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0296 +[titan] 2025-07-09 22:02:53,837 - root - INFO - lr: 2.1547e-04 gnorm: 0.77 [ 8:27:54<13:33:55] +[titan] 2025-07-09 22:02:57,735 - root - INFO - step: 38430 loss: 16.6025 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0297 +[titan] 2025-07-09 22:02:57,735 - root - INFO - lr: 2.1545e-04 gnorm: 0.80 [ 8:27:58<13:33:51] +[titan] 2025-07-09 22:03:01,655 - root - INFO - step: 38435 loss: 16.5562 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5669 global_avg_mtp_loss: 13.9893 +[titan] 2025-07-09 22:03:01,655 - root - INFO - lr: 2.1543e-04 gnorm: 0.83 [ 8:28:02<13:33:47] +[titan] 2025-07-09 22:03:05,569 - root - INFO - step: 38440 loss: 16.4727 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-09 22:03:05,569 - root - INFO - lr: 2.1541e-04 gnorm: 0.73 [ 8:28:06<13:33:43] +[titan] 2025-07-09 22:03:09,483 - root - INFO - step: 38445 loss: 16.9074 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.6269 global_avg_mtp_loss: 14.2805 +[titan] 2025-07-09 22:03:09,483 - root - INFO - lr: 2.1539e-04 gnorm: 0.72 [ 8:28:10<13:33:39] +[titan] 2025-07-09 22:03:12,612 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:03:13,409 - root - INFO - step: 38450 loss: 16.7641 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5878 global_avg_mtp_loss: 14.1763 +[titan] 2025-07-09 22:03:13,409 - root - INFO - lr: 2.1537e-04 gnorm: 0.74 [ 8:28:14<13:33:35] +[titan] 2025-07-09 22:03:17,336 - root - INFO - step: 38455 loss: 16.5605 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5541 global_avg_mtp_loss: 14.0064 +[titan] 2025-07-09 22:03:17,337 - root - INFO - lr: 2.1535e-04 gnorm: 0.79 [ 8:28:18<13:33:31] +[titan] 2025-07-09 22:03:21,263 - root - INFO - step: 38460 loss: 16.4955 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9508 +[titan] 2025-07-09 22:03:21,264 - root - INFO - lr: 2.1533e-04 gnorm: 0.79 [ 8:28:22<13:33:26] +[titan] 2025-07-09 22:03:25,181 - root - INFO - step: 38465 loss: 16.8904 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6425 global_avg_mtp_loss: 14.2479 +[titan] 2025-07-09 22:03:25,181 - root - INFO - lr: 2.1531e-04 gnorm: 0.82 [ 8:28:26<13:33:22] +[titan] 2025-07-09 22:03:29,140 - root - INFO - step: 38470 loss: 16.9706 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.6264 global_avg_mtp_loss: 14.3442 +[titan] 2025-07-09 22:03:29,140 - root - INFO - lr: 2.1529e-04 gnorm: 0.72 [ 8:28:30<13:33:18] +[titan] 2025-07-09 22:03:33,038 - root - INFO - step: 38475 loss: 16.6498 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.0720 +[titan] 2025-07-09 22:03:33,038 - root - INFO - lr: 2.1527e-04 gnorm: 0.88 [ 8:28:34<13:33:14] +[titan] 2025-07-09 22:03:36,925 - root - INFO - step: 38480 loss: 16.9597 memory: 44.58GiB(31.99%) tps: 84,300 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.6601 global_avg_mtp_loss: 14.2996 +[titan] 2025-07-09 22:03:36,926 - root - INFO - lr: 2.1525e-04 gnorm: 1.01 [ 8:28:38<13:33:10] +[titan] 2025-07-09 22:03:40,829 - root - INFO - step: 38485 loss: 16.3425 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8225 +[titan] 2025-07-09 22:03:40,829 - root - INFO - lr: 2.1523e-04 gnorm: 0.87 [ 8:28:41<13:33:06] +[titan] 2025-07-09 22:03:44,740 - root - INFO - step: 38490 loss: 16.6084 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.0363 +[titan] 2025-07-09 22:03:44,741 - root - INFO - lr: 2.1521e-04 gnorm: 0.85 [ 8:28:45<13:33:02] +[titan] 2025-07-09 22:03:48,643 - root - INFO - step: 38495 loss: 16.5765 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 14.0253 +[titan] 2025-07-09 22:03:48,643 - root - INFO - lr: 2.1519e-04 gnorm: 0.77 [ 8:28:49<13:32:58] +[titan] 2025-07-09 22:03:51,769 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:03:52,563 - root - INFO - step: 38500 loss: 16.4178 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.8845 +[titan] 2025-07-09 22:03:52,563 - root - INFO - lr: 2.1517e-04 gnorm: 0.84 [ 8:28:53<13:32:54] +[titan] 2025-07-09 22:03:56,462 - root - INFO - step: 38505 loss: 16.6978 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1138 +[titan] 2025-07-09 22:03:56,462 - root - INFO - lr: 2.1515e-04 gnorm: 0.78 [ 8:28:57<13:32:50] +[titan] 2025-07-09 22:04:00,369 - root - INFO - step: 38510 loss: 16.8733 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.6185 global_avg_mtp_loss: 14.2548 +[titan] 2025-07-09 22:04:00,370 - root - INFO - lr: 2.1513e-04 gnorm: 0.79 [ 8:29:01<13:32:46] +[titan] 2025-07-09 22:04:04,271 - root - INFO - step: 38515 loss: 16.5808 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 14.0305 +[titan] 2025-07-09 22:04:04,272 - root - INFO - lr: 2.1511e-04 gnorm: 0.79 [ 8:29:05<13:32:42] +[titan] 2025-07-09 22:04:08,192 - root - INFO - step: 38520 loss: 16.7244 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.1257 +[titan] 2025-07-09 22:04:08,193 - root - INFO - lr: 2.1509e-04 gnorm: 0.76 [ 8:29:09<13:32:38] +[titan] 2025-07-09 22:04:12,094 - root - INFO - step: 38525 loss: 16.7467 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.1527 +[titan] 2025-07-09 22:04:12,094 - root - INFO - lr: 2.1507e-04 gnorm: 0.72 [ 8:29:13<13:32:34] +[titan] 2025-07-09 22:04:16,001 - root - INFO - step: 38530 loss: 16.4729 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9336 +[titan] 2025-07-09 22:04:16,001 - root - INFO - lr: 2.1505e-04 gnorm: 0.74 [ 8:29:17<13:32:30] +[titan] 2025-07-09 22:04:19,934 - root - INFO - step: 38535 loss: 16.6016 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0393 +[titan] 2025-07-09 22:04:19,934 - root - INFO - lr: 2.1503e-04 gnorm: 0.74 [ 8:29:21<13:32:26] +[titan] 2025-07-09 22:04:23,832 - root - INFO - step: 38540 loss: 16.8405 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.12 mfu: 29.34% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.2318 +[titan] 2025-07-09 22:04:23,832 - root - INFO - lr: 2.1501e-04 gnorm: 0.79 [ 8:29:24<13:32:22] +[titan] 2025-07-09 22:04:27,759 - root - INFO - step: 38545 loss: 16.4057 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.8705 +[titan] 2025-07-09 22:04:27,759 - root - INFO - lr: 2.1499e-04 gnorm: 0.80 [ 8:29:28<13:32:18] +[titan] 2025-07-09 22:04:30,873 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:04:31,660 - root - INFO - step: 38550 loss: 16.7422 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1483 +[titan] 2025-07-09 22:04:31,660 - root - INFO - lr: 2.1497e-04 gnorm: 0.80 [ 8:29:32<13:32:14] +[titan] 2025-07-09 22:04:35,581 - root - INFO - step: 38555 loss: 16.9150 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6238 global_avg_mtp_loss: 14.2912 +[titan] 2025-07-09 22:04:35,581 - root - INFO - lr: 2.1495e-04 gnorm: 0.77 [ 8:29:36<13:32:09] +[titan] 2025-07-09 22:04:39,497 - root - INFO - step: 38560 loss: 16.6830 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-09 22:04:39,497 - root - INFO - lr: 2.1493e-04 gnorm: 0.75 [ 8:29:40<13:32:05] +[titan] 2025-07-09 22:04:43,425 - root - INFO - step: 38565 loss: 16.6516 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.0668 +[titan] 2025-07-09 22:04:43,425 - root - INFO - lr: 2.1491e-04 gnorm: 0.73 [ 8:29:44<13:32:01] +[titan] 2025-07-09 22:04:47,340 - root - INFO - step: 38570 loss: 16.5360 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 13.9879 +[titan] 2025-07-09 22:04:47,340 - root - INFO - lr: 2.1489e-04 gnorm: 0.83 [ 8:29:48<13:31:57] +[titan] 2025-07-09 22:04:51,242 - root - INFO - step: 38575 loss: 16.6828 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1042 +[titan] 2025-07-09 22:04:51,242 - root - INFO - lr: 2.1487e-04 gnorm: 0.73 [ 8:29:52<13:31:53] +[titan] 2025-07-09 22:04:55,188 - root - INFO - step: 38580 loss: 16.4699 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 13.9212 +[titan] 2025-07-09 22:04:55,189 - root - INFO - lr: 2.1485e-04 gnorm: 0.76 [ 8:29:56<13:31:49] +[titan] 2025-07-09 22:04:59,124 - root - INFO - step: 38585 loss: 16.6872 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.1134 +[titan] 2025-07-09 22:04:59,124 - root - INFO - lr: 2.1483e-04 gnorm: 0.74 [ 8:30:00<13:31:45] +[titan] 2025-07-09 22:05:03,049 - root - INFO - step: 38590 loss: 16.6996 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1117 +[titan] 2025-07-09 22:05:03,049 - root - INFO - lr: 2.1481e-04 gnorm: 0.77 [ 8:30:04<13:31:41] +[titan] 2025-07-09 22:05:06,952 - root - INFO - step: 38595 loss: 16.8087 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.1999 +[titan] 2025-07-09 22:05:06,952 - root - INFO - lr: 2.1479e-04 gnorm: 0.75 [ 8:30:08<13:31:37] +[titan] 2025-07-09 22:05:10,057 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:05:10,842 - root - INFO - step: 38600 loss: 16.7928 memory: 44.58GiB(31.99%) tps: 84,242 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.6137 global_avg_mtp_loss: 14.1791 +[titan] 2025-07-09 22:05:10,843 - root - INFO - lr: 2.1477e-04 gnorm: 0.80 [ 8:30:11<13:31:33] +[titan] 2025-07-09 22:05:14,774 - root - INFO - step: 38605 loss: 16.7131 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.1411 +[titan] 2025-07-09 22:05:14,774 - root - INFO - lr: 2.1475e-04 gnorm: 0.74 [ 8:30:15<13:31:29] +[titan] 2025-07-09 22:05:18,684 - root - INFO - step: 38610 loss: 16.6541 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-09 22:05:18,684 - root - INFO - lr: 2.1473e-04 gnorm: 0.78 [ 8:30:19<13:31:25] +[titan] 2025-07-09 22:05:22,593 - root - INFO - step: 38615 loss: 16.6229 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 14.0554 +[titan] 2025-07-09 22:05:22,593 - root - INFO - lr: 2.1471e-04 gnorm: 0.79 [ 8:30:23<13:31:21] +[titan] 2025-07-09 22:05:26,525 - root - INFO - step: 38620 loss: 16.4714 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 13.9187 +[titan] 2025-07-09 22:05:26,526 - root - INFO - lr: 2.1469e-04 gnorm: 0.85 [ 8:30:27<13:31:17] +[titan] 2025-07-09 22:05:30,435 - root - INFO - step: 38625 loss: 16.6283 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.0546 +[titan] 2025-07-09 22:05:30,435 - root - INFO - lr: 2.1467e-04 gnorm: 0.79 [ 8:30:31<13:31:13] +[titan] 2025-07-09 22:05:34,342 - root - INFO - step: 38630 loss: 16.4704 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 13.9183 +[titan] 2025-07-09 22:05:34,342 - root - INFO - lr: 2.1465e-04 gnorm: 0.77 [ 8:30:35<13:31:09] +[titan] 2025-07-09 22:05:38,242 - root - INFO - step: 38635 loss: 16.4385 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.8941 +[titan] 2025-07-09 22:05:38,242 - root - INFO - lr: 2.1463e-04 gnorm: 0.80 [ 8:30:39<13:31:05] +[titan] 2025-07-09 22:05:42,172 - root - INFO - step: 38640 loss: 16.5721 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0110 +[titan] 2025-07-09 22:05:42,173 - root - INFO - lr: 2.1461e-04 gnorm: 0.75 [ 8:30:43<13:31:01] +[titan] 2025-07-09 22:05:46,108 - root - INFO - step: 38645 loss: 16.6261 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.0575 +[titan] 2025-07-09 22:05:46,109 - root - INFO - lr: 2.1459e-04 gnorm: 0.79 [ 8:30:47<13:30:57] +[titan] 2025-07-09 22:05:49,243 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:05:50,030 - root - INFO - step: 38650 loss: 16.6926 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5806 global_avg_mtp_loss: 14.1121 +[titan] 2025-07-09 22:05:50,031 - root - INFO - lr: 2.1457e-04 gnorm: 0.78 [ 8:30:51<13:30:53] +[titan] 2025-07-09 22:05:54,042 - root - INFO - step: 38655 loss: 16.7241 memory: 44.58GiB(31.99%) tps: 81,695 tflops: 281.94 mfu: 28.51% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.1520 +[titan] 2025-07-09 22:05:54,042 - root - INFO - lr: 2.1455e-04 gnorm: 0.85 [ 8:30:55<13:30:49] +[titan] 2025-07-09 22:05:57,991 - root - INFO - step: 38660 loss: 16.5837 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 14.0254 +[titan] 2025-07-09 22:05:57,992 - root - INFO - lr: 2.1453e-04 gnorm: 0.80 [ 8:30:59<13:30:45] +[titan] 2025-07-09 22:06:01,918 - root - INFO - step: 38665 loss: 16.6744 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5766 global_avg_mtp_loss: 14.0979 +[titan] 2025-07-09 22:06:01,919 - root - INFO - lr: 2.1452e-04 gnorm: 0.83 [ 8:31:03<13:30:41] +[titan] 2025-07-09 22:06:05,815 - root - INFO - step: 38670 loss: 16.5965 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0431 +[titan] 2025-07-09 22:06:05,815 - root - INFO - lr: 2.1450e-04 gnorm: 0.77 [ 8:31:06<13:30:37] +[titan] 2025-07-09 22:06:09,731 - root - INFO - step: 38675 loss: 16.6903 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.1170 +[titan] 2025-07-09 22:06:09,731 - root - INFO - lr: 2.1448e-04 gnorm: 0.78 [ 8:31:10<13:30:33] +[titan] 2025-07-09 22:06:13,667 - root - INFO - step: 38680 loss: 16.4505 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9155 +[titan] 2025-07-09 22:06:13,667 - root - INFO - lr: 2.1446e-04 gnorm: 0.75 [ 8:31:14<13:30:29] +[titan] 2025-07-09 22:06:17,601 - root - INFO - step: 38685 loss: 16.6310 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5564 global_avg_mtp_loss: 14.0746 +[titan] 2025-07-09 22:06:17,602 - root - INFO - lr: 2.1444e-04 gnorm: 0.77 [ 8:31:18<13:30:25] +[titan] 2025-07-09 22:06:21,526 - root - INFO - step: 38690 loss: 16.6566 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0918 +[titan] 2025-07-09 22:06:21,526 - root - INFO - lr: 2.1442e-04 gnorm: 0.77 [ 8:31:22<13:30:21] +[titan] 2025-07-09 22:06:25,435 - root - INFO - step: 38695 loss: 16.8505 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.6218 global_avg_mtp_loss: 14.2288 +[titan] 2025-07-09 22:06:25,435 - root - INFO - lr: 2.1440e-04 gnorm: 0.80 [ 8:31:26<13:30:17] +[titan] 2025-07-09 22:06:28,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:06:29,352 - root - INFO - step: 38700 loss: 16.6888 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1066 +[titan] 2025-07-09 22:06:29,352 - root - INFO - lr: 2.1438e-04 gnorm: 0.75 [ 8:31:30<13:30:13] +[titan] 2025-07-09 22:06:33,359 - root - INFO - step: 38705 loss: 16.8171 memory: 44.58GiB(31.99%) tps: 81,768 tflops: 282.19 mfu: 28.53% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.2110 +[titan] 2025-07-09 22:06:33,360 - root - INFO - lr: 2.1436e-04 gnorm: 0.80 [ 8:31:34<13:30:09] +[titan] 2025-07-09 22:06:37,295 - root - INFO - step: 38710 loss: 16.5445 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9994 +[titan] 2025-07-09 22:06:37,295 - root - INFO - lr: 2.1434e-04 gnorm: 0.80 [ 8:31:38<13:30:05] +[titan] 2025-07-09 22:06:41,215 - root - INFO - step: 38715 loss: 16.3667 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-09 22:06:41,215 - root - INFO - lr: 2.1432e-04 gnorm: 0.71 [ 8:31:42<13:30:01] +[titan] 2025-07-09 22:06:45,167 - root - INFO - step: 38720 loss: 17.0345 memory: 44.58GiB(31.99%) tps: 82,924 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.6444 global_avg_mtp_loss: 14.3902 +[titan] 2025-07-09 22:06:45,167 - root - INFO - lr: 2.1430e-04 gnorm: 0.76 [ 8:31:46<13:29:57] +[titan] 2025-07-09 22:06:49,095 - root - INFO - step: 38725 loss: 16.6609 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 14.0819 +[titan] 2025-07-09 22:06:49,095 - root - INFO - lr: 2.1428e-04 gnorm: 0.76 [ 8:31:50<13:29:53] +[titan] 2025-07-09 22:06:53,077 - root - INFO - step: 38730 loss: 16.6517 memory: 44.58GiB(31.99%) tps: 82,301 tflops: 284.03 mfu: 28.72% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0877 +[titan] 2025-07-09 22:06:53,077 - root - INFO - lr: 2.1426e-04 gnorm: 0.78 [ 8:31:54<13:29:49] +[titan] 2025-07-09 22:06:57,107 - root - INFO - step: 38735 loss: 16.5953 memory: 44.58GiB(31.99%) tps: 81,305 tflops: 280.60 mfu: 28.37% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0252 +[titan] 2025-07-09 22:06:57,108 - root - INFO - lr: 2.1424e-04 gnorm: 0.79 [ 8:31:58<13:29:45] +[titan] 2025-07-09 22:07:01,030 - root - INFO - step: 38740 loss: 16.4473 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9020 +[titan] 2025-07-09 22:07:01,030 - root - INFO - lr: 2.1422e-04 gnorm: 0.76 [ 8:32:02<13:29:41] +[titan] 2025-07-09 22:07:04,946 - root - INFO - step: 38745 loss: 16.7804 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.1774 +[titan] 2025-07-09 22:07:04,947 - root - INFO - lr: 2.1420e-04 gnorm: 0.78 [ 8:32:06<13:29:37] +[titan] 2025-07-09 22:07:08,098 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:07:08,902 - root - INFO - step: 38750 loss: 16.5703 memory: 44.58GiB(31.99%) tps: 82,850 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5572 global_avg_mtp_loss: 14.0131 +[titan] 2025-07-09 22:07:08,902 - root - INFO - lr: 2.1418e-04 gnorm: 0.78 [ 8:32:09<13:29:33] +[titan] 2025-07-09 22:07:12,841 - root - INFO - step: 38755 loss: 16.5262 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 13.9718 +[titan] 2025-07-09 22:07:12,842 - root - INFO - lr: 2.1416e-04 gnorm: 0.75 [ 8:32:13<13:29:29] +[titan] 2025-07-09 22:07:16,804 - root - INFO - step: 38760 loss: 16.5122 memory: 44.58GiB(31.99%) tps: 82,698 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9696 +[titan] 2025-07-09 22:07:16,804 - root - INFO - lr: 2.1414e-04 gnorm: 0.80 [ 8:32:17<13:29:25] +[titan] 2025-07-09 22:07:20,715 - root - INFO - step: 38765 loss: 16.7252 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.1431 +[titan] 2025-07-09 22:07:20,716 - root - INFO - lr: 2.1412e-04 gnorm: 0.71 [ 8:32:21<13:29:21] +[titan] 2025-07-09 22:07:24,952 - root - INFO - step: 38770 loss: 16.7482 memory: 44.58GiB(31.99%) tps: 77,352 tflops: 266.95 mfu: 26.99% global_avg_ntp_loss: 2.5927 global_avg_mtp_loss: 14.1556 +[titan] 2025-07-09 22:07:24,952 - root - INFO - lr: 2.1410e-04 gnorm: 0.76 [ 8:32:26<13:29:17] +[titan] 2025-07-09 22:07:28,852 - root - INFO - step: 38775 loss: 16.7556 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1633 +[titan] 2025-07-09 22:07:28,852 - root - INFO - lr: 2.1408e-04 gnorm: 0.79 [ 8:32:29<13:29:13] +[titan] 2025-07-09 22:07:32,780 - root - INFO - step: 38780 loss: 16.6800 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.1048 +[titan] 2025-07-09 22:07:32,781 - root - INFO - lr: 2.1406e-04 gnorm: 0.77 [ 8:32:33<13:29:09] +[titan] 2025-07-09 22:07:36,709 - root - INFO - step: 38785 loss: 16.6620 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.0792 +[titan] 2025-07-09 22:07:36,710 - root - INFO - lr: 2.1404e-04 gnorm: 0.76 [ 8:32:37<13:29:05] +[titan] 2025-07-09 22:07:40,635 - root - INFO - step: 38790 loss: 16.6394 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 14.0702 +[titan] 2025-07-09 22:07:40,635 - root - INFO - lr: 2.1402e-04 gnorm: 0.75 [ 8:32:41<13:29:01] +[titan] 2025-07-09 22:07:44,557 - root - INFO - step: 38795 loss: 16.4386 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-09 22:07:44,557 - root - INFO - lr: 2.1400e-04 gnorm: 0.73 [ 8:32:45<13:28:57] +[titan] 2025-07-09 22:07:47,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:07:48,497 - root - INFO - step: 38800 loss: 16.4669 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9282 +[titan] 2025-07-09 22:07:48,497 - root - INFO - lr: 2.1398e-04 gnorm: 0.75 [ 8:32:49<13:28:53] +[titan] 2025-07-09 22:07:52,443 - root - INFO - step: 38805 loss: 16.6995 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5770 global_avg_mtp_loss: 14.1225 +[titan] 2025-07-09 22:07:52,443 - root - INFO - lr: 2.1396e-04 gnorm: 0.74 [ 8:32:53<13:28:49] +[titan] 2025-07-09 22:07:56,378 - root - INFO - step: 38810 loss: 16.6890 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5941 global_avg_mtp_loss: 14.0949 +[titan] 2025-07-09 22:07:56,379 - root - INFO - lr: 2.1394e-04 gnorm: 0.79 [ 8:32:57<13:28:45] +[titan] 2025-07-09 22:08:00,297 - root - INFO - step: 38815 loss: 16.5749 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0210 +[titan] 2025-07-09 22:08:00,297 - root - INFO - lr: 2.1392e-04 gnorm: 0.84 [ 8:33:01<13:28:41] +[titan] 2025-07-09 22:08:04,277 - root - INFO - step: 38820 loss: 16.6485 memory: 44.58GiB(31.99%) tps: 82,347 tflops: 284.19 mfu: 28.74% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.0764 +[titan] 2025-07-09 22:08:04,277 - root - INFO - lr: 2.1390e-04 gnorm: 0.81 [ 8:33:05<13:28:37] +[titan] 2025-07-09 22:08:08,183 - root - INFO - step: 38825 loss: 16.6084 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.0323 +[titan] 2025-07-09 22:08:08,183 - root - INFO - lr: 2.1388e-04 gnorm: 0.81 [ 8:33:09<13:28:33] +[titan] 2025-07-09 22:08:12,098 - root - INFO - step: 38830 loss: 16.5070 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9733 +[titan] 2025-07-09 22:08:12,098 - root - INFO - lr: 2.1386e-04 gnorm: 0.75 [ 8:33:13<13:28:29] +[titan] 2025-07-09 22:08:16,054 - root - INFO - step: 38835 loss: 16.4989 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9428 +[titan] 2025-07-09 22:08:16,054 - root - INFO - lr: 2.1384e-04 gnorm: 0.74 [ 8:33:17<13:28:25] +[titan] 2025-07-09 22:08:19,961 - root - INFO - step: 38840 loss: 16.4966 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 13.9563 +[titan] 2025-07-09 22:08:19,961 - root - INFO - lr: 2.1382e-04 gnorm: 0.77 [ 8:33:21<13:28:21] +[titan] 2025-07-09 22:08:23,915 - root - INFO - step: 38845 loss: 16.6549 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0848 +[titan] 2025-07-09 22:08:23,916 - root - INFO - lr: 2.1380e-04 gnorm: 0.75 [ 8:33:24<13:28:17] +[titan] 2025-07-09 22:08:27,044 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:08:27,846 - root - INFO - step: 38850 loss: 16.8739 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.6054 global_avg_mtp_loss: 14.2685 +[titan] 2025-07-09 22:08:27,846 - root - INFO - lr: 2.1378e-04 gnorm: 0.72 [ 8:33:28<13:28:13] +[titan] 2025-07-09 22:08:31,752 - root - INFO - step: 38855 loss: 16.3922 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.8589 +[titan] 2025-07-09 22:08:31,753 - root - INFO - lr: 2.1376e-04 gnorm: 0.82 [ 8:33:32<13:28:09] +[titan] 2025-07-09 22:08:35,666 - root - INFO - step: 38860 loss: 16.7132 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5887 global_avg_mtp_loss: 14.1246 +[titan] 2025-07-09 22:08:35,666 - root - INFO - lr: 2.1374e-04 gnorm: 0.78 [ 8:33:36<13:28:05] +[titan] 2025-07-09 22:08:39,567 - root - INFO - step: 38865 loss: 16.6432 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5706 global_avg_mtp_loss: 14.0726 +[titan] 2025-07-09 22:08:39,567 - root - INFO - lr: 2.1372e-04 gnorm: 0.79 [ 8:33:40<13:28:01] +[titan] 2025-07-09 22:08:43,473 - root - INFO - step: 38870 loss: 16.6488 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0757 +[titan] 2025-07-09 22:08:43,473 - root - INFO - lr: 2.1370e-04 gnorm: 0.78 [ 8:33:44<13:27:57] +[titan] 2025-07-09 22:08:47,371 - root - INFO - step: 38875 loss: 16.3680 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-09 22:08:47,371 - root - INFO - lr: 2.1368e-04 gnorm: 0.74 [ 8:33:48<13:27:53] +[titan] 2025-07-09 22:08:51,271 - root - INFO - step: 38880 loss: 16.7985 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.6060 global_avg_mtp_loss: 14.1925 +[titan] 2025-07-09 22:08:51,271 - root - INFO - lr: 2.1366e-04 gnorm: 0.77 [ 8:33:52<13:27:48] +[titan] 2025-07-09 22:08:55,175 - root - INFO - step: 38885 loss: 16.4939 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5548 global_avg_mtp_loss: 13.9391 +[titan] 2025-07-09 22:08:55,175 - root - INFO - lr: 2.1364e-04 gnorm: 0.78 [ 8:33:56<13:27:44] +[titan] 2025-07-09 22:08:59,087 - root - INFO - step: 38890 loss: 16.9241 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.6321 global_avg_mtp_loss: 14.2920 +[titan] 2025-07-09 22:08:59,087 - root - INFO - lr: 2.1362e-04 gnorm: 0.69 [ 8:34:00<13:27:40] +[titan] 2025-07-09 22:09:02,982 - root - INFO - step: 38895 loss: 16.5375 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 13.9789 +[titan] 2025-07-09 22:09:02,983 - root - INFO - lr: 2.1360e-04 gnorm: 0.78 [ 8:34:04<13:27:36] +[titan] 2025-07-09 22:09:06,106 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:09:06,899 - root - INFO - step: 38900 loss: 16.7075 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1276 +[titan] 2025-07-09 22:09:06,899 - root - INFO - lr: 2.1358e-04 gnorm: 0.82 [ 8:34:07<13:27:32] +[titan] 2025-07-09 22:09:10,859 - root - INFO - step: 38905 loss: 16.6382 memory: 44.58GiB(31.99%) tps: 82,766 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.5624 global_avg_mtp_loss: 14.0758 +[titan] 2025-07-09 22:09:10,859 - root - INFO - lr: 2.1356e-04 gnorm: 0.88 [ 8:34:11<13:27:28] +[titan] 2025-07-09 22:09:14,867 - root - INFO - step: 38910 loss: 16.7631 memory: 44.58GiB(31.99%) tps: 81,747 tflops: 282.12 mfu: 28.53% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1707 +[titan] 2025-07-09 22:09:14,868 - root - INFO - lr: 2.1354e-04 gnorm: 0.81 [ 8:34:15<13:27:24] +[titan] 2025-07-09 22:09:16,593 - root - INFO - Dumping profiler traces at step 38912 +[titan] 2025-07-09 22:09:16,626 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:09:18,993 - root - INFO - step: 38915 loss: 16.5149 memory: 44.58GiB(31.99%) tps: 79,429 tflops: 274.12 mfu: 27.72% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9674 +[titan] 2025-07-09 22:09:18,994 - root - INFO - lr: 2.1352e-04 gnorm: 0.83 [ 8:34:20<13:27:21] +[titan] 2025-07-09 22:09:22,934 - root - INFO - step: 38920 loss: 16.4088 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.8688 +[titan] 2025-07-09 22:09:22,934 - root - INFO - lr: 2.1350e-04 gnorm: 0.81 [ 8:34:24<13:27:17] +[titan] 2025-07-09 22:09:26,935 - root - INFO - step: 38925 loss: 16.7069 memory: 44.58GiB(31.99%) tps: 81,906 tflops: 282.67 mfu: 28.58% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 22:09:26,935 - root - INFO - lr: 2.1348e-04 gnorm: 0.76 [ 8:34:28<13:27:13] +[titan] 2025-07-09 22:09:30,846 - root - INFO - step: 38930 loss: 16.7103 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1239 +[titan] 2025-07-09 22:09:30,846 - root - INFO - lr: 2.1346e-04 gnorm: 0.80 [ 8:34:31<13:27:09] +[titan] 2025-07-09 22:09:34,810 - root - INFO - step: 38935 loss: 16.7670 memory: 44.58GiB(31.99%) tps: 82,682 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1767 +[titan] 2025-07-09 22:09:34,810 - root - INFO - lr: 2.1344e-04 gnorm: 0.83 [ 8:34:35<13:27:05] +[titan] 2025-07-09 22:09:38,745 - root - INFO - step: 38940 loss: 16.5778 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 14.0262 +[titan] 2025-07-09 22:09:38,745 - root - INFO - lr: 2.1342e-04 gnorm: 0.82 [ 8:34:39<13:27:01] +[titan] 2025-07-09 22:09:42,666 - root - INFO - step: 38945 loss: 16.7817 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1953 +[titan] 2025-07-09 22:09:42,666 - root - INFO - lr: 2.1340e-04 gnorm: 0.80 [ 8:34:43<13:26:57] +[titan] 2025-07-09 22:09:45,813 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:09:46,616 - root - INFO - step: 38950 loss: 16.6562 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-09 22:09:46,617 - root - INFO - lr: 2.1338e-04 gnorm: 0.80 [ 8:34:47<13:26:53] +[titan] 2025-07-09 22:09:50,593 - root - INFO - step: 38955 loss: 16.4847 memory: 44.58GiB(31.99%) tps: 82,399 tflops: 284.37 mfu: 28.75% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 13.9371 +[titan] 2025-07-09 22:09:50,594 - root - INFO - lr: 2.1336e-04 gnorm: 0.79 [ 8:34:51<13:26:49] +[titan] 2025-07-09 22:09:54,521 - root - INFO - step: 38960 loss: 16.5657 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 14.0180 +[titan] 2025-07-09 22:09:54,521 - root - INFO - lr: 2.1334e-04 gnorm: 0.76 [ 8:34:55<13:26:45] +[titan] 2025-07-09 22:09:58,441 - root - INFO - step: 38965 loss: 16.5892 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.0210 +[titan] 2025-07-09 22:09:58,441 - root - INFO - lr: 2.1332e-04 gnorm: 0.78 [ 8:34:59<13:26:41] +[titan] 2025-07-09 22:10:02,337 - root - INFO - step: 38970 loss: 16.3172 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.8052 +[titan] 2025-07-09 22:10:02,338 - root - INFO - lr: 2.1330e-04 gnorm: 0.76 [ 8:35:03<13:26:37] +[titan] 2025-07-09 22:10:06,257 - root - INFO - step: 38975 loss: 16.6236 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0655 +[titan] 2025-07-09 22:10:06,258 - root - INFO - lr: 2.1328e-04 gnorm: 0.75 [ 8:35:07<13:26:33] +[titan] 2025-07-09 22:10:10,171 - root - INFO - step: 38980 loss: 16.9444 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.6206 global_avg_mtp_loss: 14.3238 +[titan] 2025-07-09 22:10:10,171 - root - INFO - lr: 2.1326e-04 gnorm: 0.79 [ 8:35:11<13:26:29] +[titan] 2025-07-09 22:10:14,103 - root - INFO - step: 38985 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.8082 +[titan] 2025-07-09 22:10:14,104 - root - INFO - lr: 2.1324e-04 gnorm: 0.83 [ 8:35:15<13:26:25] +[titan] 2025-07-09 22:10:18,033 - root - INFO - step: 38990 loss: 16.8174 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.2150 +[titan] 2025-07-09 22:10:18,033 - root - INFO - lr: 2.1322e-04 gnorm: 0.75 [ 8:35:19<13:26:20] +[titan] 2025-07-09 22:10:21,977 - root - INFO - step: 38995 loss: 16.9885 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.6387 global_avg_mtp_loss: 14.3498 +[titan] 2025-07-09 22:10:21,977 - root - INFO - lr: 2.1320e-04 gnorm: 0.88 [ 8:35:23<13:26:16] +[titan] 2025-07-09 22:10:25,102 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:10:25,894 - root - INFO - step: 39000 loss: 16.6212 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0571 +[titan] 2025-07-09 22:10:25,894 - root - INFO - lr: 2.1318e-04 gnorm: 0.81 [ 8:35:26<13:26:12] +[titan] 2025-07-09 22:10:29,803 - root - INFO - step: 39005 loss: 16.6855 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-09 22:10:29,803 - root - INFO - lr: 2.1316e-04 gnorm: 0.75 [ 8:35:30<13:26:08] +[titan] 2025-07-09 22:10:33,735 - root - INFO - step: 39010 loss: 16.3841 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-09 22:10:33,735 - root - INFO - lr: 2.1314e-04 gnorm: 0.72 [ 8:35:34<13:26:04] +[titan] 2025-07-09 22:10:37,655 - root - INFO - step: 39015 loss: 16.5351 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9886 +[titan] 2025-07-09 22:10:37,656 - root - INFO - lr: 2.1312e-04 gnorm: 0.81 [ 8:35:38<13:26:00] +[titan] 2025-07-09 22:10:41,598 - root - INFO - step: 39020 loss: 16.6000 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5783 global_avg_mtp_loss: 14.0217 +[titan] 2025-07-09 22:10:41,599 - root - INFO - lr: 2.1310e-04 gnorm: 0.80 [ 8:35:42<13:25:56] +[titan] 2025-07-09 22:10:45,520 - root - INFO - step: 39025 loss: 16.6237 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5786 global_avg_mtp_loss: 14.0451 +[titan] 2025-07-09 22:10:45,520 - root - INFO - lr: 2.1308e-04 gnorm: 0.78 [ 8:35:46<13:25:52] +[titan] 2025-07-09 22:10:49,475 - root - INFO - step: 39030 loss: 16.5330 memory: 44.58GiB(31.99%) tps: 82,855 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9973 +[titan] 2025-07-09 22:10:49,475 - root - INFO - lr: 2.1306e-04 gnorm: 0.76 [ 8:35:50<13:25:48] +[titan] 2025-07-09 22:10:53,405 - root - INFO - step: 39035 loss: 16.6128 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-09 22:10:53,405 - root - INFO - lr: 2.1304e-04 gnorm: 0.78 [ 8:35:54<13:25:44] +[titan] 2025-07-09 22:10:57,331 - root - INFO - step: 39040 loss: 16.4413 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9048 +[titan] 2025-07-09 22:10:57,331 - root - INFO - lr: 2.1302e-04 gnorm: 0.71 [ 8:35:58<13:25:40] +[titan] 2025-07-09 22:11:01,284 - root - INFO - step: 39045 loss: 16.4893 memory: 44.58GiB(31.99%) tps: 82,898 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9479 +[titan] 2025-07-09 22:11:01,285 - root - INFO - lr: 2.1300e-04 gnorm: 0.78 [ 8:36:02<13:25:36] +[titan] 2025-07-09 22:11:04,388 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:11:05,179 - root - INFO - step: 39050 loss: 16.4409 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9071 +[titan] 2025-07-09 22:11:05,180 - root - INFO - lr: 2.1298e-04 gnorm: 0.75 [ 8:36:06<13:25:32] +[titan] 2025-07-09 22:11:09,093 - root - INFO - step: 39055 loss: 16.5263 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9847 +[titan] 2025-07-09 22:11:09,093 - root - INFO - lr: 2.1296e-04 gnorm: 0.74 [ 8:36:10<13:25:28] +[titan] 2025-07-09 22:11:13,154 - root - INFO - step: 39060 loss: 16.5856 memory: 44.58GiB(31.99%) tps: 80,700 tflops: 278.51 mfu: 28.16% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0198 +[titan] 2025-07-09 22:11:13,154 - root - INFO - lr: 2.1294e-04 gnorm: 0.76 [ 8:36:14<13:25:24] +[titan] 2025-07-09 22:11:17,079 - root - INFO - step: 39065 loss: 16.7324 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5876 global_avg_mtp_loss: 14.1448 +[titan] 2025-07-09 22:11:17,079 - root - INFO - lr: 2.1292e-04 gnorm: 0.75 [ 8:36:18<13:25:20] +[titan] 2025-07-09 22:11:20,999 - root - INFO - step: 39070 loss: 16.5921 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0284 +[titan] 2025-07-09 22:11:21,000 - root - INFO - lr: 2.1290e-04 gnorm: 0.76 [ 8:36:22<13:25:16] +[titan] 2025-07-09 22:11:24,915 - root - INFO - step: 39075 loss: 16.8987 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6256 global_avg_mtp_loss: 14.2731 +[titan] 2025-07-09 22:11:24,915 - root - INFO - lr: 2.1288e-04 gnorm: 0.76 [ 8:36:25<13:25:12] +[titan] 2025-07-09 22:11:28,853 - root - INFO - step: 39080 loss: 16.6588 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.0816 +[titan] 2025-07-09 22:11:28,854 - root - INFO - lr: 2.1286e-04 gnorm: 0.77 [ 8:36:29<13:25:08] +[titan] 2025-07-09 22:11:32,756 - root - INFO - step: 39085 loss: 16.6044 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5685 global_avg_mtp_loss: 14.0359 +[titan] 2025-07-09 22:11:32,756 - root - INFO - lr: 2.1284e-04 gnorm: 0.77 [ 8:36:33<13:25:04] +[titan] 2025-07-09 22:11:36,678 - root - INFO - step: 39090 loss: 16.7341 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1493 +[titan] 2025-07-09 22:11:36,678 - root - INFO - lr: 2.1282e-04 gnorm: 0.79 [ 8:36:37<13:25:00] +[titan] 2025-07-09 22:11:40,606 - root - INFO - step: 39095 loss: 16.9419 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.6410 global_avg_mtp_loss: 14.3009 +[titan] 2025-07-09 22:11:40,606 - root - INFO - lr: 2.1280e-04 gnorm: 0.81 [ 8:36:41<13:24:56] +[titan] 2025-07-09 22:11:43,709 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:11:44,503 - root - INFO - step: 39100 loss: 16.5264 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 13.9757 +[titan] 2025-07-09 22:11:44,504 - root - INFO - lr: 2.1278e-04 gnorm: 0.80 [ 8:36:45<13:24:52] +[titan] 2025-07-09 22:11:48,396 - root - INFO - step: 39105 loss: 16.6497 memory: 44.58GiB(31.99%) tps: 84,193 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-09 22:11:48,396 - root - INFO - lr: 2.1276e-04 gnorm: 0.83 [ 8:36:49<13:24:48] +[titan] 2025-07-09 22:11:52,350 - root - INFO - step: 39110 loss: 16.4886 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9405 +[titan] 2025-07-09 22:11:52,350 - root - INFO - lr: 2.1274e-04 gnorm: 0.85 [ 8:36:53<13:24:44] +[titan] 2025-07-09 22:11:56,303 - root - INFO - step: 39115 loss: 16.8635 memory: 44.58GiB(31.99%) tps: 82,897 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.6212 global_avg_mtp_loss: 14.2423 +[titan] 2025-07-09 22:11:56,303 - root - INFO - lr: 2.1272e-04 gnorm: 0.78 [ 8:36:57<13:24:40] +[titan] 2025-07-09 22:12:00,204 - root - INFO - step: 39120 loss: 16.4084 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8890 +[titan] 2025-07-09 22:12:00,204 - root - INFO - lr: 2.1270e-04 gnorm: 0.86 [ 8:37:01<13:24:36] +[titan] 2025-07-09 22:12:04,116 - root - INFO - step: 39125 loss: 16.5123 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9637 +[titan] 2025-07-09 22:12:04,116 - root - INFO - lr: 2.1268e-04 gnorm: 0.79 [ 8:37:05<13:24:32] +[titan] 2025-07-09 22:12:08,043 - root - INFO - step: 39130 loss: 16.5149 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-09 22:12:08,043 - root - INFO - lr: 2.1266e-04 gnorm: 0.77 [ 8:37:09<13:24:28] +[titan] 2025-07-09 22:12:11,968 - root - INFO - step: 39135 loss: 16.4016 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.8674 +[titan] 2025-07-09 22:12:11,968 - root - INFO - lr: 2.1264e-04 gnorm: 0.74 [ 8:37:13<13:24:24] +[titan] 2025-07-09 22:12:15,905 - root - INFO - step: 39140 loss: 16.6715 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0983 +[titan] 2025-07-09 22:12:15,906 - root - INFO - lr: 2.1262e-04 gnorm: 0.80 [ 8:37:16<13:24:20] +[titan] 2025-07-09 22:12:19,805 - root - INFO - step: 39145 loss: 16.4720 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 13.9283 +[titan] 2025-07-09 22:12:19,806 - root - INFO - lr: 2.1260e-04 gnorm: 0.82 [ 8:37:20<13:24:16] +[titan] 2025-07-09 22:12:22,945 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:12:23,731 - root - INFO - step: 39150 loss: 16.6660 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.0910 +[titan] 2025-07-09 22:12:23,732 - root - INFO - lr: 2.1258e-04 gnorm: 0.84 [ 8:37:24<13:24:12] +[titan] 2025-07-09 22:12:27,643 - root - INFO - step: 39155 loss: 16.6455 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.0721 +[titan] 2025-07-09 22:12:27,643 - root - INFO - lr: 2.1256e-04 gnorm: 0.80 [ 8:37:28<13:24:08] +[titan] 2025-07-09 22:12:31,579 - root - INFO - step: 39160 loss: 16.5876 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.0221 +[titan] 2025-07-09 22:12:31,579 - root - INFO - lr: 2.1254e-04 gnorm: 0.79 [ 8:37:32<13:24:04] +[titan] 2025-07-09 22:12:35,497 - root - INFO - step: 39165 loss: 16.5775 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0197 +[titan] 2025-07-09 22:12:35,497 - root - INFO - lr: 2.1252e-04 gnorm: 0.76 [ 8:37:36<13:24:00] +[titan] 2025-07-09 22:12:39,403 - root - INFO - step: 39170 loss: 16.9434 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.6514 global_avg_mtp_loss: 14.2920 +[titan] 2025-07-09 22:12:39,404 - root - INFO - lr: 2.1250e-04 gnorm: 0.76 [ 8:37:40<13:23:56] +[titan] 2025-07-09 22:12:43,338 - root - INFO - step: 39175 loss: 16.5250 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 13.9698 +[titan] 2025-07-09 22:12:43,338 - root - INFO - lr: 2.1248e-04 gnorm: 0.78 [ 8:37:44<13:23:52] +[titan] 2025-07-09 22:12:47,254 - root - INFO - step: 39180 loss: 16.8945 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.6317 global_avg_mtp_loss: 14.2629 +[titan] 2025-07-09 22:12:47,254 - root - INFO - lr: 2.1246e-04 gnorm: 0.87 [ 8:37:48<13:23:48] +[titan] 2025-07-09 22:12:51,184 - root - INFO - step: 39185 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9109 +[titan] 2025-07-09 22:12:51,185 - root - INFO - lr: 2.1244e-04 gnorm: 0.89 [ 8:37:52<13:23:44] +[titan] 2025-07-09 22:12:55,090 - root - INFO - step: 39190 loss: 16.8733 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2667 +[titan] 2025-07-09 22:12:55,091 - root - INFO - lr: 2.1242e-04 gnorm: 0.74 [ 8:37:56<13:23:39] +[titan] 2025-07-09 22:12:59,007 - root - INFO - step: 39195 loss: 16.6242 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0561 +[titan] 2025-07-09 22:12:59,007 - root - INFO - lr: 2.1240e-04 gnorm: 0.77 [ 8:38:00<13:23:35] +[titan] 2025-07-09 22:13:02,132 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:13:02,922 - root - INFO - step: 39200 loss: 16.5531 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 14.0048 +[titan] 2025-07-09 22:13:02,922 - root - INFO - lr: 2.1238e-04 gnorm: 0.82 [ 8:38:03<13:23:31] +[titan] 2025-07-09 22:13:06,840 - root - INFO - step: 39205 loss: 16.5719 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0059 +[titan] 2025-07-09 22:13:06,840 - root - INFO - lr: 2.1236e-04 gnorm: 0.75 [ 8:38:07<13:23:27] +[titan] 2025-07-09 22:13:10,763 - root - INFO - step: 39210 loss: 16.5301 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 13.9731 +[titan] 2025-07-09 22:13:10,764 - root - INFO - lr: 2.1234e-04 gnorm: 0.81 [ 8:38:11<13:23:23] +[titan] 2025-07-09 22:13:14,660 - root - INFO - step: 39215 loss: 16.6767 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0994 +[titan] 2025-07-09 22:13:14,661 - root - INFO - lr: 2.1232e-04 gnorm: 0.79 [ 8:38:15<13:23:19] +[titan] 2025-07-09 22:13:18,597 - root - INFO - step: 39220 loss: 16.8397 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.6075 global_avg_mtp_loss: 14.2322 +[titan] 2025-07-09 22:13:18,597 - root - INFO - lr: 2.1230e-04 gnorm: 0.76 [ 8:38:19<13:23:15] +[titan] 2025-07-09 22:13:22,531 - root - INFO - step: 39225 loss: 16.6316 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5825 global_avg_mtp_loss: 14.0491 +[titan] 2025-07-09 22:13:22,531 - root - INFO - lr: 2.1228e-04 gnorm: 0.81 [ 8:38:23<13:23:11] +[titan] 2025-07-09 22:13:26,461 - root - INFO - step: 39230 loss: 16.8354 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.6157 global_avg_mtp_loss: 14.2197 +[titan] 2025-07-09 22:13:26,461 - root - INFO - lr: 2.1226e-04 gnorm: 0.82 [ 8:38:27<13:23:07] +[titan] 2025-07-09 22:13:30,399 - root - INFO - step: 39235 loss: 16.5494 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 14.0041 +[titan] 2025-07-09 22:13:30,399 - root - INFO - lr: 2.1224e-04 gnorm: 0.78 [ 8:38:31<13:23:03] +[titan] 2025-07-09 22:13:34,280 - root - INFO - step: 39240 loss: 16.5755 memory: 44.58GiB(31.99%) tps: 84,434 tflops: 291.40 mfu: 29.46% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0150 +[titan] 2025-07-09 22:13:34,280 - root - INFO - lr: 2.1222e-04 gnorm: 0.78 [ 8:38:35<13:22:59] +[titan] 2025-07-09 22:13:38,190 - root - INFO - step: 39245 loss: 16.6309 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5618 global_avg_mtp_loss: 14.0691 +[titan] 2025-07-09 22:13:38,191 - root - INFO - lr: 2.1220e-04 gnorm: 0.78 [ 8:38:39<13:22:55] +[titan] 2025-07-09 22:13:41,315 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:13:42,103 - root - INFO - step: 39250 loss: 16.7367 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 22:13:42,103 - root - INFO - lr: 2.1218e-04 gnorm: 0.75 [ 8:38:43<13:22:51] +[titan] 2025-07-09 22:13:46,003 - root - INFO - step: 39255 loss: 16.5857 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0265 +[titan] 2025-07-09 22:13:46,003 - root - INFO - lr: 2.1216e-04 gnorm: 0.80 [ 8:38:47<13:22:47] +[titan] 2025-07-09 22:13:49,935 - root - INFO - step: 39260 loss: 17.0162 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.6591 global_avg_mtp_loss: 14.3572 +[titan] 2025-07-09 22:13:49,935 - root - INFO - lr: 2.1214e-04 gnorm: 0.80 [ 8:38:50<13:22:43] +[titan] 2025-07-09 22:13:53,848 - root - INFO - step: 39265 loss: 16.5870 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 14.0311 +[titan] 2025-07-09 22:13:53,848 - root - INFO - lr: 2.1211e-04 gnorm: 0.72 [ 8:38:54<13:22:39] +[titan] 2025-07-09 22:13:57,790 - root - INFO - step: 39270 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 83,140 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5624 global_avg_mtp_loss: 14.0535 +[titan] 2025-07-09 22:13:57,790 - root - INFO - lr: 2.1209e-04 gnorm: 0.82 [ 8:38:58<13:22:35] +[titan] 2025-07-09 22:14:01,714 - root - INFO - step: 39275 loss: 16.6628 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0954 +[titan] 2025-07-09 22:14:01,714 - root - INFO - lr: 2.1207e-04 gnorm: 0.73 [ 8:39:02<13:22:31] +[titan] 2025-07-09 22:14:05,640 - root - INFO - step: 39280 loss: 16.4894 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.9456 +[titan] 2025-07-09 22:14:05,640 - root - INFO - lr: 2.1205e-04 gnorm: 0.82 [ 8:39:06<13:22:27] +[titan] 2025-07-09 22:14:09,551 - root - INFO - step: 39285 loss: 16.6806 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0989 +[titan] 2025-07-09 22:14:09,551 - root - INFO - lr: 2.1203e-04 gnorm: 1.01 [ 8:39:10<13:22:23] +[titan] 2025-07-09 22:14:13,461 - root - INFO - step: 39290 loss: 16.6579 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.0685 +[titan] 2025-07-09 22:14:13,461 - root - INFO - lr: 2.1201e-04 gnorm: 0.78 [ 8:39:14<13:22:19] +[titan] 2025-07-09 22:14:17,398 - root - INFO - step: 39295 loss: 16.5911 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0147 +[titan] 2025-07-09 22:14:17,398 - root - INFO - lr: 2.1199e-04 gnorm: 0.81 [ 8:39:18<13:22:15] +[titan] 2025-07-09 22:14:20,533 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:14:21,332 - root - INFO - step: 39300 loss: 16.7965 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6100 global_avg_mtp_loss: 14.1865 +[titan] 2025-07-09 22:14:21,332 - root - INFO - lr: 2.1197e-04 gnorm: 0.79 [ 8:39:22<13:22:11] +[titan] 2025-07-09 22:14:25,269 - root - INFO - step: 39305 loss: 16.5299 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-09 22:14:25,269 - root - INFO - lr: 2.1195e-04 gnorm: 0.82 [ 8:39:26<13:22:07] +[titan] 2025-07-09 22:14:29,192 - root - INFO - step: 39310 loss: 16.8038 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.2049 +[titan] 2025-07-09 22:14:29,193 - root - INFO - lr: 2.1193e-04 gnorm: 0.79 [ 8:39:30<13:22:03] +[titan] 2025-07-09 22:14:33,098 - root - INFO - step: 39315 loss: 16.8016 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5985 global_avg_mtp_loss: 14.2031 +[titan] 2025-07-09 22:14:33,099 - root - INFO - lr: 2.1191e-04 gnorm: 0.76 [ 8:39:34<13:21:59] +[titan] 2025-07-09 22:14:37,027 - root - INFO - step: 39320 loss: 16.5118 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 13.9572 +[titan] 2025-07-09 22:14:37,027 - root - INFO - lr: 2.1189e-04 gnorm: 0.80 [ 8:39:38<13:21:55] +[titan] 2025-07-09 22:14:40,965 - root - INFO - step: 39325 loss: 16.7261 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1497 +[titan] 2025-07-09 22:14:40,966 - root - INFO - lr: 2.1187e-04 gnorm: 0.77 [ 8:39:42<13:21:51] +[titan] 2025-07-09 22:14:44,900 - root - INFO - step: 39330 loss: 16.7488 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5863 global_avg_mtp_loss: 14.1625 +[titan] 2025-07-09 22:14:44,901 - root - INFO - lr: 2.1185e-04 gnorm: 0.77 [ 8:39:45<13:21:47] +[titan] 2025-07-09 22:14:48,837 - root - INFO - step: 39335 loss: 16.7277 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5968 global_avg_mtp_loss: 14.1309 +[titan] 2025-07-09 22:14:48,837 - root - INFO - lr: 2.1183e-04 gnorm: 0.74 [ 8:39:49<13:21:43] +[titan] 2025-07-09 22:14:52,792 - root - INFO - step: 39340 loss: 16.9597 memory: 44.58GiB(31.99%) tps: 82,858 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.6274 global_avg_mtp_loss: 14.3322 +[titan] 2025-07-09 22:14:52,792 - root - INFO - lr: 2.1181e-04 gnorm: 0.81 [ 8:39:53<13:21:39] +[titan] 2025-07-09 22:14:56,721 - root - INFO - step: 39345 loss: 16.6380 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0707 +[titan] 2025-07-09 22:14:56,722 - root - INFO - lr: 2.1179e-04 gnorm: 0.78 [ 8:39:57<13:21:35] +[titan] 2025-07-09 22:14:59,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:15:00,696 - root - INFO - step: 39350 loss: 16.9283 memory: 44.58GiB(31.99%) tps: 82,446 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.3096 +[titan] 2025-07-09 22:15:00,697 - root - INFO - lr: 2.1177e-04 gnorm: 0.79 [ 8:40:01<13:21:31] +[titan] 2025-07-09 22:15:04,614 - root - INFO - step: 39355 loss: 16.6913 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5883 global_avg_mtp_loss: 14.1030 +[titan] 2025-07-09 22:15:04,614 - root - INFO - lr: 2.1175e-04 gnorm: 0.76 [ 8:40:05<13:21:27] +[titan] 2025-07-09 22:15:08,534 - root - INFO - step: 39360 loss: 16.5397 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5590 global_avg_mtp_loss: 13.9807 +[titan] 2025-07-09 22:15:08,535 - root - INFO - lr: 2.1173e-04 gnorm: 0.73 [ 8:40:09<13:21:23] +[titan] 2025-07-09 22:15:12,461 - root - INFO - step: 39365 loss: 16.5130 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9762 +[titan] 2025-07-09 22:15:12,461 - root - INFO - lr: 2.1171e-04 gnorm: 0.80 [ 8:40:13<13:21:19] +[titan] 2025-07-09 22:15:16,405 - root - INFO - step: 39370 loss: 16.8200 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.6074 global_avg_mtp_loss: 14.2126 +[titan] 2025-07-09 22:15:16,406 - root - INFO - lr: 2.1169e-04 gnorm: 0.76 [ 8:40:17<13:21:15] +[titan] 2025-07-09 22:15:20,328 - root - INFO - step: 39375 loss: 16.7228 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1338 +[titan] 2025-07-09 22:15:20,328 - root - INFO - lr: 2.1167e-04 gnorm: 0.74 [ 8:40:21<13:21:10] +[titan] 2025-07-09 22:15:24,231 - root - INFO - step: 39380 loss: 16.1629 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.6818 +[titan] 2025-07-09 22:15:24,232 - root - INFO - lr: 2.1165e-04 gnorm: 0.81 [ 8:40:25<13:21:06] +[titan] 2025-07-09 22:15:28,155 - root - INFO - step: 39385 loss: 16.4568 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-09 22:15:28,155 - root - INFO - lr: 2.1163e-04 gnorm: 0.77 [ 8:40:29<13:21:02] +[titan] 2025-07-09 22:15:32,062 - root - INFO - step: 39390 loss: 16.4952 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9497 +[titan] 2025-07-09 22:15:32,063 - root - INFO - lr: 2.1161e-04 gnorm: 0.85 [ 8:40:33<13:20:58] +[titan] 2025-07-09 22:15:36,018 - root - INFO - step: 39395 loss: 16.5049 memory: 44.58GiB(31.99%) tps: 82,842 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9643 +[titan] 2025-07-09 22:15:36,018 - root - INFO - lr: 2.1159e-04 gnorm: 0.83 [ 8:40:37<13:20:54] +[titan] 2025-07-09 22:15:39,129 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:15:39,930 - root - INFO - step: 39400 loss: 16.4299 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.8982 +[titan] 2025-07-09 22:15:39,930 - root - INFO - lr: 2.1157e-04 gnorm: 0.74 [ 8:40:40<13:20:50] +[titan] 2025-07-09 22:15:43,831 - root - INFO - step: 39405 loss: 16.6097 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 14.0518 +[titan] 2025-07-09 22:15:43,831 - root - INFO - lr: 2.1155e-04 gnorm: 0.80 [ 8:40:44<13:20:46] +[titan] 2025-07-09 22:15:47,768 - root - INFO - step: 39410 loss: 16.4079 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.8669 +[titan] 2025-07-09 22:15:47,768 - root - INFO - lr: 2.1153e-04 gnorm: 0.74 [ 8:40:48<13:20:42] +[titan] 2025-07-09 22:15:51,701 - root - INFO - step: 39415 loss: 16.5906 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 22:15:51,701 - root - INFO - lr: 2.1151e-04 gnorm: 0.76 [ 8:40:52<13:20:38] +[titan] 2025-07-09 22:15:55,673 - root - INFO - step: 39420 loss: 16.6558 memory: 44.58GiB(31.99%) tps: 82,490 tflops: 284.69 mfu: 28.79% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0877 +[titan] 2025-07-09 22:15:55,674 - root - INFO - lr: 2.1149e-04 gnorm: 0.77 [ 8:40:56<13:20:34] +[titan] 2025-07-09 22:15:59,104 - root - INFO - Dumping profiler traces at step 39424 +[titan] 2025-07-09 22:15:59,136 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:15:59,932 - root - INFO - step: 39425 loss: 16.7979 memory: 44.58GiB(31.99%) tps: 76,947 tflops: 265.56 mfu: 26.85% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.2058 +[titan] 2025-07-09 22:15:59,933 - root - INFO - lr: 2.1147e-04 gnorm: 0.83 [ 8:41:00<13:20:31] +[titan] 2025-07-09 22:16:03,843 - root - INFO - step: 39430 loss: 16.7213 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.1354 +[titan] 2025-07-09 22:16:03,843 - root - INFO - lr: 2.1145e-04 gnorm: 0.78 [ 8:41:04<13:20:27] +[titan] 2025-07-09 22:16:07,750 - root - INFO - step: 39435 loss: 16.7591 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.1698 +[titan] 2025-07-09 22:16:07,750 - root - INFO - lr: 2.1143e-04 gnorm: 0.79 [ 8:41:08<13:20:23] +[titan] 2025-07-09 22:16:11,665 - root - INFO - step: 39440 loss: 16.7013 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.1191 +[titan] 2025-07-09 22:16:11,665 - root - INFO - lr: 2.1141e-04 gnorm: 0.76 [ 8:41:12<13:20:19] +[titan] 2025-07-09 22:16:15,597 - root - INFO - step: 39445 loss: 16.6630 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.0925 +[titan] 2025-07-09 22:16:15,598 - root - INFO - lr: 2.1139e-04 gnorm: 0.80 [ 8:41:16<13:20:15] +[titan] 2025-07-09 22:16:18,765 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:16:19,553 - root - INFO - step: 39450 loss: 16.6268 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0697 +[titan] 2025-07-09 22:16:19,553 - root - INFO - lr: 2.1137e-04 gnorm: 0.75 [ 8:41:20<13:20:11] +[titan] 2025-07-09 22:16:23,472 - root - INFO - step: 39455 loss: 16.7102 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1255 +[titan] 2025-07-09 22:16:23,472 - root - INFO - lr: 2.1135e-04 gnorm: 0.74 [ 8:41:24<13:20:07] +[titan] 2025-07-09 22:16:27,384 - root - INFO - step: 39460 loss: 16.6600 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.0888 +[titan] 2025-07-09 22:16:27,384 - root - INFO - lr: 2.1133e-04 gnorm: 0.75 [ 8:41:28<13:20:03] +[titan] 2025-07-09 22:16:31,296 - root - INFO - step: 39465 loss: 16.5024 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9609 +[titan] 2025-07-09 22:16:31,296 - root - INFO - lr: 2.1131e-04 gnorm: 0.73 [ 8:41:32<13:19:59] +[titan] 2025-07-09 22:16:35,194 - root - INFO - step: 39470 loss: 16.6845 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.1183 +[titan] 2025-07-09 22:16:35,194 - root - INFO - lr: 2.1129e-04 gnorm: 0.76 [ 8:41:36<13:19:54] +[titan] 2025-07-09 22:16:39,100 - root - INFO - step: 39475 loss: 16.4858 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5525 global_avg_mtp_loss: 13.9333 +[titan] 2025-07-09 22:16:39,100 - root - INFO - lr: 2.1127e-04 gnorm: 0.77 [ 8:41:40<13:19:50] +[titan] 2025-07-09 22:16:43,008 - root - INFO - step: 39480 loss: 16.5665 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 14.0142 +[titan] 2025-07-09 22:16:43,009 - root - INFO - lr: 2.1125e-04 gnorm: 0.77 [ 8:41:44<13:19:46] +[titan] 2025-07-09 22:16:46,898 - root - INFO - step: 39485 loss: 16.6612 memory: 44.58GiB(31.99%) tps: 84,256 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0923 +[titan] 2025-07-09 22:16:46,898 - root - INFO - lr: 2.1123e-04 gnorm: 0.72 [ 8:41:47<13:19:42] +[titan] 2025-07-09 22:16:50,814 - root - INFO - step: 39490 loss: 16.5856 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 14.0333 +[titan] 2025-07-09 22:16:50,815 - root - INFO - lr: 2.1121e-04 gnorm: 0.78 [ 8:41:51<13:19:38] +[titan] 2025-07-09 22:16:54,741 - root - INFO - step: 39495 loss: 16.8527 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.2535 +[titan] 2025-07-09 22:16:54,741 - root - INFO - lr: 2.1119e-04 gnorm: 0.86 [ 8:41:55<13:19:34] +[titan] 2025-07-09 22:16:57,858 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:16:58,648 - root - INFO - step: 39500 loss: 16.3813 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.8551 +[titan] 2025-07-09 22:16:58,649 - root - INFO - lr: 2.1117e-04 gnorm: 0.82 [ 8:41:59<13:19:30] +[titan] 2025-07-09 22:17:02,569 - root - INFO - step: 39505 loss: 16.3529 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.8253 +[titan] 2025-07-09 22:17:02,569 - root - INFO - lr: 2.1115e-04 gnorm: 0.85 [ 8:42:03<13:19:26] +[titan] 2025-07-09 22:17:06,524 - root - INFO - step: 39510 loss: 16.5896 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0314 +[titan] 2025-07-09 22:17:06,524 - root - INFO - lr: 2.1113e-04 gnorm: 0.82 [ 8:42:07<13:19:22] +[titan] 2025-07-09 22:17:10,470 - root - INFO - step: 39515 loss: 16.7759 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1868 +[titan] 2025-07-09 22:17:10,471 - root - INFO - lr: 2.1111e-04 gnorm: 0.79 [ 8:42:11<13:19:18] +[titan] 2025-07-09 22:17:14,374 - root - INFO - step: 39520 loss: 16.7316 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5865 global_avg_mtp_loss: 14.1451 +[titan] 2025-07-09 22:17:14,374 - root - INFO - lr: 2.1109e-04 gnorm: 0.85 [ 8:42:15<13:19:14] +[titan] 2025-07-09 22:17:18,313 - root - INFO - step: 39525 loss: 16.6462 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.0775 +[titan] 2025-07-09 22:17:18,313 - root - INFO - lr: 2.1107e-04 gnorm: 0.77 [ 8:42:19<13:19:10] +[titan] 2025-07-09 22:17:22,232 - root - INFO - step: 39530 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.8032 +[titan] 2025-07-09 22:17:22,232 - root - INFO - lr: 2.1105e-04 gnorm: 0.85 [ 8:42:23<13:19:06] +[titan] 2025-07-09 22:17:26,150 - root - INFO - step: 39535 loss: 16.3962 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5470 global_avg_mtp_loss: 13.8492 +[titan] 2025-07-09 22:17:26,150 - root - INFO - lr: 2.1103e-04 gnorm: 0.79 [ 8:42:27<13:19:02] +[titan] 2025-07-09 22:17:30,078 - root - INFO - step: 39540 loss: 16.7411 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6003 global_avg_mtp_loss: 14.1408 +[titan] 2025-07-09 22:17:30,078 - root - INFO - lr: 2.1101e-04 gnorm: 0.76 [ 8:42:31<13:18:58] +[titan] 2025-07-09 22:17:33,996 - root - INFO - step: 39545 loss: 16.7804 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5972 global_avg_mtp_loss: 14.1832 +[titan] 2025-07-09 22:17:33,996 - root - INFO - lr: 2.1099e-04 gnorm: 0.75 [ 8:42:35<13:18:54] +[titan] 2025-07-09 22:17:37,105 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:17:37,895 - root - INFO - step: 39550 loss: 16.7633 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.1719 +[titan] 2025-07-09 22:17:37,895 - root - INFO - lr: 2.1097e-04 gnorm: 0.78 [ 8:42:38<13:18:50] +[titan] 2025-07-09 22:17:41,783 - root - INFO - step: 39555 loss: 16.7149 memory: 44.58GiB(31.99%) tps: 84,286 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.5813 global_avg_mtp_loss: 14.1336 +[titan] 2025-07-09 22:17:41,783 - root - INFO - lr: 2.1095e-04 gnorm: 0.84 [ 8:42:42<13:18:46] +[titan] 2025-07-09 22:17:45,707 - root - INFO - step: 39560 loss: 16.8092 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.6213 global_avg_mtp_loss: 14.1880 +[titan] 2025-07-09 22:17:45,707 - root - INFO - lr: 2.1093e-04 gnorm: 0.86 [ 8:42:46<13:18:42] +[titan] 2025-07-09 22:17:49,673 - root - INFO - step: 39565 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 82,620 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.8826 +[titan] 2025-07-09 22:17:49,674 - root - INFO - lr: 2.1091e-04 gnorm: 0.74 [ 8:42:50<13:18:38] +[titan] 2025-07-09 22:17:53,574 - root - INFO - step: 39570 loss: 16.5665 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0083 +[titan] 2025-07-09 22:17:53,574 - root - INFO - lr: 2.1089e-04 gnorm: 0.76 [ 8:42:54<13:18:34] +[titan] 2025-07-09 22:17:57,504 - root - INFO - step: 39575 loss: 16.8091 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.6115 global_avg_mtp_loss: 14.1976 +[titan] 2025-07-09 22:17:57,505 - root - INFO - lr: 2.1087e-04 gnorm: 0.84 [ 8:42:58<13:18:30] +[titan] 2025-07-09 22:18:01,447 - root - INFO - step: 39580 loss: 16.8064 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5909 global_avg_mtp_loss: 14.2155 +[titan] 2025-07-09 22:18:01,447 - root - INFO - lr: 2.1085e-04 gnorm: 0.81 [ 8:43:02<13:18:26] +[titan] 2025-07-09 22:18:05,350 - root - INFO - step: 39585 loss: 16.7214 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5887 global_avg_mtp_loss: 14.1328 +[titan] 2025-07-09 22:18:05,350 - root - INFO - lr: 2.1083e-04 gnorm: 0.80 [ 8:43:06<13:18:22] +[titan] 2025-07-09 22:18:09,261 - root - INFO - step: 39590 loss: 16.3854 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8754 +[titan] 2025-07-09 22:18:09,262 - root - INFO - lr: 2.1081e-04 gnorm: 0.80 [ 8:43:10<13:18:18] +[titan] 2025-07-09 22:18:13,178 - root - INFO - step: 39595 loss: 16.8809 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6090 global_avg_mtp_loss: 14.2719 +[titan] 2025-07-09 22:18:13,178 - root - INFO - lr: 2.1079e-04 gnorm: 0.75 [ 8:43:14<13:18:14] +[titan] 2025-07-09 22:18:16,301 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:18:17,090 - root - INFO - step: 39600 loss: 16.5885 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0280 +[titan] 2025-07-09 22:18:17,091 - root - INFO - lr: 2.1077e-04 gnorm: 0.76 [ 8:43:18<13:18:10] +[titan] 2025-07-09 22:18:21,005 - root - INFO - step: 39605 loss: 16.3691 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.8438 +[titan] 2025-07-09 22:18:21,006 - root - INFO - lr: 2.1075e-04 gnorm: 0.76 [ 8:43:22<13:18:05] +[titan] 2025-07-09 22:18:24,988 - root - INFO - step: 39610 loss: 16.7394 memory: 44.58GiB(31.99%) tps: 82,284 tflops: 283.98 mfu: 28.71% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1591 +[titan] 2025-07-09 22:18:24,988 - root - INFO - lr: 2.1073e-04 gnorm: 0.80 [ 8:43:25<13:18:02] +[titan] 2025-07-09 22:18:28,897 - root - INFO - step: 39615 loss: 16.7103 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5913 global_avg_mtp_loss: 14.1190 +[titan] 2025-07-09 22:18:28,898 - root - INFO - lr: 2.1071e-04 gnorm: 0.76 [ 8:43:29<13:17:58] +[titan] 2025-07-09 22:18:32,797 - root - INFO - step: 39620 loss: 16.8907 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6162 global_avg_mtp_loss: 14.2745 +[titan] 2025-07-09 22:18:32,797 - root - INFO - lr: 2.1069e-04 gnorm: 0.74 [ 8:43:33<13:17:53] +[titan] 2025-07-09 22:18:36,721 - root - INFO - step: 39625 loss: 16.8677 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.2489 +[titan] 2025-07-09 22:18:36,721 - root - INFO - lr: 2.1067e-04 gnorm: 0.77 [ 8:43:37<13:17:49] +[titan] 2025-07-09 22:18:40,636 - root - INFO - step: 39630 loss: 16.6615 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-09 22:18:40,636 - root - INFO - lr: 2.1065e-04 gnorm: 0.77 [ 8:43:41<13:17:45] +[titan] 2025-07-09 22:18:44,563 - root - INFO - step: 39635 loss: 16.6535 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.0754 +[titan] 2025-07-09 22:18:44,564 - root - INFO - lr: 2.1063e-04 gnorm: 0.74 [ 8:43:45<13:17:41] +[titan] 2025-07-09 22:18:48,490 - root - INFO - step: 39640 loss: 16.7464 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5901 global_avg_mtp_loss: 14.1563 +[titan] 2025-07-09 22:18:48,490 - root - INFO - lr: 2.1061e-04 gnorm: 0.74 [ 8:43:49<13:17:37] +[titan] 2025-07-09 22:18:52,416 - root - INFO - step: 39645 loss: 16.6874 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-09 22:18:52,416 - root - INFO - lr: 2.1059e-04 gnorm: 0.77 [ 8:43:53<13:17:33] +[titan] 2025-07-09 22:18:55,528 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:18:56,324 - root - INFO - step: 39650 loss: 16.4922 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.9560 +[titan] 2025-07-09 22:18:56,324 - root - INFO - lr: 2.1057e-04 gnorm: 0.79 [ 8:43:57<13:17:29] +[titan] 2025-07-09 22:19:00,270 - root - INFO - step: 39655 loss: 16.7807 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.5950 global_avg_mtp_loss: 14.1857 +[titan] 2025-07-09 22:19:00,270 - root - INFO - lr: 2.1055e-04 gnorm: 0.81 [ 8:44:01<13:17:25] +[titan] 2025-07-09 22:19:04,184 - root - INFO - step: 39660 loss: 16.7292 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1466 +[titan] 2025-07-09 22:19:04,184 - root - INFO - lr: 2.1053e-04 gnorm: 0.73 [ 8:44:05<13:17:21] +[titan] 2025-07-09 22:19:08,101 - root - INFO - step: 39665 loss: 16.5975 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0260 +[titan] 2025-07-09 22:19:08,101 - root - INFO - lr: 2.1051e-04 gnorm: 0.86 [ 8:44:09<13:17:17] +[titan] 2025-07-09 22:19:11,999 - root - INFO - step: 39670 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.8919 +[titan] 2025-07-09 22:19:11,999 - root - INFO - lr: 2.1049e-04 gnorm: 0.76 [ 8:44:13<13:17:13] +[titan] 2025-07-09 22:19:15,933 - root - INFO - step: 39675 loss: 16.5411 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 13.9841 +[titan] 2025-07-09 22:19:15,933 - root - INFO - lr: 2.1047e-04 gnorm: 0.79 [ 8:44:16<13:17:09] +[titan] 2025-07-09 22:19:19,873 - root - INFO - step: 39680 loss: 16.7630 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.6010 global_avg_mtp_loss: 14.1620 +[titan] 2025-07-09 22:19:19,874 - root - INFO - lr: 2.1044e-04 gnorm: 0.74 [ 8:44:20<13:17:05] +[titan] 2025-07-09 22:19:23,804 - root - INFO - step: 39685 loss: 16.8392 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.6030 global_avg_mtp_loss: 14.2362 +[titan] 2025-07-09 22:19:23,804 - root - INFO - lr: 2.1042e-04 gnorm: 0.76 [ 8:44:24<13:17:01] +[titan] 2025-07-09 22:19:27,768 - root - INFO - step: 39690 loss: 16.6551 memory: 44.58GiB(31.99%) tps: 82,667 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0886 +[titan] 2025-07-09 22:19:27,768 - root - INFO - lr: 2.1040e-04 gnorm: 0.80 [ 8:44:28<13:16:57] +[titan] 2025-07-09 22:19:31,724 - root - INFO - step: 39695 loss: 16.7913 memory: 44.58GiB(31.99%) tps: 82,831 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.1832 +[titan] 2025-07-09 22:19:31,724 - root - INFO - lr: 2.1038e-04 gnorm: 0.72 [ 8:44:32<13:16:53] +[titan] 2025-07-09 22:19:34,873 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:19:35,661 - root - INFO - step: 39700 loss: 16.6582 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.0899 +[titan] 2025-07-09 22:19:35,661 - root - INFO - lr: 2.1036e-04 gnorm: 0.79 [ 8:44:36<13:16:49] +[titan] 2025-07-09 22:19:39,563 - root - INFO - step: 39705 loss: 16.7846 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.1901 +[titan] 2025-07-09 22:19:39,563 - root - INFO - lr: 2.1034e-04 gnorm: 0.85 [ 8:44:40<13:16:45] +[titan] 2025-07-09 22:19:43,476 - root - INFO - step: 39710 loss: 16.7983 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.2058 +[titan] 2025-07-09 22:19:43,476 - root - INFO - lr: 2.1032e-04 gnorm: 0.78 [ 8:44:44<13:16:41] +[titan] 2025-07-09 22:19:47,393 - root - INFO - step: 39715 loss: 16.7209 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5900 global_avg_mtp_loss: 14.1309 +[titan] 2025-07-09 22:19:47,393 - root - INFO - lr: 2.1030e-04 gnorm: 0.80 [ 8:44:48<13:16:37] +[titan] 2025-07-09 22:19:51,332 - root - INFO - step: 39720 loss: 16.6342 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0612 +[titan] 2025-07-09 22:19:51,333 - root - INFO - lr: 2.1028e-04 gnorm: 0.82 [ 8:44:52<13:16:33] +[titan] 2025-07-09 22:19:55,285 - root - INFO - step: 39725 loss: 16.3424 memory: 44.58GiB(31.99%) tps: 82,918 tflops: 286.17 mfu: 28.93% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8368 +[titan] 2025-07-09 22:19:55,285 - root - INFO - lr: 2.1026e-04 gnorm: 0.80 [ 8:44:56<13:16:29] +[titan] 2025-07-09 22:19:59,212 - root - INFO - step: 39730 loss: 16.9793 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6413 global_avg_mtp_loss: 14.3380 +[titan] 2025-07-09 22:19:59,212 - root - INFO - lr: 2.1024e-04 gnorm: 0.80 [ 8:45:00<13:16:25] +[titan] 2025-07-09 22:20:03,132 - root - INFO - step: 39735 loss: 16.9152 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6183 global_avg_mtp_loss: 14.2969 +[titan] 2025-07-09 22:20:03,132 - root - INFO - lr: 2.1022e-04 gnorm: 0.79 [ 8:45:04<13:16:21] +[titan] 2025-07-09 22:20:07,066 - root - INFO - step: 39740 loss: 16.6596 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-09 22:20:07,066 - root - INFO - lr: 2.1020e-04 gnorm: 0.85 [ 8:45:08<13:16:17] +[titan] 2025-07-09 22:20:11,011 - root - INFO - step: 39745 loss: 16.3366 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.8253 +[titan] 2025-07-09 22:20:11,011 - root - INFO - lr: 2.1018e-04 gnorm: 0.79 [ 8:45:12<13:16:13] +[titan] 2025-07-09 22:20:14,157 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:20:14,947 - root - INFO - step: 39750 loss: 16.7019 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5869 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-09 22:20:14,947 - root - INFO - lr: 2.1016e-04 gnorm: 0.78 [ 8:45:15<13:16:09] +[titan] 2025-07-09 22:20:18,868 - root - INFO - step: 39755 loss: 16.5987 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0395 +[titan] 2025-07-09 22:20:18,868 - root - INFO - lr: 2.1014e-04 gnorm: 0.75 [ 8:45:19<13:16:05] +[titan] 2025-07-09 22:20:22,790 - root - INFO - step: 39760 loss: 16.5790 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 14.0128 +[titan] 2025-07-09 22:20:22,790 - root - INFO - lr: 2.1012e-04 gnorm: 0.80 [ 8:45:23<13:16:01] +[titan] 2025-07-09 22:20:26,689 - root - INFO - step: 39765 loss: 16.6654 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.0856 +[titan] 2025-07-09 22:20:26,689 - root - INFO - lr: 2.1010e-04 gnorm: 0.83 [ 8:45:27<13:15:57] +[titan] 2025-07-09 22:20:30,599 - root - INFO - step: 39770 loss: 16.6560 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.0785 +[titan] 2025-07-09 22:20:30,599 - root - INFO - lr: 2.1008e-04 gnorm: 0.84 [ 8:45:31<13:15:53] +[titan] 2025-07-09 22:20:34,535 - root - INFO - step: 39775 loss: 16.6387 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.0600 +[titan] 2025-07-09 22:20:34,535 - root - INFO - lr: 2.1006e-04 gnorm: 0.78 [ 8:45:35<13:15:49] +[titan] 2025-07-09 22:20:38,440 - root - INFO - step: 39780 loss: 16.8713 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6179 global_avg_mtp_loss: 14.2534 +[titan] 2025-07-09 22:20:38,440 - root - INFO - lr: 2.1004e-04 gnorm: 0.77 [ 8:45:39<13:15:45] +[titan] 2025-07-09 22:20:42,343 - root - INFO - step: 39785 loss: 16.4430 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9102 +[titan] 2025-07-09 22:20:42,343 - root - INFO - lr: 2.1002e-04 gnorm: 0.73 [ 8:45:43<13:15:41] +[titan] 2025-07-09 22:20:46,236 - root - INFO - step: 39790 loss: 16.5684 memory: 44.58GiB(31.99%) tps: 84,187 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0050 +[titan] 2025-07-09 22:20:46,236 - root - INFO - lr: 2.1000e-04 gnorm: 0.80 [ 8:45:47<13:15:37] +[titan] 2025-07-09 22:20:50,166 - root - INFO - step: 39795 loss: 16.6663 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0956 +[titan] 2025-07-09 22:20:50,166 - root - INFO - lr: 2.0998e-04 gnorm: 0.77 [ 8:45:51<13:15:33] +[titan] 2025-07-09 22:20:53,316 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:20:54,104 - root - INFO - step: 39800 loss: 16.8815 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.6189 global_avg_mtp_loss: 14.2625 +[titan] 2025-07-09 22:20:54,104 - root - INFO - lr: 2.0996e-04 gnorm: 0.77 [ 8:45:55<13:15:29] +[titan] 2025-07-09 22:20:58,026 - root - INFO - step: 39805 loss: 16.5611 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 14.0158 +[titan] 2025-07-09 22:20:58,026 - root - INFO - lr: 2.0994e-04 gnorm: 0.85 [ 8:45:59<13:15:25] +[titan] 2025-07-09 22:21:01,955 - root - INFO - step: 39810 loss: 16.6373 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0652 +[titan] 2025-07-09 22:21:01,955 - root - INFO - lr: 2.0992e-04 gnorm: 0.83 [ 8:46:02<13:15:21] +[titan] 2025-07-09 22:21:05,904 - root - INFO - step: 39815 loss: 16.6520 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0795 +[titan] 2025-07-09 22:21:05,904 - root - INFO - lr: 2.0990e-04 gnorm: 0.79 [ 8:46:06<13:15:17] +[titan] 2025-07-09 22:21:09,834 - root - INFO - step: 39820 loss: 16.4792 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9369 +[titan] 2025-07-09 22:21:09,834 - root - INFO - lr: 2.0988e-04 gnorm: 0.80 [ 8:46:10<13:15:13] +[titan] 2025-07-09 22:21:13,758 - root - INFO - step: 39825 loss: 16.8775 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.6233 global_avg_mtp_loss: 14.2542 +[titan] 2025-07-09 22:21:13,759 - root - INFO - lr: 2.0986e-04 gnorm: 0.77 [ 8:46:14<13:15:08] +[titan] 2025-07-09 22:21:17,702 - root - INFO - step: 39830 loss: 16.4079 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-09 22:21:17,702 - root - INFO - lr: 2.0984e-04 gnorm: 0.79 [ 8:46:18<13:15:04] +[titan] 2025-07-09 22:21:21,607 - root - INFO - step: 39835 loss: 16.8586 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.2505 +[titan] 2025-07-09 22:21:21,608 - root - INFO - lr: 2.0982e-04 gnorm: 0.78 [ 8:46:22<13:15:00] +[titan] 2025-07-09 22:21:25,516 - root - INFO - step: 39840 loss: 16.6675 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.0855 +[titan] 2025-07-09 22:21:25,517 - root - INFO - lr: 2.0980e-04 gnorm: 0.80 [ 8:46:26<13:14:56] +[titan] 2025-07-09 22:21:29,448 - root - INFO - step: 39845 loss: 16.5679 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5525 global_avg_mtp_loss: 14.0155 +[titan] 2025-07-09 22:21:29,448 - root - INFO - lr: 2.0978e-04 gnorm: 0.76 [ 8:46:30<13:14:52] +[titan] 2025-07-09 22:21:32,599 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:21:33,387 - root - INFO - step: 39850 loss: 16.8178 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.6015 global_avg_mtp_loss: 14.2163 +[titan] 2025-07-09 22:21:33,387 - root - INFO - lr: 2.0976e-04 gnorm: 0.73 [ 8:46:34<13:14:48] +[titan] 2025-07-09 22:21:37,327 - root - INFO - step: 39855 loss: 16.4628 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9217 +[titan] 2025-07-09 22:21:37,327 - root - INFO - lr: 2.0974e-04 gnorm: 0.75 [ 8:46:38<13:14:44] +[titan] 2025-07-09 22:21:41,243 - root - INFO - step: 39860 loss: 16.7479 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5928 global_avg_mtp_loss: 14.1551 +[titan] 2025-07-09 22:21:41,243 - root - INFO - lr: 2.0972e-04 gnorm: 0.78 [ 8:46:42<13:14:40] +[titan] 2025-07-09 22:21:45,153 - root - INFO - step: 39865 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5355 global_avg_mtp_loss: 13.8930 +[titan] 2025-07-09 22:21:45,153 - root - INFO - lr: 2.0970e-04 gnorm: 0.78 [ 8:46:46<13:14:36] +[titan] 2025-07-09 22:21:49,084 - root - INFO - step: 39870 loss: 16.6961 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.1245 +[titan] 2025-07-09 22:21:49,084 - root - INFO - lr: 2.0968e-04 gnorm: 0.77 [ 8:46:50<13:14:32] +[titan] 2025-07-09 22:21:53,005 - root - INFO - step: 39875 loss: 16.5717 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 14.0070 +[titan] 2025-07-09 22:21:53,005 - root - INFO - lr: 2.0966e-04 gnorm: 0.78 [ 8:46:53<13:14:28] +[titan] 2025-07-09 22:21:56,900 - root - INFO - step: 39880 loss: 16.7521 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.1585 +[titan] 2025-07-09 22:21:56,901 - root - INFO - lr: 2.0964e-04 gnorm: 0.75 [ 8:46:57<13:14:24] +[titan] 2025-07-09 22:22:00,815 - root - INFO - step: 39885 loss: 16.6150 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5636 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-09 22:22:00,815 - root - INFO - lr: 2.0962e-04 gnorm: 0.99 [ 8:47:01<13:14:20] +[titan] 2025-07-09 22:22:04,741 - root - INFO - step: 39890 loss: 16.6616 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.0817 +[titan] 2025-07-09 22:22:04,741 - root - INFO - lr: 2.0960e-04 gnorm: 0.75 [ 8:47:05<13:14:16] +[titan] 2025-07-09 22:22:08,635 - root - INFO - step: 39895 loss: 16.5043 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9608 +[titan] 2025-07-09 22:22:08,635 - root - INFO - lr: 2.0958e-04 gnorm: 0.78 [ 8:47:09<13:14:12] +[titan] 2025-07-09 22:22:11,751 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:22:12,555 - root - INFO - step: 39900 loss: 16.7836 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1914 +[titan] 2025-07-09 22:22:12,555 - root - INFO - lr: 2.0956e-04 gnorm: 0.79 [ 8:47:13<13:14:08] +[titan] 2025-07-09 22:22:16,474 - root - INFO - step: 39905 loss: 16.5309 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 13.9796 +[titan] 2025-07-09 22:22:16,474 - root - INFO - lr: 2.0954e-04 gnorm: 0.73 [ 8:47:17<13:14:04] +[titan] 2025-07-09 22:22:20,411 - root - INFO - step: 39910 loss: 16.7258 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1449 +[titan] 2025-07-09 22:22:20,412 - root - INFO - lr: 2.0952e-04 gnorm: 0.75 [ 8:47:21<13:14:00] +[titan] 2025-07-09 22:22:24,310 - root - INFO - step: 39915 loss: 16.7409 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5771 global_avg_mtp_loss: 14.1638 +[titan] 2025-07-09 22:22:24,310 - root - INFO - lr: 2.0950e-04 gnorm: 0.71 [ 8:47:25<13:13:56] +[titan] 2025-07-09 22:22:28,223 - root - INFO - step: 39920 loss: 16.6744 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.0964 +[titan] 2025-07-09 22:22:28,223 - root - INFO - lr: 2.0948e-04 gnorm: 0.75 [ 8:47:29<13:13:52] +[titan] 2025-07-09 22:22:32,121 - root - INFO - step: 39925 loss: 16.5309 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 13.9908 +[titan] 2025-07-09 22:22:32,121 - root - INFO - lr: 2.0946e-04 gnorm: 0.78 [ 8:47:33<13:13:48] +[titan] 2025-07-09 22:22:36,053 - root - INFO - step: 39930 loss: 16.4459 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9011 +[titan] 2025-07-09 22:22:36,053 - root - INFO - lr: 2.0943e-04 gnorm: 0.77 [ 8:47:37<13:13:44] +[titan] 2025-07-09 22:22:40,051 - root - INFO - step: 39935 loss: 16.6189 memory: 44.58GiB(31.99%) tps: 81,970 tflops: 282.89 mfu: 28.60% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.0502 +[titan] 2025-07-09 22:22:40,051 - root - INFO - lr: 2.0941e-04 gnorm: 0.77 [ 8:47:41<13:13:40] +[titan] 2025-07-09 22:22:40,981 - root - INFO - Dumping profiler traces at step 39936 +[titan] 2025-07-09 22:22:41,012 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:22:44,166 - root - INFO - step: 39940 loss: 16.5081 memory: 44.58GiB(31.99%) tps: 79,631 tflops: 274.82 mfu: 27.79% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9672 +[titan] 2025-07-09 22:22:44,166 - root - INFO - lr: 2.0939e-04 gnorm: 0.72 [ 8:47:45<13:13:36] +[titan] 2025-07-09 22:22:48,081 - root - INFO - step: 39945 loss: 16.3051 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.7895 +[titan] 2025-07-09 22:22:48,081 - root - INFO - lr: 2.0937e-04 gnorm: 0.78 [ 8:47:49<13:13:32] +[titan] 2025-07-09 22:22:51,215 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:22:52,044 - root - INFO - step: 39950 loss: 16.5500 memory: 44.58GiB(31.99%) tps: 82,687 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 13.9922 +[titan] 2025-07-09 22:22:52,044 - root - INFO - lr: 2.0935e-04 gnorm: 0.86 [ 8:47:53<13:13:28] +[titan] 2025-07-09 22:22:55,957 - root - INFO - step: 39955 loss: 16.3830 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8669 +[titan] 2025-07-09 22:22:55,958 - root - INFO - lr: 2.0933e-04 gnorm: 0.76 [ 8:47:56<13:13:24] +[titan] 2025-07-09 22:22:59,874 - root - INFO - step: 39960 loss: 16.5353 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5492 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-09 22:22:59,874 - root - INFO - lr: 2.0931e-04 gnorm: 0.75 [ 8:48:00<13:13:20] +[titan] 2025-07-09 22:23:03,785 - root - INFO - step: 39965 loss: 16.3241 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8117 +[titan] 2025-07-09 22:23:03,785 - root - INFO - lr: 2.0929e-04 gnorm: 0.80 [ 8:48:04<13:13:16] +[titan] 2025-07-09 22:23:07,692 - root - INFO - step: 39970 loss: 16.4276 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.8927 +[titan] 2025-07-09 22:23:07,692 - root - INFO - lr: 2.0927e-04 gnorm: 0.79 [ 8:48:08<13:13:12] +[titan] 2025-07-09 22:23:11,597 - root - INFO - step: 39975 loss: 16.7508 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1561 +[titan] 2025-07-09 22:23:11,597 - root - INFO - lr: 2.0925e-04 gnorm: 0.75 [ 8:48:12<13:13:08] +[titan] 2025-07-09 22:23:15,502 - root - INFO - step: 39980 loss: 16.6212 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0478 +[titan] 2025-07-09 22:23:15,503 - root - INFO - lr: 2.0923e-04 gnorm: 0.75 [ 8:48:16<13:13:04] +[titan] 2025-07-09 22:23:19,421 - root - INFO - step: 39985 loss: 17.1156 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.6631 global_avg_mtp_loss: 14.4525 +[titan] 2025-07-09 22:23:19,421 - root - INFO - lr: 2.0921e-04 gnorm: 0.76 [ 8:48:20<13:13:00] +[titan] 2025-07-09 22:23:23,392 - root - INFO - step: 39990 loss: 16.7710 memory: 44.58GiB(31.99%) tps: 82,532 tflops: 284.83 mfu: 28.80% global_avg_ntp_loss: 2.5961 global_avg_mtp_loss: 14.1749 +[titan] 2025-07-09 22:23:23,392 - root - INFO - lr: 2.0919e-04 gnorm: 0.80 [ 8:48:24<13:12:56] +[titan] 2025-07-09 22:23:27,296 - root - INFO - step: 39995 loss: 16.8228 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2145 +[titan] 2025-07-09 22:23:27,296 - root - INFO - lr: 2.0917e-04 gnorm: 0.76 [ 8:48:28<13:12:52] +[titan] 2025-07-09 22:23:30,424 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:23:31,221 - root - INFO - step: 40000 loss: 16.5747 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0161 +[titan] 2025-07-09 22:23:31,222 - root - INFO - lr: 2.0915e-04 gnorm: 0.73 [ 8:48:32<13:12:48] +[titan] 2025-07-09 22:23:31,222 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-09 22:23:33,417 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-09 22:23:33,418 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.20 seconds. +[titan] 2025-07-09 22:24:36,463 - root - INFO - step: 40005 loss: 16.8114 memory: 44.58GiB(31.99%) tps: 5,023 tflops: 17.33 mfu: 1.75% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2080 +[titan] 2025-07-09 22:24:36,463 - root - INFO - lr: 2.0913e-04 gnorm: 0.78 [ 8:49:37<13:14:16] +[titan] 2025-07-09 22:24:40,370 - root - INFO - step: 40010 loss: 16.7198 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.1429 +[titan] 2025-07-09 22:24:40,371 - root - INFO - lr: 2.0911e-04 gnorm: 0.80 [ 8:49:41<13:14:12] +[titan] 2025-07-09 22:24:44,308 - root - INFO - step: 40015 loss: 16.4961 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 13.9473 +[titan] 2025-07-09 22:24:44,308 - root - INFO - lr: 2.0909e-04 gnorm: 0.81 [ 8:49:45<13:14:08] +[titan] 2025-07-09 22:24:48,230 - root - INFO - step: 40020 loss: 16.4481 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9193 +[titan] 2025-07-09 22:24:48,230 - root - INFO - lr: 2.0907e-04 gnorm: 0.79 [ 8:49:49<13:14:04] +[titan] 2025-07-09 22:24:52,286 - root - INFO - step: 40025 loss: 16.6563 memory: 44.58GiB(31.99%) tps: 80,805 tflops: 278.87 mfu: 28.20% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0895 +[titan] 2025-07-09 22:24:52,286 - root - INFO - lr: 2.0905e-04 gnorm: 0.84 [ 8:49:53<13:14:00] +[titan] 2025-07-09 22:24:56,182 - root - INFO - step: 40030 loss: 16.9257 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.6293 global_avg_mtp_loss: 14.2963 +[titan] 2025-07-09 22:24:56,182 - root - INFO - lr: 2.0903e-04 gnorm: 0.81 [ 8:49:57<13:13:56] +[titan] 2025-07-09 22:25:00,116 - root - INFO - step: 40035 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5360 global_avg_mtp_loss: 13.9024 +[titan] 2025-07-09 22:25:00,117 - root - INFO - lr: 2.0901e-04 gnorm: 0.84 [ 8:50:01<13:13:52] +[titan] 2025-07-09 22:25:04,038 - root - INFO - step: 40040 loss: 16.6454 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.0768 +[titan] 2025-07-09 22:25:04,039 - root - INFO - lr: 2.0899e-04 gnorm: 0.78 [ 8:50:05<13:13:48] +[titan] 2025-07-09 22:25:08,008 - root - INFO - step: 40045 loss: 16.4322 memory: 44.58GiB(31.99%) tps: 82,558 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.9045 +[titan] 2025-07-09 22:25:08,008 - root - INFO - lr: 2.0897e-04 gnorm: 0.82 [ 8:50:08<13:13:44] +[titan] 2025-07-09 22:25:11,258 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:25:12,080 - root - INFO - step: 40050 loss: 16.5738 memory: 44.58GiB(31.99%) tps: 80,470 tflops: 277.72 mfu: 28.08% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0184 +[titan] 2025-07-09 22:25:12,080 - root - INFO - lr: 2.0895e-04 gnorm: 0.79 [ 8:50:13<13:13:40] +[titan] 2025-07-09 22:25:16,054 - root - INFO - step: 40055 loss: 16.6752 memory: 44.58GiB(31.99%) tps: 82,460 tflops: 284.58 mfu: 28.77% global_avg_ntp_loss: 2.5719 global_avg_mtp_loss: 14.1033 +[titan] 2025-07-09 22:25:16,055 - root - INFO - lr: 2.0893e-04 gnorm: 0.76 [ 8:50:17<13:13:36] +[titan] 2025-07-09 22:25:19,985 - root - INFO - step: 40060 loss: 16.7285 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.1224 +[titan] 2025-07-09 22:25:19,985 - root - INFO - lr: 2.0891e-04 gnorm: 0.77 [ 8:50:20<13:13:32] +[titan] 2025-07-09 22:25:23,940 - root - INFO - step: 40065 loss: 16.7354 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1481 +[titan] 2025-07-09 22:25:23,940 - root - INFO - lr: 2.0889e-04 gnorm: 0.77 [ 8:50:24<13:13:28] +[titan] 2025-07-09 22:25:27,907 - root - INFO - step: 40070 loss: 16.5595 memory: 44.58GiB(31.99%) tps: 82,623 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0073 +[titan] 2025-07-09 22:25:27,907 - root - INFO - lr: 2.0887e-04 gnorm: 0.73 [ 8:50:28<13:13:24] +[titan] 2025-07-09 22:25:31,857 - root - INFO - step: 40075 loss: 16.4810 memory: 44.58GiB(31.99%) tps: 82,953 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 13.9198 +[titan] 2025-07-09 22:25:31,857 - root - INFO - lr: 2.0885e-04 gnorm: 0.82 [ 8:50:32<13:13:20] +[titan] 2025-07-09 22:25:35,824 - root - INFO - step: 40080 loss: 16.7642 memory: 44.58GiB(31.99%) tps: 82,611 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.5975 global_avg_mtp_loss: 14.1666 +[titan] 2025-07-09 22:25:35,824 - root - INFO - lr: 2.0883e-04 gnorm: 0.77 [ 8:50:36<13:13:16] +[titan] 2025-07-09 22:25:39,755 - root - INFO - step: 40085 loss: 16.5319 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9845 +[titan] 2025-07-09 22:25:39,755 - root - INFO - lr: 2.0881e-04 gnorm: 0.78 [ 8:50:40<13:13:12] +[titan] 2025-07-09 22:25:43,691 - root - INFO - step: 40090 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8797 +[titan] 2025-07-09 22:25:43,691 - root - INFO - lr: 2.0879e-04 gnorm: 0.78 [ 8:50:44<13:13:08] +[titan] 2025-07-09 22:25:47,595 - root - INFO - step: 40095 loss: 16.4933 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 13.9497 +[titan] 2025-07-09 22:25:47,595 - root - INFO - lr: 2.0877e-04 gnorm: 0.78 [ 8:50:48<13:13:04] +[titan] 2025-07-09 22:25:50,742 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:25:51,530 - root - INFO - step: 40100 loss: 16.4977 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5555 global_avg_mtp_loss: 13.9422 +[titan] 2025-07-09 22:25:51,531 - root - INFO - lr: 2.0875e-04 gnorm: 0.86 [ 8:50:52<13:13:00] +[titan] 2025-07-09 22:25:55,463 - root - INFO - step: 40105 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.8706 +[titan] 2025-07-09 22:25:55,464 - root - INFO - lr: 2.0873e-04 gnorm: 0.89 [ 8:50:56<13:12:56] +[titan] 2025-07-09 22:25:59,429 - root - INFO - step: 40110 loss: 16.3654 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8501 +[titan] 2025-07-09 22:25:59,429 - root - INFO - lr: 2.0871e-04 gnorm: 0.84 [ 8:51:00<13:12:52] +[titan] 2025-07-09 22:26:03,344 - root - INFO - step: 40115 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5597 global_avg_mtp_loss: 14.0468 +[titan] 2025-07-09 22:26:03,344 - root - INFO - lr: 2.0869e-04 gnorm: 0.74 [ 8:51:04<13:12:48] +[titan] 2025-07-09 22:26:07,278 - root - INFO - step: 40120 loss: 16.4452 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9114 +[titan] 2025-07-09 22:26:07,278 - root - INFO - lr: 2.0867e-04 gnorm: 0.80 [ 8:51:08<13:12:44] +[titan] 2025-07-09 22:26:11,195 - root - INFO - step: 40125 loss: 16.5467 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 13.9886 +[titan] 2025-07-09 22:26:11,195 - root - INFO - lr: 2.0865e-04 gnorm: 0.79 [ 8:51:12<13:12:40] +[titan] 2025-07-09 22:26:15,139 - root - INFO - step: 40130 loss: 16.6940 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5792 global_avg_mtp_loss: 14.1148 +[titan] 2025-07-09 22:26:15,139 - root - INFO - lr: 2.0863e-04 gnorm: 0.80 [ 8:51:16<13:12:35] +[titan] 2025-07-09 22:26:19,038 - root - INFO - step: 40135 loss: 16.6243 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0632 +[titan] 2025-07-09 22:26:19,038 - root - INFO - lr: 2.0860e-04 gnorm: 0.76 [ 8:51:20<13:12:31] +[titan] 2025-07-09 22:26:22,977 - root - INFO - step: 40140 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 83,190 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-09 22:26:22,978 - root - INFO - lr: 2.0858e-04 gnorm: 0.78 [ 8:51:23<13:12:27] +[titan] 2025-07-09 22:26:26,900 - root - INFO - step: 40145 loss: 16.5217 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9811 +[titan] 2025-07-09 22:26:26,900 - root - INFO - lr: 2.0856e-04 gnorm: 0.77 [ 8:51:27<13:12:23] +[titan] 2025-07-09 22:26:30,006 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:26:30,794 - root - INFO - step: 40150 loss: 16.7285 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.1414 +[titan] 2025-07-09 22:26:30,794 - root - INFO - lr: 2.0854e-04 gnorm: 0.75 [ 8:51:31<13:12:19] +[titan] 2025-07-09 22:26:34,752 - root - INFO - step: 40155 loss: 16.5247 memory: 44.58GiB(31.99%) tps: 82,797 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5509 global_avg_mtp_loss: 13.9739 +[titan] 2025-07-09 22:26:34,752 - root - INFO - lr: 2.0852e-04 gnorm: 0.78 [ 8:51:35<13:12:15] +[titan] 2025-07-09 22:26:38,663 - root - INFO - step: 40160 loss: 16.4404 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-09 22:26:38,663 - root - INFO - lr: 2.0850e-04 gnorm: 0.72 [ 8:51:39<13:12:11] +[titan] 2025-07-09 22:26:42,577 - root - INFO - step: 40165 loss: 16.6390 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 14.0743 +[titan] 2025-07-09 22:26:42,578 - root - INFO - lr: 2.0848e-04 gnorm: 0.78 [ 8:51:43<13:12:07] +[titan] 2025-07-09 22:26:46,509 - root - INFO - step: 40170 loss: 16.6887 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1100 +[titan] 2025-07-09 22:26:46,509 - root - INFO - lr: 2.0846e-04 gnorm: 0.77 [ 8:51:47<13:12:03] +[titan] 2025-07-09 22:26:50,410 - root - INFO - step: 40175 loss: 16.6507 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5782 global_avg_mtp_loss: 14.0725 +[titan] 2025-07-09 22:26:50,410 - root - INFO - lr: 2.0844e-04 gnorm: 0.74 [ 8:51:51<13:11:59] +[titan] 2025-07-09 22:26:54,318 - root - INFO - step: 40180 loss: 16.5773 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5604 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-09 22:26:54,319 - root - INFO - lr: 2.0842e-04 gnorm: 0.77 [ 8:51:55<13:11:55] +[titan] 2025-07-09 22:26:58,243 - root - INFO - step: 40185 loss: 16.7457 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5958 global_avg_mtp_loss: 14.1499 +[titan] 2025-07-09 22:26:58,243 - root - INFO - lr: 2.0840e-04 gnorm: 0.76 [ 8:51:59<13:11:51] +[titan] 2025-07-09 22:27:02,163 - root - INFO - step: 40190 loss: 16.6818 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5661 global_avg_mtp_loss: 14.1157 +[titan] 2025-07-09 22:27:02,163 - root - INFO - lr: 2.0838e-04 gnorm: 0.76 [ 8:52:03<13:11:47] +[titan] 2025-07-09 22:27:06,086 - root - INFO - step: 40195 loss: 16.7378 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 22:27:06,086 - root - INFO - lr: 2.0836e-04 gnorm: 0.74 [ 8:52:07<13:11:43] +[titan] 2025-07-09 22:27:09,217 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:27:10,027 - root - INFO - step: 40200 loss: 16.5078 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9613 +[titan] 2025-07-09 22:27:10,027 - root - INFO - lr: 2.0834e-04 gnorm: 0.79 [ 8:52:10<13:11:39] +[titan] 2025-07-09 22:27:13,968 - root - INFO - step: 40205 loss: 16.7118 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.1260 +[titan] 2025-07-09 22:27:13,968 - root - INFO - lr: 2.0832e-04 gnorm: 0.79 [ 8:52:14<13:11:35] +[titan] 2025-07-09 22:27:17,880 - root - INFO - step: 40210 loss: 16.7133 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.1380 +[titan] 2025-07-09 22:27:17,880 - root - INFO - lr: 2.0830e-04 gnorm: 0.74 [ 8:52:18<13:11:31] +[titan] 2025-07-09 22:27:21,799 - root - INFO - step: 40215 loss: 16.6563 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0925 +[titan] 2025-07-09 22:27:21,799 - root - INFO - lr: 2.0828e-04 gnorm: 0.76 [ 8:52:22<13:11:27] +[titan] 2025-07-09 22:27:25,726 - root - INFO - step: 40220 loss: 16.8390 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.2483 +[titan] 2025-07-09 22:27:25,726 - root - INFO - lr: 2.0826e-04 gnorm: 0.80 [ 8:52:26<13:11:23] +[titan] 2025-07-09 22:27:29,642 - root - INFO - step: 40225 loss: 16.6493 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.0672 +[titan] 2025-07-09 22:27:29,642 - root - INFO - lr: 2.0824e-04 gnorm: 0.79 [ 8:52:30<13:11:19] +[titan] 2025-07-09 22:27:33,541 - root - INFO - step: 40230 loss: 16.5647 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-09 22:27:33,541 - root - INFO - lr: 2.0822e-04 gnorm: 0.74 [ 8:52:34<13:11:15] +[titan] 2025-07-09 22:27:37,483 - root - INFO - step: 40235 loss: 16.8537 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.2377 +[titan] 2025-07-09 22:27:37,484 - root - INFO - lr: 2.0820e-04 gnorm: 0.78 [ 8:52:38<13:11:11] +[titan] 2025-07-09 22:27:41,403 - root - INFO - step: 40240 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0522 +[titan] 2025-07-09 22:27:41,403 - root - INFO - lr: 2.0818e-04 gnorm: 0.79 [ 8:52:42<13:11:06] +[titan] 2025-07-09 22:27:45,319 - root - INFO - step: 40245 loss: 16.5160 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 13.9659 +[titan] 2025-07-09 22:27:45,319 - root - INFO - lr: 2.0816e-04 gnorm: 0.83 [ 8:52:46<13:11:02] +[titan] 2025-07-09 22:27:48,478 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:27:49,297 - root - INFO - step: 40250 loss: 16.7498 memory: 44.58GiB(31.99%) tps: 82,382 tflops: 284.32 mfu: 28.75% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.1582 +[titan] 2025-07-09 22:27:49,297 - root - INFO - lr: 2.0814e-04 gnorm: 0.77 [ 8:52:50<13:10:58] +[titan] 2025-07-09 22:27:53,198 - root - INFO - step: 40255 loss: 16.6125 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0416 +[titan] 2025-07-09 22:27:53,198 - root - INFO - lr: 2.0812e-04 gnorm: 0.76 [ 8:52:54<13:10:54] +[titan] 2025-07-09 22:27:57,144 - root - INFO - step: 40260 loss: 16.5159 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9598 +[titan] 2025-07-09 22:27:57,144 - root - INFO - lr: 2.0810e-04 gnorm: 0.81 [ 8:52:58<13:10:50] +[titan] 2025-07-09 22:28:01,046 - root - INFO - step: 40265 loss: 16.6566 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0873 +[titan] 2025-07-09 22:28:01,047 - root - INFO - lr: 2.0808e-04 gnorm: 0.80 [ 8:53:01<13:10:46] +[titan] 2025-07-09 22:28:04,966 - root - INFO - step: 40270 loss: 16.4133 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5542 global_avg_mtp_loss: 13.8591 +[titan] 2025-07-09 22:28:04,967 - root - INFO - lr: 2.0806e-04 gnorm: 0.84 [ 8:53:05<13:10:42] +[titan] 2025-07-09 22:28:08,889 - root - INFO - step: 40275 loss: 16.9100 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.6199 global_avg_mtp_loss: 14.2901 +[titan] 2025-07-09 22:28:08,889 - root - INFO - lr: 2.0804e-04 gnorm: 0.80 [ 8:53:09<13:10:38] +[titan] 2025-07-09 22:28:12,811 - root - INFO - step: 40280 loss: 16.7304 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5899 global_avg_mtp_loss: 14.1405 +[titan] 2025-07-09 22:28:12,811 - root - INFO - lr: 2.0802e-04 gnorm: 0.78 [ 8:53:13<13:10:34] +[titan] 2025-07-09 22:28:16,701 - root - INFO - step: 40285 loss: 16.8582 memory: 44.58GiB(31.99%) tps: 84,224 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.6093 global_avg_mtp_loss: 14.2489 +[titan] 2025-07-09 22:28:16,702 - root - INFO - lr: 2.0800e-04 gnorm: 0.82 [ 8:53:17<13:10:30] +[titan] 2025-07-09 22:28:20,595 - root - INFO - step: 40290 loss: 16.5523 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-09 22:28:20,596 - root - INFO - lr: 2.0798e-04 gnorm: 0.81 [ 8:53:21<13:10:26] +[titan] 2025-07-09 22:28:24,509 - root - INFO - step: 40295 loss: 16.6273 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0652 +[titan] 2025-07-09 22:28:24,509 - root - INFO - lr: 2.0796e-04 gnorm: 0.78 [ 8:53:25<13:10:22] +[titan] 2025-07-09 22:28:27,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:28:28,435 - root - INFO - step: 40300 loss: 16.5852 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0241 +[titan] 2025-07-09 22:28:28,435 - root - INFO - lr: 2.0794e-04 gnorm: 0.77 [ 8:53:29<13:10:18] +[titan] 2025-07-09 22:28:32,338 - root - INFO - step: 40305 loss: 16.9742 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.6288 global_avg_mtp_loss: 14.3454 +[titan] 2025-07-09 22:28:32,339 - root - INFO - lr: 2.0791e-04 gnorm: 0.74 [ 8:53:33<13:10:14] +[titan] 2025-07-09 22:28:36,248 - root - INFO - step: 40310 loss: 16.4734 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 13.9314 +[titan] 2025-07-09 22:28:36,248 - root - INFO - lr: 2.0789e-04 gnorm: 0.80 [ 8:53:37<13:10:10] +[titan] 2025-07-09 22:28:40,188 - root - INFO - step: 40315 loss: 16.5939 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0281 +[titan] 2025-07-09 22:28:40,189 - root - INFO - lr: 2.0787e-04 gnorm: 1.33 [ 8:53:41<13:10:06] +[titan] 2025-07-09 22:28:44,136 - root - INFO - step: 40320 loss: 16.5221 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9792 +[titan] 2025-07-09 22:28:44,136 - root - INFO - lr: 2.0785e-04 gnorm: 0.79 [ 8:53:45<13:10:02] +[titan] 2025-07-09 22:28:48,050 - root - INFO - step: 40325 loss: 16.7266 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1364 +[titan] 2025-07-09 22:28:48,050 - root - INFO - lr: 2.0783e-04 gnorm: 0.79 [ 8:53:48<13:09:58] +[titan] 2025-07-09 22:28:51,972 - root - INFO - step: 40330 loss: 16.5135 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5568 global_avg_mtp_loss: 13.9567 +[titan] 2025-07-09 22:28:51,972 - root - INFO - lr: 2.0781e-04 gnorm: 0.77 [ 8:53:52<13:09:54] +[titan] 2025-07-09 22:28:55,907 - root - INFO - step: 40335 loss: 16.4342 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.9095 +[titan] 2025-07-09 22:28:55,908 - root - INFO - lr: 2.0779e-04 gnorm: 0.74 [ 8:53:56<13:09:50] +[titan] 2025-07-09 22:28:59,829 - root - INFO - step: 40340 loss: 16.9286 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6388 global_avg_mtp_loss: 14.2899 +[titan] 2025-07-09 22:28:59,829 - root - INFO - lr: 2.0777e-04 gnorm: 0.80 [ 8:54:00<13:09:46] +[titan] 2025-07-09 22:29:03,771 - root - INFO - step: 40345 loss: 16.8122 memory: 44.58GiB(31.99%) tps: 83,127 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.6079 global_avg_mtp_loss: 14.2043 +[titan] 2025-07-09 22:29:03,771 - root - INFO - lr: 2.0775e-04 gnorm: 0.79 [ 8:54:04<13:09:42] +[titan] 2025-07-09 22:29:06,910 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:29:07,695 - root - INFO - step: 40350 loss: 16.6572 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.0851 +[titan] 2025-07-09 22:29:07,695 - root - INFO - lr: 2.0773e-04 gnorm: 0.77 [ 8:54:08<13:09:37] +[titan] 2025-07-09 22:29:11,594 - root - INFO - step: 40355 loss: 16.4155 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.8802 +[titan] 2025-07-09 22:29:11,594 - root - INFO - lr: 2.0771e-04 gnorm: 0.77 [ 8:54:12<13:09:33] +[titan] 2025-07-09 22:29:15,521 - root - INFO - step: 40360 loss: 16.7424 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1557 +[titan] 2025-07-09 22:29:15,521 - root - INFO - lr: 2.0769e-04 gnorm: 0.88 [ 8:54:16<13:09:29] +[titan] 2025-07-09 22:29:19,500 - root - INFO - step: 40365 loss: 16.8795 memory: 44.58GiB(31.99%) tps: 82,367 tflops: 284.26 mfu: 28.74% global_avg_ntp_loss: 2.6155 global_avg_mtp_loss: 14.2640 +[titan] 2025-07-09 22:29:19,500 - root - INFO - lr: 2.0767e-04 gnorm: 0.81 [ 8:54:20<13:09:25] +[titan] 2025-07-09 22:29:23,433 - root - INFO - step: 40370 loss: 16.6402 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 14.0612 +[titan] 2025-07-09 22:29:23,433 - root - INFO - lr: 2.0765e-04 gnorm: 0.84 [ 8:54:24<13:09:21] +[titan] 2025-07-09 22:29:27,337 - root - INFO - step: 40375 loss: 16.6968 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5817 global_avg_mtp_loss: 14.1151 +[titan] 2025-07-09 22:29:27,337 - root - INFO - lr: 2.0763e-04 gnorm: 0.82 [ 8:54:28<13:09:17] +[titan] 2025-07-09 22:29:31,277 - root - INFO - step: 40380 loss: 16.4175 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.8902 +[titan] 2025-07-09 22:29:31,277 - root - INFO - lr: 2.0761e-04 gnorm: 0.82 [ 8:54:32<13:09:13] +[titan] 2025-07-09 22:29:35,188 - root - INFO - step: 40385 loss: 16.7083 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.1320 +[titan] 2025-07-09 22:29:35,188 - root - INFO - lr: 2.0759e-04 gnorm: 0.85 [ 8:54:36<13:09:09] +[titan] 2025-07-09 22:29:39,100 - root - INFO - step: 40390 loss: 16.7292 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1417 +[titan] 2025-07-09 22:29:39,101 - root - INFO - lr: 2.0757e-04 gnorm: 0.75 [ 8:54:40<13:09:05] +[titan] 2025-07-09 22:29:43,034 - root - INFO - step: 40395 loss: 16.5096 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9656 +[titan] 2025-07-09 22:29:43,034 - root - INFO - lr: 2.0755e-04 gnorm: 0.74 [ 8:54:43<13:09:01] +[titan] 2025-07-09 22:29:46,194 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:29:46,985 - root - INFO - step: 40400 loss: 16.8901 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.6144 global_avg_mtp_loss: 14.2756 +[titan] 2025-07-09 22:29:46,985 - root - INFO - lr: 2.0753e-04 gnorm: 0.78 [ 8:54:47<13:08:57] +[titan] 2025-07-09 22:29:50,914 - root - INFO - step: 40405 loss: 16.6606 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0932 +[titan] 2025-07-09 22:29:50,915 - root - INFO - lr: 2.0751e-04 gnorm: 0.75 [ 8:54:51<13:08:53] +[titan] 2025-07-09 22:29:54,828 - root - INFO - step: 40410 loss: 16.6603 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0930 +[titan] 2025-07-09 22:29:54,828 - root - INFO - lr: 2.0749e-04 gnorm: 0.78 [ 8:54:55<13:08:49] +[titan] 2025-07-09 22:29:58,761 - root - INFO - step: 40415 loss: 16.8264 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.2214 +[titan] 2025-07-09 22:29:58,761 - root - INFO - lr: 2.0747e-04 gnorm: 0.73 [ 8:54:59<13:08:45] +[titan] 2025-07-09 22:30:02,702 - root - INFO - step: 40420 loss: 16.5292 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 13.9778 +[titan] 2025-07-09 22:30:02,702 - root - INFO - lr: 2.0745e-04 gnorm: 0.74 [ 8:55:03<13:08:41] +[titan] 2025-07-09 22:30:06,622 - root - INFO - step: 40425 loss: 16.7290 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.1537 +[titan] 2025-07-09 22:30:06,623 - root - INFO - lr: 2.0743e-04 gnorm: 0.73 [ 8:55:07<13:08:37] +[titan] 2025-07-09 22:30:10,521 - root - INFO - step: 40430 loss: 16.5199 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9746 +[titan] 2025-07-09 22:30:10,521 - root - INFO - lr: 2.0741e-04 gnorm: 0.83 [ 8:55:11<13:08:33] +[titan] 2025-07-09 22:30:14,428 - root - INFO - step: 40435 loss: 16.5387 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 13.9833 +[titan] 2025-07-09 22:30:14,428 - root - INFO - lr: 2.0739e-04 gnorm: 0.76 [ 8:55:15<13:08:29] +[titan] 2025-07-09 22:30:18,402 - root - INFO - step: 40440 loss: 16.8211 memory: 44.58GiB(31.99%) tps: 82,466 tflops: 284.60 mfu: 28.78% global_avg_ntp_loss: 2.6095 global_avg_mtp_loss: 14.2116 +[titan] 2025-07-09 22:30:18,402 - root - INFO - lr: 2.0737e-04 gnorm: 1.56 [ 8:55:19<13:08:25] +[titan] 2025-07-09 22:30:22,419 - root - INFO - step: 40445 loss: 16.6760 memory: 44.58GiB(31.99%) tps: 81,584 tflops: 281.56 mfu: 28.47% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.0887 +[titan] 2025-07-09 22:30:22,419 - root - INFO - lr: 2.0735e-04 gnorm: 0.75 [ 8:55:23<13:08:21] +[titan] 2025-07-09 22:30:24,916 - root - INFO - Dumping profiler traces at step 40448 +[titan] 2025-07-09 22:30:24,948 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:30:25,737 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:30:26,524 - root - INFO - step: 40450 loss: 16.5359 memory: 44.58GiB(31.99%) tps: 79,837 tflops: 275.53 mfu: 27.86% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 13.9794 +[titan] 2025-07-09 22:30:26,524 - root - INFO - lr: 2.0733e-04 gnorm: 0.79 [ 8:55:27<13:08:17] +[titan] 2025-07-09 22:30:30,444 - root - INFO - step: 40455 loss: 16.4508 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9138 +[titan] 2025-07-09 22:30:30,444 - root - INFO - lr: 2.0731e-04 gnorm: 0.82 [ 8:55:31<13:08:13] +[titan] 2025-07-09 22:30:34,343 - root - INFO - step: 40460 loss: 16.5419 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 13.9906 +[titan] 2025-07-09 22:30:34,343 - root - INFO - lr: 2.0728e-04 gnorm: 0.83 [ 8:55:35<13:08:09] +[titan] 2025-07-09 22:30:38,266 - root - INFO - step: 40465 loss: 16.6731 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5678 global_avg_mtp_loss: 14.1053 +[titan] 2025-07-09 22:30:38,267 - root - INFO - lr: 2.0726e-04 gnorm: 0.80 [ 8:55:39<13:08:05] +[titan] 2025-07-09 22:30:42,191 - root - INFO - step: 40470 loss: 16.6407 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0784 +[titan] 2025-07-09 22:30:42,192 - root - INFO - lr: 2.0724e-04 gnorm: 0.78 [ 8:55:43<13:08:01] +[titan] 2025-07-09 22:30:46,148 - root - INFO - step: 40475 loss: 16.5275 memory: 44.58GiB(31.99%) tps: 82,823 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-09 22:30:46,148 - root - INFO - lr: 2.0722e-04 gnorm: 0.81 [ 8:55:47<13:07:57] +[titan] 2025-07-09 22:30:50,041 - root - INFO - step: 40480 loss: 16.6443 memory: 44.58GiB(31.99%) tps: 84,186 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.5878 global_avg_mtp_loss: 14.0565 +[titan] 2025-07-09 22:30:50,041 - root - INFO - lr: 2.0720e-04 gnorm: 0.88 [ 8:55:50<13:07:53] +[titan] 2025-07-09 22:30:53,959 - root - INFO - step: 40485 loss: 16.6358 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0651 +[titan] 2025-07-09 22:30:53,959 - root - INFO - lr: 2.0718e-04 gnorm: 0.81 [ 8:55:54<13:07:49] +[titan] 2025-07-09 22:30:57,905 - root - INFO - step: 40490 loss: 16.6554 memory: 44.58GiB(31.99%) tps: 83,045 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.0836 +[titan] 2025-07-09 22:30:57,905 - root - INFO - lr: 2.0716e-04 gnorm: 0.78 [ 8:55:58<13:07:45] +[titan] 2025-07-09 22:31:01,859 - root - INFO - step: 40495 loss: 17.0392 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.6414 global_avg_mtp_loss: 14.3978 +[titan] 2025-07-09 22:31:01,859 - root - INFO - lr: 2.0714e-04 gnorm: 0.82 [ 8:56:02<13:07:41] +[titan] 2025-07-09 22:31:04,999 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:31:05,793 - root - INFO - step: 40500 loss: 16.4283 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.8936 +[titan] 2025-07-09 22:31:05,793 - root - INFO - lr: 2.0712e-04 gnorm: 0.77 [ 8:56:06<13:07:37] +[titan] 2025-07-09 22:31:09,713 - root - INFO - step: 40505 loss: 16.4734 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9299 +[titan] 2025-07-09 22:31:09,713 - root - INFO - lr: 2.0710e-04 gnorm: 0.81 [ 8:56:10<13:07:33] +[titan] 2025-07-09 22:31:13,623 - root - INFO - step: 40510 loss: 16.5058 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 13.9595 +[titan] 2025-07-09 22:31:13,623 - root - INFO - lr: 2.0708e-04 gnorm: 0.74 [ 8:56:14<13:07:29] +[titan] 2025-07-09 22:31:17,542 - root - INFO - step: 40515 loss: 16.6036 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0496 +[titan] 2025-07-09 22:31:17,542 - root - INFO - lr: 2.0706e-04 gnorm: 0.78 [ 8:56:18<13:07:25] +[titan] 2025-07-09 22:31:21,457 - root - INFO - step: 40520 loss: 16.5176 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 13.9630 +[titan] 2025-07-09 22:31:21,457 - root - INFO - lr: 2.0704e-04 gnorm: 0.75 [ 8:56:22<13:07:21] +[titan] 2025-07-09 22:31:25,356 - root - INFO - step: 40525 loss: 16.7472 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5975 global_avg_mtp_loss: 14.1497 +[titan] 2025-07-09 22:31:25,356 - root - INFO - lr: 2.0702e-04 gnorm: 0.81 [ 8:56:26<13:07:16] +[titan] 2025-07-09 22:31:29,262 - root - INFO - step: 40530 loss: 16.9422 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6276 global_avg_mtp_loss: 14.3146 +[titan] 2025-07-09 22:31:29,262 - root - INFO - lr: 2.0700e-04 gnorm: 0.73 [ 8:56:30<13:07:12] +[titan] 2025-07-09 22:31:33,162 - root - INFO - step: 40535 loss: 16.9789 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6343 global_avg_mtp_loss: 14.3446 +[titan] 2025-07-09 22:31:33,162 - root - INFO - lr: 2.0698e-04 gnorm: 0.84 [ 8:56:34<13:07:08] +[titan] 2025-07-09 22:31:37,049 - root - INFO - step: 40540 loss: 16.5290 memory: 44.58GiB(31.99%) tps: 84,309 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.5371 global_avg_mtp_loss: 13.9919 +[titan] 2025-07-09 22:31:37,049 - root - INFO - lr: 2.0696e-04 gnorm: 0.76 [ 8:56:37<13:07:04] +[titan] 2025-07-09 22:31:40,966 - root - INFO - step: 40545 loss: 16.8445 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.6087 global_avg_mtp_loss: 14.2358 +[titan] 2025-07-09 22:31:40,966 - root - INFO - lr: 2.0694e-04 gnorm: 0.85 [ 8:56:41<13:07:00] +[titan] 2025-07-09 22:31:44,096 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:31:44,882 - root - INFO - step: 40550 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.1196 +[titan] 2025-07-09 22:31:44,882 - root - INFO - lr: 2.0692e-04 gnorm: 0.83 [ 8:56:45<13:06:56] +[titan] 2025-07-09 22:31:48,790 - root - INFO - step: 40555 loss: 16.3982 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.8630 +[titan] 2025-07-09 22:31:48,791 - root - INFO - lr: 2.0690e-04 gnorm: 0.77 [ 8:56:49<13:06:52] +[titan] 2025-07-09 22:31:52,714 - root - INFO - step: 40560 loss: 16.5592 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0001 +[titan] 2025-07-09 22:31:52,714 - root - INFO - lr: 2.0688e-04 gnorm: 0.78 [ 8:56:53<13:06:48] +[titan] 2025-07-09 22:31:56,647 - root - INFO - step: 40565 loss: 16.5038 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 13.9615 +[titan] 2025-07-09 22:31:56,647 - root - INFO - lr: 2.0686e-04 gnorm: 0.75 [ 8:56:57<13:06:44] +[titan] 2025-07-09 22:32:00,568 - root - INFO - step: 40570 loss: 16.7234 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 22:32:00,568 - root - INFO - lr: 2.0684e-04 gnorm: 0.78 [ 8:57:01<13:06:40] +[titan] 2025-07-09 22:32:04,463 - root - INFO - step: 40575 loss: 16.5676 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0001 +[titan] 2025-07-09 22:32:04,463 - root - INFO - lr: 2.0682e-04 gnorm: 0.82 [ 8:57:05<13:06:36] +[titan] 2025-07-09 22:32:08,429 - root - INFO - step: 40580 loss: 16.6539 memory: 44.58GiB(31.99%) tps: 82,620 tflops: 285.14 mfu: 28.83% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.0735 +[titan] 2025-07-09 22:32:08,430 - root - INFO - lr: 2.0680e-04 gnorm: 0.77 [ 8:57:09<13:06:32] +[titan] 2025-07-09 22:32:12,329 - root - INFO - step: 40585 loss: 16.9769 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.3509 +[titan] 2025-07-09 22:32:12,329 - root - INFO - lr: 2.0678e-04 gnorm: 0.80 [ 8:57:13<13:06:28] +[titan] 2025-07-09 22:32:16,267 - root - INFO - step: 40590 loss: 16.6142 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0478 +[titan] 2025-07-09 22:32:16,267 - root - INFO - lr: 2.0676e-04 gnorm: 0.76 [ 8:57:17<13:06:24] +[titan] 2025-07-09 22:32:20,152 - root - INFO - step: 40595 loss: 16.7191 memory: 44.58GiB(31.99%) tps: 84,361 tflops: 291.14 mfu: 29.44% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.1316 +[titan] 2025-07-09 22:32:20,152 - root - INFO - lr: 2.0674e-04 gnorm: 0.77 [ 8:57:21<13:06:20] +[titan] 2025-07-09 22:32:23,276 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:32:24,074 - root - INFO - step: 40600 loss: 16.6352 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.0552 +[titan] 2025-07-09 22:32:24,075 - root - INFO - lr: 2.0671e-04 gnorm: 0.82 [ 8:57:24<13:06:16] +[titan] 2025-07-09 22:32:28,043 - root - INFO - step: 40605 loss: 16.6964 memory: 44.58GiB(31.99%) tps: 82,582 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1070 +[titan] 2025-07-09 22:32:28,043 - root - INFO - lr: 2.0669e-04 gnorm: 0.83 [ 8:57:28<13:06:12] +[titan] 2025-07-09 22:32:31,953 - root - INFO - step: 40610 loss: 16.5897 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0254 +[titan] 2025-07-09 22:32:31,953 - root - INFO - lr: 2.0667e-04 gnorm: 0.75 [ 8:57:32<13:06:08] +[titan] 2025-07-09 22:32:35,875 - root - INFO - step: 40615 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5727 global_avg_mtp_loss: 14.0779 +[titan] 2025-07-09 22:32:35,875 - root - INFO - lr: 2.0665e-04 gnorm: 0.78 [ 8:57:36<13:06:04] +[titan] 2025-07-09 22:32:39,798 - root - INFO - step: 40620 loss: 16.7279 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1459 +[titan] 2025-07-09 22:32:39,798 - root - INFO - lr: 2.0663e-04 gnorm: 0.78 [ 8:57:40<13:06:00] +[titan] 2025-07-09 22:32:43,710 - root - INFO - step: 40625 loss: 16.5312 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 13.9817 +[titan] 2025-07-09 22:32:43,710 - root - INFO - lr: 2.0661e-04 gnorm: 0.80 [ 8:57:44<13:05:55] +[titan] 2025-07-09 22:32:47,626 - root - INFO - step: 40630 loss: 16.5335 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9817 +[titan] 2025-07-09 22:32:47,627 - root - INFO - lr: 2.0659e-04 gnorm: 0.75 [ 8:57:48<13:05:51] +[titan] 2025-07-09 22:32:51,564 - root - INFO - step: 40635 loss: 16.7088 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1197 +[titan] 2025-07-09 22:32:51,564 - root - INFO - lr: 2.0657e-04 gnorm: 0.77 [ 8:57:52<13:05:47] +[titan] 2025-07-09 22:32:55,478 - root - INFO - step: 40640 loss: 16.4614 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.9319 +[titan] 2025-07-09 22:32:55,479 - root - INFO - lr: 2.0655e-04 gnorm: 0.78 [ 8:57:56<13:05:43] +[titan] 2025-07-09 22:32:59,390 - root - INFO - step: 40645 loss: 16.6012 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0515 +[titan] 2025-07-09 22:32:59,390 - root - INFO - lr: 2.0653e-04 gnorm: 0.72 [ 8:58:00<13:05:39] +[titan] 2025-07-09 22:33:02,510 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:33:03,308 - root - INFO - step: 40650 loss: 16.5654 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0147 +[titan] 2025-07-09 22:33:03,308 - root - INFO - lr: 2.0651e-04 gnorm: 0.82 [ 8:58:04<13:05:35] +[titan] 2025-07-09 22:33:07,225 - root - INFO - step: 40655 loss: 16.5681 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 14.0107 +[titan] 2025-07-09 22:33:07,225 - root - INFO - lr: 2.0649e-04 gnorm: 0.74 [ 8:58:08<13:05:31] +[titan] 2025-07-09 22:33:11,183 - root - INFO - step: 40660 loss: 16.5085 memory: 44.58GiB(31.99%) tps: 82,793 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 13.9591 +[titan] 2025-07-09 22:33:11,183 - root - INFO - lr: 2.0647e-04 gnorm: 0.75 [ 8:58:12<13:05:27] +[titan] 2025-07-09 22:33:15,129 - root - INFO - step: 40665 loss: 16.6390 memory: 44.58GiB(31.99%) tps: 83,060 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5829 global_avg_mtp_loss: 14.0561 +[titan] 2025-07-09 22:33:15,129 - root - INFO - lr: 2.0645e-04 gnorm: 0.77 [ 8:58:16<13:05:23] +[titan] 2025-07-09 22:33:19,040 - root - INFO - step: 40670 loss: 16.5406 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9888 +[titan] 2025-07-09 22:33:19,041 - root - INFO - lr: 2.0643e-04 gnorm: 0.78 [ 8:58:19<13:05:19] +[titan] 2025-07-09 22:33:22,972 - root - INFO - step: 40675 loss: 16.5341 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 13.9837 +[titan] 2025-07-09 22:33:22,972 - root - INFO - lr: 2.0641e-04 gnorm: 0.81 [ 8:58:23<13:05:15] +[titan] 2025-07-09 22:33:26,879 - root - INFO - step: 40680 loss: 16.6396 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0744 +[titan] 2025-07-09 22:33:26,879 - root - INFO - lr: 2.0639e-04 gnorm: 0.78 [ 8:58:27<13:05:11] +[titan] 2025-07-09 22:33:30,785 - root - INFO - step: 40685 loss: 16.7055 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.1295 +[titan] 2025-07-09 22:33:30,785 - root - INFO - lr: 2.0637e-04 gnorm: 0.77 [ 8:58:31<13:05:07] +[titan] 2025-07-09 22:33:34,686 - root - INFO - step: 40690 loss: 16.4753 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5458 global_avg_mtp_loss: 13.9296 +[titan] 2025-07-09 22:33:34,686 - root - INFO - lr: 2.0635e-04 gnorm: 0.77 [ 8:58:35<13:05:03] +[titan] 2025-07-09 22:33:38,602 - root - INFO - step: 40695 loss: 16.7128 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1308 +[titan] 2025-07-09 22:33:38,602 - root - INFO - lr: 2.0633e-04 gnorm: 0.79 [ 8:58:39<13:04:59] +[titan] 2025-07-09 22:33:41,713 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:33:42,514 - root - INFO - step: 40700 loss: 16.8280 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.6005 global_avg_mtp_loss: 14.2275 +[titan] 2025-07-09 22:33:42,515 - root - INFO - lr: 2.0631e-04 gnorm: 0.77 [ 8:58:43<13:04:55] +[titan] 2025-07-09 22:33:46,436 - root - INFO - step: 40705 loss: 16.8121 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.6056 global_avg_mtp_loss: 14.2065 +[titan] 2025-07-09 22:33:46,436 - root - INFO - lr: 2.0629e-04 gnorm: 0.75 [ 8:58:47<13:04:51] +[titan] 2025-07-09 22:33:50,341 - root - INFO - step: 40710 loss: 16.9477 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.6291 global_avg_mtp_loss: 14.3185 +[titan] 2025-07-09 22:33:50,341 - root - INFO - lr: 2.0627e-04 gnorm: 0.77 [ 8:58:51<13:04:47] +[titan] 2025-07-09 22:33:54,307 - root - INFO - step: 40715 loss: 16.4301 memory: 44.58GiB(31.99%) tps: 82,619 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 13.8880 +[titan] 2025-07-09 22:33:54,307 - root - INFO - lr: 2.0625e-04 gnorm: 0.74 [ 8:58:55<13:04:43] +[titan] 2025-07-09 22:33:58,226 - root - INFO - step: 40720 loss: 16.3841 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-09 22:33:58,226 - root - INFO - lr: 2.0623e-04 gnorm: 0.79 [ 8:58:59<13:04:39] +[titan] 2025-07-09 22:34:02,127 - root - INFO - step: 40725 loss: 16.8473 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.2481 +[titan] 2025-07-09 22:34:02,127 - root - INFO - lr: 2.0621e-04 gnorm: 0.78 [ 8:59:03<13:04:35] +[titan] 2025-07-09 22:34:06,032 - root - INFO - step: 40730 loss: 16.7137 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5884 global_avg_mtp_loss: 14.1253 +[titan] 2025-07-09 22:34:06,032 - root - INFO - lr: 2.0618e-04 gnorm: 0.80 [ 8:59:06<13:04:31] +[titan] 2025-07-09 22:34:09,938 - root - INFO - step: 40735 loss: 16.5739 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0101 +[titan] 2025-07-09 22:34:09,939 - root - INFO - lr: 2.0616e-04 gnorm: 0.80 [ 8:59:10<13:04:26] +[titan] 2025-07-09 22:34:13,851 - root - INFO - step: 40740 loss: 16.7435 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5883 global_avg_mtp_loss: 14.1552 +[titan] 2025-07-09 22:34:13,851 - root - INFO - lr: 2.0614e-04 gnorm: 0.77 [ 8:59:14<13:04:22] +[titan] 2025-07-09 22:34:17,761 - root - INFO - step: 40745 loss: 16.6806 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5845 global_avg_mtp_loss: 14.0961 +[titan] 2025-07-09 22:34:17,761 - root - INFO - lr: 2.0612e-04 gnorm: 0.80 [ 8:59:18<13:04:18] +[titan] 2025-07-09 22:34:20,934 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:34:21,718 - root - INFO - step: 40750 loss: 16.6673 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.1145 +[titan] 2025-07-09 22:34:21,719 - root - INFO - lr: 2.0610e-04 gnorm: 0.77 [ 8:59:22<13:04:14] +[titan] 2025-07-09 22:34:25,625 - root - INFO - step: 40755 loss: 16.5179 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9757 +[titan] 2025-07-09 22:34:25,626 - root - INFO - lr: 2.0608e-04 gnorm: 0.77 [ 8:59:26<13:04:10] +[titan] 2025-07-09 22:34:29,518 - root - INFO - step: 40760 loss: 16.6079 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0557 +[titan] 2025-07-09 22:34:29,518 - root - INFO - lr: 2.0606e-04 gnorm: 0.73 [ 8:59:30<13:04:06] +[titan] 2025-07-09 22:34:33,419 - root - INFO - step: 40765 loss: 16.5868 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0277 +[titan] 2025-07-09 22:34:33,420 - root - INFO - lr: 2.0604e-04 gnorm: 0.75 [ 8:59:34<13:04:02] +[titan] 2025-07-09 22:34:37,329 - root - INFO - step: 40770 loss: 16.5875 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-09 22:34:37,329 - root - INFO - lr: 2.0602e-04 gnorm: 0.78 [ 8:59:38<13:03:58] +[titan] 2025-07-09 22:34:41,224 - root - INFO - step: 40775 loss: 16.8860 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.6094 global_avg_mtp_loss: 14.2766 +[titan] 2025-07-09 22:34:41,224 - root - INFO - lr: 2.0600e-04 gnorm: 0.86 [ 8:59:42<13:03:54] +[titan] 2025-07-09 22:34:45,164 - root - INFO - step: 40780 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0888 +[titan] 2025-07-09 22:34:45,164 - root - INFO - lr: 2.0598e-04 gnorm: 0.77 [ 8:59:46<13:03:50] +[titan] 2025-07-09 22:34:49,104 - root - INFO - step: 40785 loss: 16.8849 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2815 +[titan] 2025-07-09 22:34:49,104 - root - INFO - lr: 2.0596e-04 gnorm: 0.76 [ 8:59:50<13:03:46] +[titan] 2025-07-09 22:34:53,024 - root - INFO - step: 40790 loss: 16.4797 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9236 +[titan] 2025-07-09 22:34:53,024 - root - INFO - lr: 2.0594e-04 gnorm: 0.75 [ 8:59:53<13:03:42] +[titan] 2025-07-09 22:34:56,944 - root - INFO - step: 40795 loss: 16.9546 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6341 global_avg_mtp_loss: 14.3205 +[titan] 2025-07-09 22:34:56,944 - root - INFO - lr: 2.0592e-04 gnorm: 0.78 [ 8:59:57<13:03:38] +[titan] 2025-07-09 22:35:00,094 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:35:00,880 - root - INFO - step: 40800 loss: 16.3509 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-09 22:35:00,880 - root - INFO - lr: 2.0590e-04 gnorm: 0.76 [ 9:00:01<13:03:34] +[titan] 2025-07-09 22:35:04,827 - root - INFO - step: 40805 loss: 16.6915 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1211 +[titan] 2025-07-09 22:35:04,827 - root - INFO - lr: 2.0588e-04 gnorm: 0.77 [ 9:00:05<13:03:30] +[titan] 2025-07-09 22:35:08,723 - root - INFO - step: 40810 loss: 16.4870 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9518 +[titan] 2025-07-09 22:35:08,723 - root - INFO - lr: 2.0586e-04 gnorm: 0.75 [ 9:00:09<13:03:26] +[titan] 2025-07-09 22:35:12,639 - root - INFO - step: 40815 loss: 16.6593 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0912 +[titan] 2025-07-09 22:35:12,639 - root - INFO - lr: 2.0584e-04 gnorm: 0.77 [ 9:00:13<13:03:22] +[titan] 2025-07-09 22:35:16,570 - root - INFO - step: 40820 loss: 16.6961 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1135 +[titan] 2025-07-09 22:35:16,571 - root - INFO - lr: 2.0582e-04 gnorm: 0.84 [ 9:00:17<13:03:18] +[titan] 2025-07-09 22:35:20,479 - root - INFO - step: 40825 loss: 16.2802 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.7824 +[titan] 2025-07-09 22:35:20,479 - root - INFO - lr: 2.0580e-04 gnorm: 0.82 [ 9:00:21<13:03:14] +[titan] 2025-07-09 22:35:24,398 - root - INFO - step: 40830 loss: 16.5436 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 14.0013 +[titan] 2025-07-09 22:35:24,398 - root - INFO - lr: 2.0578e-04 gnorm: 0.76 [ 9:00:25<13:03:10] +[titan] 2025-07-09 22:35:28,349 - root - INFO - step: 40835 loss: 16.9000 memory: 44.58GiB(31.99%) tps: 82,936 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.6360 global_avg_mtp_loss: 14.2641 +[titan] 2025-07-09 22:35:28,350 - root - INFO - lr: 2.0576e-04 gnorm: 0.77 [ 9:00:29<13:03:06] +[titan] 2025-07-09 22:35:32,248 - root - INFO - step: 40840 loss: 16.7133 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1334 +[titan] 2025-07-09 22:35:32,249 - root - INFO - lr: 2.0574e-04 gnorm: 0.78 [ 9:00:33<13:03:01] +[titan] 2025-07-09 22:35:36,202 - root - INFO - step: 40845 loss: 16.7225 memory: 44.58GiB(31.99%) tps: 82,897 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5810 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 22:35:36,202 - root - INFO - lr: 2.0572e-04 gnorm: 0.78 [ 9:00:37<13:02:58] +[titan] 2025-07-09 22:35:39,321 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:35:40,107 - root - INFO - step: 40850 loss: 16.6332 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-09 22:35:40,107 - root - INFO - lr: 2.0569e-04 gnorm: 0.83 [ 9:00:41<13:02:53] +[titan] 2025-07-09 22:35:44,059 - root - INFO - step: 40855 loss: 16.4545 memory: 44.58GiB(31.99%) tps: 82,919 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 13.9111 +[titan] 2025-07-09 22:35:44,059 - root - INFO - lr: 2.0567e-04 gnorm: 0.80 [ 9:00:44<13:02:49] +[titan] 2025-07-09 22:35:48,004 - root - INFO - step: 40860 loss: 16.5749 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0098 +[titan] 2025-07-09 22:35:48,004 - root - INFO - lr: 2.0565e-04 gnorm: 0.78 [ 9:00:48<13:02:45] +[titan] 2025-07-09 22:35:51,959 - root - INFO - step: 40865 loss: 16.6344 memory: 44.58GiB(31.99%) tps: 82,854 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.0616 +[titan] 2025-07-09 22:35:51,960 - root - INFO - lr: 2.0563e-04 gnorm: 0.76 [ 9:00:52<13:02:41] +[titan] 2025-07-09 22:35:55,864 - root - INFO - step: 40870 loss: 16.6901 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.1213 +[titan] 2025-07-09 22:35:55,864 - root - INFO - lr: 2.0561e-04 gnorm: 0.81 [ 9:00:56<13:02:37] +[titan] 2025-07-09 22:35:59,783 - root - INFO - step: 40875 loss: 16.8413 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6036 global_avg_mtp_loss: 14.2377 +[titan] 2025-07-09 22:35:59,783 - root - INFO - lr: 2.0559e-04 gnorm: 0.91 [ 9:01:00<13:02:33] +[titan] 2025-07-09 22:36:03,724 - root - INFO - step: 40880 loss: 16.4495 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.9069 +[titan] 2025-07-09 22:36:03,724 - root - INFO - lr: 2.0557e-04 gnorm: 0.78 [ 9:01:04<13:02:29] +[titan] 2025-07-09 22:36:07,662 - root - INFO - step: 40885 loss: 16.6654 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0899 +[titan] 2025-07-09 22:36:07,662 - root - INFO - lr: 2.0555e-04 gnorm: 0.79 [ 9:01:08<13:02:25] +[titan] 2025-07-09 22:36:11,604 - root - INFO - step: 40890 loss: 16.6836 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5888 global_avg_mtp_loss: 14.0948 +[titan] 2025-07-09 22:36:11,604 - root - INFO - lr: 2.0553e-04 gnorm: 0.90 [ 9:01:12<13:02:21] +[titan] 2025-07-09 22:36:15,549 - root - INFO - step: 40895 loss: 16.6046 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.0279 +[titan] 2025-07-09 22:36:15,549 - root - INFO - lr: 2.0551e-04 gnorm: 0.85 [ 9:01:16<13:02:17] +[titan] 2025-07-09 22:36:18,666 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:36:19,455 - root - INFO - step: 40900 loss: 16.4353 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.8986 +[titan] 2025-07-09 22:36:19,456 - root - INFO - lr: 2.0549e-04 gnorm: 0.78 [ 9:01:20<13:02:13] +[titan] 2025-07-09 22:36:23,367 - root - INFO - step: 40905 loss: 16.8025 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6022 global_avg_mtp_loss: 14.2003 +[titan] 2025-07-09 22:36:23,368 - root - INFO - lr: 2.0547e-04 gnorm: 1.64 [ 9:01:24<13:02:09] +[titan] 2025-07-09 22:36:27,326 - root - INFO - step: 40910 loss: 16.5057 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9622 +[titan] 2025-07-09 22:36:27,327 - root - INFO - lr: 2.0545e-04 gnorm: 0.82 [ 9:01:28<13:02:05] +[titan] 2025-07-09 22:36:31,240 - root - INFO - step: 40915 loss: 16.5254 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9891 +[titan] 2025-07-09 22:36:31,240 - root - INFO - lr: 2.0543e-04 gnorm: 0.86 [ 9:01:32<13:02:01] +[titan] 2025-07-09 22:36:35,148 - root - INFO - step: 40920 loss: 16.6368 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0699 +[titan] 2025-07-09 22:36:35,148 - root - INFO - lr: 2.0541e-04 gnorm: 0.82 [ 9:01:36<13:01:57] +[titan] 2025-07-09 22:36:39,059 - root - INFO - step: 40925 loss: 16.9028 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.6255 global_avg_mtp_loss: 14.2773 +[titan] 2025-07-09 22:36:39,060 - root - INFO - lr: 2.0539e-04 gnorm: 0.82 [ 9:01:39<13:01:53] +[titan] 2025-07-09 22:36:42,984 - root - INFO - step: 40930 loss: 16.6902 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.1104 +[titan] 2025-07-09 22:36:42,985 - root - INFO - lr: 2.0537e-04 gnorm: 0.75 [ 9:01:43<13:01:49] +[titan] 2025-07-09 22:36:46,950 - root - INFO - step: 40935 loss: 16.3827 memory: 44.58GiB(31.99%) tps: 82,648 tflops: 285.23 mfu: 28.84% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8734 +[titan] 2025-07-09 22:36:46,950 - root - INFO - lr: 2.0535e-04 gnorm: 0.75 [ 9:01:47<13:01:45] +[titan] 2025-07-09 22:36:50,862 - root - INFO - step: 40940 loss: 17.0739 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.6519 global_avg_mtp_loss: 14.4221 +[titan] 2025-07-09 22:36:50,862 - root - INFO - lr: 2.0533e-04 gnorm: 0.82 [ 9:01:51<13:01:41] +[titan] 2025-07-09 22:36:54,776 - root - INFO - step: 40945 loss: 16.5382 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 13.9789 +[titan] 2025-07-09 22:36:54,777 - root - INFO - lr: 2.0531e-04 gnorm: 0.79 [ 9:01:55<13:01:37] +[titan] 2025-07-09 22:36:57,898 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:36:58,692 - root - INFO - step: 40950 loss: 16.3790 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.8550 +[titan] 2025-07-09 22:36:58,692 - root - INFO - lr: 2.0529e-04 gnorm: 0.74 [ 9:01:59<13:01:33] +[titan] 2025-07-09 22:37:02,588 - root - INFO - step: 40955 loss: 16.7013 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.1253 +[titan] 2025-07-09 22:37:02,588 - root - INFO - lr: 2.0527e-04 gnorm: 0.80 [ 9:02:03<13:01:29] +[titan] 2025-07-09 22:37:06,573 - root - INFO - step: 40960 loss: 16.4402 memory: 44.58GiB(31.99%) tps: 82,237 tflops: 283.81 mfu: 28.70% global_avg_ntp_loss: 2.5348 global_avg_mtp_loss: 13.9054 +[titan] 2025-07-09 22:37:06,573 - root - INFO - lr: 2.0525e-04 gnorm: 0.78 [ 9:02:07<13:01:25] +[titan] 2025-07-09 22:37:06,727 - root - INFO - Dumping profiler traces at step 40960 +[titan] 2025-07-09 22:37:06,761 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:37:10,661 - root - INFO - step: 40965 loss: 16.7509 memory: 44.58GiB(31.99%) tps: 80,163 tflops: 276.66 mfu: 27.97% global_avg_ntp_loss: 2.5880 global_avg_mtp_loss: 14.1629 +[titan] 2025-07-09 22:37:10,661 - root - INFO - lr: 2.0522e-04 gnorm: 0.79 [ 9:02:11<13:01:21] +[titan] 2025-07-09 22:37:14,556 - root - INFO - step: 40970 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.8244 +[titan] 2025-07-09 22:37:14,556 - root - INFO - lr: 2.0520e-04 gnorm: 0.86 [ 9:02:15<13:01:17] +[titan] 2025-07-09 22:37:18,469 - root - INFO - step: 40975 loss: 16.7790 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1917 +[titan] 2025-07-09 22:37:18,470 - root - INFO - lr: 2.0518e-04 gnorm: 0.75 [ 9:02:19<13:01:13] +[titan] 2025-07-09 22:37:22,366 - root - INFO - step: 40980 loss: 16.8038 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.1929 +[titan] 2025-07-09 22:37:22,366 - root - INFO - lr: 2.0516e-04 gnorm: 0.82 [ 9:02:23<13:01:09] +[titan] 2025-07-09 22:37:26,269 - root - INFO - step: 40985 loss: 16.7427 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5900 global_avg_mtp_loss: 14.1526 +[titan] 2025-07-09 22:37:26,269 - root - INFO - lr: 2.0514e-04 gnorm: 0.78 [ 9:02:27<13:01:05] +[titan] 2025-07-09 22:37:30,187 - root - INFO - step: 40990 loss: 16.7030 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5777 global_avg_mtp_loss: 14.1254 +[titan] 2025-07-09 22:37:30,187 - root - INFO - lr: 2.0512e-04 gnorm: 0.81 [ 9:02:31<13:01:01] +[titan] 2025-07-09 22:37:34,095 - root - INFO - step: 40995 loss: 16.6288 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.0589 +[titan] 2025-07-09 22:37:34,096 - root - INFO - lr: 2.0510e-04 gnorm: 0.80 [ 9:02:34<13:00:57] +[titan] 2025-07-09 22:37:37,230 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:37:38,022 - root - INFO - step: 41000 loss: 16.6921 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.1199 +[titan] 2025-07-09 22:37:38,022 - root - INFO - lr: 2.0508e-04 gnorm: 0.76 [ 9:02:38<13:00:53] +[titan] 2025-07-09 22:37:42,020 - root - INFO - step: 41005 loss: 16.2879 memory: 44.58GiB(31.99%) tps: 81,956 tflops: 282.85 mfu: 28.60% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.7725 +[titan] 2025-07-09 22:37:42,021 - root - INFO - lr: 2.0506e-04 gnorm: 0.82 [ 9:02:42<13:00:49] +[titan] 2025-07-09 22:37:45,931 - root - INFO - step: 41010 loss: 16.5661 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0123 +[titan] 2025-07-09 22:37:45,931 - root - INFO - lr: 2.0504e-04 gnorm: 0.85 [ 9:02:46<13:00:45] +[titan] 2025-07-09 22:37:49,858 - root - INFO - step: 41015 loss: 16.9020 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.6248 global_avg_mtp_loss: 14.2773 +[titan] 2025-07-09 22:37:49,858 - root - INFO - lr: 2.0502e-04 gnorm: 0.82 [ 9:02:50<13:00:41] +[titan] 2025-07-09 22:37:53,797 - root - INFO - step: 41020 loss: 16.6949 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-09 22:37:53,797 - root - INFO - lr: 2.0500e-04 gnorm: 0.75 [ 9:02:54<13:00:37] +[titan] 2025-07-09 22:37:57,743 - root - INFO - step: 41025 loss: 16.7390 memory: 44.58GiB(31.99%) tps: 83,060 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1417 +[titan] 2025-07-09 22:37:57,743 - root - INFO - lr: 2.0498e-04 gnorm: 0.79 [ 9:02:58<13:00:33] +[titan] 2025-07-09 22:38:01,667 - root - INFO - step: 41030 loss: 16.9760 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.6360 global_avg_mtp_loss: 14.3400 +[titan] 2025-07-09 22:38:01,668 - root - INFO - lr: 2.0496e-04 gnorm: 0.80 [ 9:03:02<13:00:28] +[titan] 2025-07-09 22:38:05,579 - root - INFO - step: 41035 loss: 16.6714 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 14.1002 +[titan] 2025-07-09 22:38:05,580 - root - INFO - lr: 2.0494e-04 gnorm: 0.77 [ 9:03:06<13:00:24] +[titan] 2025-07-09 22:38:09,469 - root - INFO - step: 41040 loss: 16.9182 memory: 44.58GiB(31.99%) tps: 84,252 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.6184 global_avg_mtp_loss: 14.2998 +[titan] 2025-07-09 22:38:09,469 - root - INFO - lr: 2.0492e-04 gnorm: 0.76 [ 9:03:10<13:00:20] +[titan] 2025-07-09 22:38:13,387 - root - INFO - step: 41045 loss: 16.6606 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0993 +[titan] 2025-07-09 22:38:13,387 - root - INFO - lr: 2.0490e-04 gnorm: 0.79 [ 9:03:14<13:00:16] +[titan] 2025-07-09 22:38:16,524 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:38:17,318 - root - INFO - step: 41050 loss: 16.7023 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1116 +[titan] 2025-07-09 22:38:17,318 - root - INFO - lr: 2.0488e-04 gnorm: 0.79 [ 9:03:18<13:00:12] +[titan] 2025-07-09 22:38:21,228 - root - INFO - step: 41055 loss: 16.6339 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0662 +[titan] 2025-07-09 22:38:21,229 - root - INFO - lr: 2.0486e-04 gnorm: 0.80 [ 9:03:22<13:00:08] +[titan] 2025-07-09 22:38:25,133 - root - INFO - step: 41060 loss: 16.4641 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9272 +[titan] 2025-07-09 22:38:25,133 - root - INFO - lr: 2.0484e-04 gnorm: 0.84 [ 9:03:26<13:00:04] +[titan] 2025-07-09 22:38:29,051 - root - INFO - step: 41065 loss: 16.3848 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.8623 +[titan] 2025-07-09 22:38:29,051 - root - INFO - lr: 2.0482e-04 gnorm: 0.76 [ 9:03:29<13:00:00] +[titan] 2025-07-09 22:38:32,953 - root - INFO - step: 41070 loss: 16.7120 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.1161 +[titan] 2025-07-09 22:38:32,954 - root - INFO - lr: 2.0480e-04 gnorm: 1.11 [ 9:03:33<12:59:56] +[titan] 2025-07-09 22:38:36,850 - root - INFO - step: 41075 loss: 16.8359 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.6091 global_avg_mtp_loss: 14.2268 +[titan] 2025-07-09 22:38:36,850 - root - INFO - lr: 2.0477e-04 gnorm: 0.81 [ 9:03:37<12:59:52] +[titan] 2025-07-09 22:38:40,782 - root - INFO - step: 41080 loss: 16.4233 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.9005 +[titan] 2025-07-09 22:38:40,783 - root - INFO - lr: 2.0475e-04 gnorm: 0.80 [ 9:03:41<12:59:48] +[titan] 2025-07-09 22:38:44,686 - root - INFO - step: 41085 loss: 16.7467 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.1731 +[titan] 2025-07-09 22:38:44,686 - root - INFO - lr: 2.0473e-04 gnorm: 0.87 [ 9:03:45<12:59:44] +[titan] 2025-07-09 22:38:48,597 - root - INFO - step: 41090 loss: 16.5558 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 13.9992 +[titan] 2025-07-09 22:38:48,598 - root - INFO - lr: 2.0471e-04 gnorm: 0.79 [ 9:03:49<12:59:40] +[titan] 2025-07-09 22:38:52,522 - root - INFO - step: 41095 loss: 16.6975 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.1278 +[titan] 2025-07-09 22:38:52,522 - root - INFO - lr: 2.0469e-04 gnorm: 0.75 [ 9:03:53<12:59:36] +[titan] 2025-07-09 22:38:55,633 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:38:56,436 - root - INFO - step: 41100 loss: 16.3733 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.8482 +[titan] 2025-07-09 22:38:56,436 - root - INFO - lr: 2.0467e-04 gnorm: 0.73 [ 9:03:57<12:59:32] +[titan] 2025-07-09 22:39:00,338 - root - INFO - step: 41105 loss: 16.7210 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1415 +[titan] 2025-07-09 22:39:00,338 - root - INFO - lr: 2.0465e-04 gnorm: 0.83 [ 9:04:01<12:59:28] +[titan] 2025-07-09 22:39:04,247 - root - INFO - step: 41110 loss: 16.5471 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5635 global_avg_mtp_loss: 13.9835 +[titan] 2025-07-09 22:39:04,247 - root - INFO - lr: 2.0463e-04 gnorm: 0.79 [ 9:04:05<12:59:24] +[titan] 2025-07-09 22:39:08,152 - root - INFO - step: 41115 loss: 16.5186 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9700 +[titan] 2025-07-09 22:39:08,152 - root - INFO - lr: 2.0461e-04 gnorm: 0.90 [ 9:04:09<12:59:20] +[titan] 2025-07-09 22:39:12,068 - root - INFO - step: 41120 loss: 16.5872 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5663 global_avg_mtp_loss: 14.0209 +[titan] 2025-07-09 22:39:12,068 - root - INFO - lr: 2.0459e-04 gnorm: 0.84 [ 9:04:12<12:59:15] +[titan] 2025-07-09 22:39:15,978 - root - INFO - step: 41125 loss: 16.8193 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.2145 +[titan] 2025-07-09 22:39:15,978 - root - INFO - lr: 2.0457e-04 gnorm: 0.85 [ 9:04:16<12:59:11] +[titan] 2025-07-09 22:39:19,928 - root - INFO - step: 41130 loss: 16.6993 memory: 44.58GiB(31.99%) tps: 82,976 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.1251 +[titan] 2025-07-09 22:39:19,928 - root - INFO - lr: 2.0455e-04 gnorm: 0.78 [ 9:04:20<12:59:07] +[titan] 2025-07-09 22:39:23,823 - root - INFO - step: 41135 loss: 16.8191 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.6017 global_avg_mtp_loss: 14.2174 +[titan] 2025-07-09 22:39:23,823 - root - INFO - lr: 2.0453e-04 gnorm: 0.77 [ 9:04:24<12:59:03] +[titan] 2025-07-09 22:39:27,736 - root - INFO - step: 41140 loss: 16.6884 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1079 +[titan] 2025-07-09 22:39:27,736 - root - INFO - lr: 2.0451e-04 gnorm: 0.73 [ 9:04:28<12:58:59] +[titan] 2025-07-09 22:39:31,640 - root - INFO - step: 41145 loss: 16.8086 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.6042 global_avg_mtp_loss: 14.2044 +[titan] 2025-07-09 22:39:31,640 - root - INFO - lr: 2.0449e-04 gnorm: 0.78 [ 9:04:32<12:58:55] +[titan] 2025-07-09 22:39:34,755 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:39:35,571 - root - INFO - step: 41150 loss: 16.5253 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9771 +[titan] 2025-07-09 22:39:35,571 - root - INFO - lr: 2.0447e-04 gnorm: 0.79 [ 9:04:36<12:58:51] +[titan] 2025-07-09 22:39:39,500 - root - INFO - step: 41155 loss: 16.7039 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-09 22:39:39,501 - root - INFO - lr: 2.0445e-04 gnorm: 0.77 [ 9:04:40<12:58:47] +[titan] 2025-07-09 22:39:43,441 - root - INFO - step: 41160 loss: 16.6615 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.0750 +[titan] 2025-07-09 22:39:43,441 - root - INFO - lr: 2.0443e-04 gnorm: 0.76 [ 9:04:44<12:58:43] +[titan] 2025-07-09 22:39:47,339 - root - INFO - step: 41165 loss: 16.9062 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.2959 +[titan] 2025-07-09 22:39:47,340 - root - INFO - lr: 2.0441e-04 gnorm: 0.82 [ 9:04:48<12:58:39] +[titan] 2025-07-09 22:39:51,283 - root - INFO - step: 41170 loss: 16.6534 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.0760 +[titan] 2025-07-09 22:39:51,283 - root - INFO - lr: 2.0439e-04 gnorm: 0.79 [ 9:04:52<12:58:35] +[titan] 2025-07-09 22:39:55,231 - root - INFO - step: 41175 loss: 16.6954 memory: 44.58GiB(31.99%) tps: 83,020 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1110 +[titan] 2025-07-09 22:39:55,231 - root - INFO - lr: 2.0437e-04 gnorm: 0.77 [ 9:04:56<12:58:31] +[titan] 2025-07-09 22:39:59,156 - root - INFO - step: 41180 loss: 16.2378 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.7433 +[titan] 2025-07-09 22:39:59,157 - root - INFO - lr: 2.0434e-04 gnorm: 0.81 [ 9:05:00<12:58:27] +[titan] 2025-07-09 22:40:03,129 - root - INFO - step: 41185 loss: 16.6666 memory: 44.58GiB(31.99%) tps: 82,488 tflops: 284.68 mfu: 28.78% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0902 +[titan] 2025-07-09 22:40:03,129 - root - INFO - lr: 2.0432e-04 gnorm: 0.77 [ 9:05:03<12:58:23] +[titan] 2025-07-09 22:40:07,028 - root - INFO - step: 41190 loss: 16.5635 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0089 +[titan] 2025-07-09 22:40:07,028 - root - INFO - lr: 2.0430e-04 gnorm: 0.77 [ 9:05:07<12:58:19] +[titan] 2025-07-09 22:40:10,922 - root - INFO - step: 41195 loss: 16.7440 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1678 +[titan] 2025-07-09 22:40:10,922 - root - INFO - lr: 2.0428e-04 gnorm: 0.77 [ 9:05:11<12:58:15] +[titan] 2025-07-09 22:40:14,042 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:40:14,827 - root - INFO - step: 41200 loss: 16.5810 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5701 global_avg_mtp_loss: 14.0109 +[titan] 2025-07-09 22:40:14,827 - root - INFO - lr: 2.0426e-04 gnorm: 0.75 [ 9:05:15<12:58:11] +[titan] 2025-07-09 22:40:18,746 - root - INFO - step: 41205 loss: 16.7556 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1710 +[titan] 2025-07-09 22:40:18,746 - root - INFO - lr: 2.0424e-04 gnorm: 0.76 [ 9:05:19<12:58:07] +[titan] 2025-07-09 22:40:22,682 - root - INFO - step: 41210 loss: 16.7328 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5855 global_avg_mtp_loss: 14.1473 +[titan] 2025-07-09 22:40:22,682 - root - INFO - lr: 2.0422e-04 gnorm: 0.80 [ 9:05:23<12:58:03] +[titan] 2025-07-09 22:40:26,606 - root - INFO - step: 41215 loss: 16.7015 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.1183 +[titan] 2025-07-09 22:40:26,606 - root - INFO - lr: 2.0420e-04 gnorm: 0.77 [ 9:05:27<12:57:59] +[titan] 2025-07-09 22:40:30,511 - root - INFO - step: 41220 loss: 16.6642 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.0874 +[titan] 2025-07-09 22:40:30,511 - root - INFO - lr: 2.0418e-04 gnorm: 0.77 [ 9:05:31<12:57:55] +[titan] 2025-07-09 22:40:34,429 - root - INFO - step: 41225 loss: 16.5898 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 14.0243 +[titan] 2025-07-09 22:40:34,429 - root - INFO - lr: 2.0416e-04 gnorm: 0.76 [ 9:05:35<12:57:51] +[titan] 2025-07-09 22:40:38,342 - root - INFO - step: 41230 loss: 16.4136 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8943 +[titan] 2025-07-09 22:40:38,342 - root - INFO - lr: 2.0414e-04 gnorm: 0.76 [ 9:05:39<12:57:47] +[titan] 2025-07-09 22:40:42,237 - root - INFO - step: 41235 loss: 16.4823 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.9494 +[titan] 2025-07-09 22:40:42,237 - root - INFO - lr: 2.0412e-04 gnorm: 0.79 [ 9:05:43<12:57:42] +[titan] 2025-07-09 22:40:46,177 - root - INFO - step: 41240 loss: 16.5027 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9516 +[titan] 2025-07-09 22:40:46,177 - root - INFO - lr: 2.0410e-04 gnorm: 0.77 [ 9:05:47<12:57:38] +[titan] 2025-07-09 22:40:50,121 - root - INFO - step: 41245 loss: 16.6131 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0538 +[titan] 2025-07-09 22:40:50,121 - root - INFO - lr: 2.0408e-04 gnorm: 0.80 [ 9:05:50<12:57:34] +[titan] 2025-07-09 22:40:53,245 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:40:54,038 - root - INFO - step: 41250 loss: 16.5815 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0329 +[titan] 2025-07-09 22:40:54,038 - root - INFO - lr: 2.0406e-04 gnorm: 0.77 [ 9:05:54<12:57:30] +[titan] 2025-07-09 22:40:57,966 - root - INFO - step: 41255 loss: 16.5306 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 13.9719 +[titan] 2025-07-09 22:40:57,966 - root - INFO - lr: 2.0404e-04 gnorm: 0.85 [ 9:05:58<12:57:26] +[titan] 2025-07-09 22:41:01,883 - root - INFO - step: 41260 loss: 16.5123 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9642 +[titan] 2025-07-09 22:41:01,884 - root - INFO - lr: 2.0402e-04 gnorm: 0.77 [ 9:06:02<12:57:22] +[titan] 2025-07-09 22:41:05,808 - root - INFO - step: 41265 loss: 16.4690 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9307 +[titan] 2025-07-09 22:41:05,809 - root - INFO - lr: 2.0400e-04 gnorm: 0.72 [ 9:06:06<12:57:18] +[titan] 2025-07-09 22:41:09,720 - root - INFO - step: 41270 loss: 16.4047 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8775 +[titan] 2025-07-09 22:41:09,720 - root - INFO - lr: 2.0398e-04 gnorm: 0.83 [ 9:06:10<12:57:14] +[titan] 2025-07-09 22:41:13,643 - root - INFO - step: 41275 loss: 16.7457 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.1491 +[titan] 2025-07-09 22:41:13,643 - root - INFO - lr: 2.0396e-04 gnorm: 0.78 [ 9:06:14<12:57:10] +[titan] 2025-07-09 22:41:17,541 - root - INFO - step: 41280 loss: 16.6939 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.1202 +[titan] 2025-07-09 22:41:17,541 - root - INFO - lr: 2.0393e-04 gnorm: 0.82 [ 9:06:18<12:57:06] +[titan] 2025-07-09 22:41:21,448 - root - INFO - step: 41285 loss: 16.5163 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9761 +[titan] 2025-07-09 22:41:21,449 - root - INFO - lr: 2.0391e-04 gnorm: 0.79 [ 9:06:22<12:57:02] +[titan] 2025-07-09 22:41:25,343 - root - INFO - step: 41290 loss: 16.7190 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5827 global_avg_mtp_loss: 14.1363 +[titan] 2025-07-09 22:41:25,343 - root - INFO - lr: 2.0389e-04 gnorm: 0.80 [ 9:06:26<12:56:58] +[titan] 2025-07-09 22:41:29,303 - root - INFO - step: 41295 loss: 16.4648 memory: 44.58GiB(31.99%) tps: 82,750 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-09 22:41:29,304 - root - INFO - lr: 2.0387e-04 gnorm: 0.73 [ 9:06:30<12:56:54] +[titan] 2025-07-09 22:41:32,451 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:41:33,238 - root - INFO - step: 41300 loss: 16.5340 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 13.9765 +[titan] 2025-07-09 22:41:33,238 - root - INFO - lr: 2.0385e-04 gnorm: 0.83 [ 9:06:34<12:56:50] +[titan] 2025-07-09 22:41:37,166 - root - INFO - step: 41305 loss: 16.6952 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.1158 +[titan] 2025-07-09 22:41:37,166 - root - INFO - lr: 2.0383e-04 gnorm: 0.72 [ 9:06:38<12:56:46] +[titan] 2025-07-09 22:41:41,114 - root - INFO - step: 41310 loss: 16.4928 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9644 +[titan] 2025-07-09 22:41:41,114 - root - INFO - lr: 2.0381e-04 gnorm: 0.74 [ 9:06:41<12:56:42] +[titan] 2025-07-09 22:41:45,032 - root - INFO - step: 41315 loss: 16.6243 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 14.0568 +[titan] 2025-07-09 22:41:45,032 - root - INFO - lr: 2.0379e-04 gnorm: 0.80 [ 9:06:45<12:56:38] +[titan] 2025-07-09 22:41:48,953 - root - INFO - step: 41320 loss: 16.5792 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-09 22:41:48,954 - root - INFO - lr: 2.0377e-04 gnorm: 0.82 [ 9:06:49<12:56:34] +[titan] 2025-07-09 22:41:52,951 - root - INFO - step: 41325 loss: 16.5205 memory: 44.58GiB(31.99%) tps: 81,984 tflops: 282.94 mfu: 28.61% global_avg_ntp_loss: 2.5490 global_avg_mtp_loss: 13.9715 +[titan] 2025-07-09 22:41:52,951 - root - INFO - lr: 2.0375e-04 gnorm: 0.75 [ 9:06:53<12:56:30] +[titan] 2025-07-09 22:41:56,878 - root - INFO - step: 41330 loss: 16.2186 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7258 +[titan] 2025-07-09 22:41:56,878 - root - INFO - lr: 2.0373e-04 gnorm: 0.78 [ 9:06:57<12:56:26] +[titan] 2025-07-09 22:42:00,778 - root - INFO - step: 41335 loss: 16.6944 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5850 global_avg_mtp_loss: 14.1094 +[titan] 2025-07-09 22:42:00,778 - root - INFO - lr: 2.0371e-04 gnorm: 0.78 [ 9:07:01<12:56:22] +[titan] 2025-07-09 22:42:04,693 - root - INFO - step: 41340 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.1222 +[titan] 2025-07-09 22:42:04,694 - root - INFO - lr: 2.0369e-04 gnorm: 0.79 [ 9:07:05<12:56:18] +[titan] 2025-07-09 22:42:08,616 - root - INFO - step: 41345 loss: 16.6591 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.0684 +[titan] 2025-07-09 22:42:08,616 - root - INFO - lr: 2.0367e-04 gnorm: 0.76 [ 9:07:09<12:56:14] +[titan] 2025-07-09 22:42:11,746 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:42:12,571 - root - INFO - step: 41350 loss: 16.7418 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1637 +[titan] 2025-07-09 22:42:12,572 - root - INFO - lr: 2.0365e-04 gnorm: 0.74 [ 9:07:13<12:56:10] +[titan] 2025-07-09 22:42:16,491 - root - INFO - step: 41355 loss: 16.6076 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0416 +[titan] 2025-07-09 22:42:16,491 - root - INFO - lr: 2.0363e-04 gnorm: 0.83 [ 9:07:17<12:56:06] +[titan] 2025-07-09 22:42:20,411 - root - INFO - step: 41360 loss: 16.7621 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.1534 +[titan] 2025-07-09 22:42:20,412 - root - INFO - lr: 2.0361e-04 gnorm: 0.99 [ 9:07:21<12:56:02] +[titan] 2025-07-09 22:42:24,339 - root - INFO - step: 41365 loss: 16.6595 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0907 +[titan] 2025-07-09 22:42:24,339 - root - INFO - lr: 2.0359e-04 gnorm: 0.77 [ 9:07:25<12:55:58] +[titan] 2025-07-09 22:42:28,257 - root - INFO - step: 41370 loss: 16.4890 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9408 +[titan] 2025-07-09 22:42:28,257 - root - INFO - lr: 2.0357e-04 gnorm: 0.75 [ 9:07:29<12:55:54] +[titan] 2025-07-09 22:42:32,171 - root - INFO - step: 41375 loss: 16.4880 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9438 +[titan] 2025-07-09 22:42:32,172 - root - INFO - lr: 2.0354e-04 gnorm: 0.77 [ 9:07:33<12:55:50] +[titan] 2025-07-09 22:42:36,074 - root - INFO - step: 41380 loss: 16.7656 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6217 global_avg_mtp_loss: 14.1439 +[titan] 2025-07-09 22:42:36,075 - root - INFO - lr: 2.0352e-04 gnorm: 0.79 [ 9:07:36<12:55:45] +[titan] 2025-07-09 22:42:39,995 - root - INFO - step: 41385 loss: 16.4688 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9225 +[titan] 2025-07-09 22:42:39,996 - root - INFO - lr: 2.0350e-04 gnorm: 0.85 [ 9:07:40<12:55:41] +[titan] 2025-07-09 22:42:43,959 - root - INFO - step: 41390 loss: 16.7831 memory: 44.58GiB(31.99%) tps: 82,690 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.6100 global_avg_mtp_loss: 14.1731 +[titan] 2025-07-09 22:42:43,959 - root - INFO - lr: 2.0348e-04 gnorm: 0.80 [ 9:07:44<12:55:37] +[titan] 2025-07-09 22:42:47,866 - root - INFO - step: 41395 loss: 16.5147 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 13.9504 +[titan] 2025-07-09 22:42:47,867 - root - INFO - lr: 2.0346e-04 gnorm: 0.76 [ 9:07:48<12:55:33] +[titan] 2025-07-09 22:42:51,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:42:51,792 - root - INFO - step: 41400 loss: 16.8640 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5986 global_avg_mtp_loss: 14.2654 +[titan] 2025-07-09 22:42:51,792 - root - INFO - lr: 2.0344e-04 gnorm: 0.82 [ 9:07:52<12:55:29] +[titan] 2025-07-09 22:42:55,733 - root - INFO - step: 41405 loss: 16.4470 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 13.9111 +[titan] 2025-07-09 22:42:55,733 - root - INFO - lr: 2.0342e-04 gnorm: 0.82 [ 9:07:56<12:55:25] +[titan] 2025-07-09 22:42:59,633 - root - INFO - step: 41410 loss: 16.4119 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8846 +[titan] 2025-07-09 22:42:59,633 - root - INFO - lr: 2.0340e-04 gnorm: 0.88 [ 9:08:00<12:55:21] +[titan] 2025-07-09 22:43:03,529 - root - INFO - step: 41415 loss: 16.4614 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9209 +[titan] 2025-07-09 22:43:03,529 - root - INFO - lr: 2.0338e-04 gnorm: 0.81 [ 9:08:04<12:55:17] +[titan] 2025-07-09 22:43:07,438 - root - INFO - step: 41420 loss: 16.6565 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.0882 +[titan] 2025-07-09 22:43:07,438 - root - INFO - lr: 2.0336e-04 gnorm: 0.81 [ 9:08:08<12:55:13] +[titan] 2025-07-09 22:43:11,423 - root - INFO - step: 41425 loss: 16.6000 memory: 44.58GiB(31.99%) tps: 82,239 tflops: 283.82 mfu: 28.70% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-09 22:43:11,423 - root - INFO - lr: 2.0334e-04 gnorm: 0.76 [ 9:08:12<12:55:09] +[titan] 2025-07-09 22:43:15,383 - root - INFO - step: 41430 loss: 16.6707 memory: 44.58GiB(31.99%) tps: 82,750 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.0947 +[titan] 2025-07-09 22:43:15,384 - root - INFO - lr: 2.0332e-04 gnorm: 0.75 [ 9:08:16<12:55:05] +[titan] 2025-07-09 22:43:19,295 - root - INFO - step: 41435 loss: 16.5788 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.0260 +[titan] 2025-07-09 22:43:19,295 - root - INFO - lr: 2.0330e-04 gnorm: 0.74 [ 9:08:20<12:55:01] +[titan] 2025-07-09 22:43:23,205 - root - INFO - step: 41440 loss: 16.6534 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.0753 +[titan] 2025-07-09 22:43:23,205 - root - INFO - lr: 2.0328e-04 gnorm: 0.78 [ 9:08:24<12:54:57] +[titan] 2025-07-09 22:43:27,120 - root - INFO - step: 41445 loss: 16.6330 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0592 +[titan] 2025-07-09 22:43:27,120 - root - INFO - lr: 2.0326e-04 gnorm: 0.77 [ 9:08:27<12:54:53] +[titan] 2025-07-09 22:43:30,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:43:31,042 - root - INFO - step: 41450 loss: 16.7026 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5785 global_avg_mtp_loss: 14.1241 +[titan] 2025-07-09 22:43:31,043 - root - INFO - lr: 2.0324e-04 gnorm: 0.76 [ 9:08:31<12:54:49] +[titan] 2025-07-09 22:43:34,943 - root - INFO - step: 41455 loss: 16.4530 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9219 +[titan] 2025-07-09 22:43:34,943 - root - INFO - lr: 2.0322e-04 gnorm: 0.80 [ 9:08:35<12:54:45] +[titan] 2025-07-09 22:43:38,849 - root - INFO - step: 41460 loss: 16.5452 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 14.0016 +[titan] 2025-07-09 22:43:38,849 - root - INFO - lr: 2.0320e-04 gnorm: 0.77 [ 9:08:39<12:54:41] +[titan] 2025-07-09 22:43:42,756 - root - INFO - step: 41465 loss: 16.6864 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.1146 +[titan] 2025-07-09 22:43:42,756 - root - INFO - lr: 2.0318e-04 gnorm: 0.79 [ 9:08:43<12:54:37] +[titan] 2025-07-09 22:43:46,730 - root - INFO - step: 41470 loss: 16.5889 memory: 44.58GiB(31.99%) tps: 82,462 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.5573 global_avg_mtp_loss: 14.0316 +[titan] 2025-07-09 22:43:46,731 - root - INFO - lr: 2.0315e-04 gnorm: 0.76 [ 9:08:47<12:54:33] +[titan] 2025-07-09 22:43:48,441 - root - INFO - Dumping profiler traces at step 41472 +[titan] 2025-07-09 22:43:48,473 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:43:50,819 - root - INFO - step: 41475 loss: 16.6018 memory: 44.58GiB(31.99%) tps: 80,159 tflops: 276.64 mfu: 27.97% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0479 +[titan] 2025-07-09 22:43:50,819 - root - INFO - lr: 2.0313e-04 gnorm: 0.82 [ 9:08:51<12:54:29] +[titan] 2025-07-09 22:43:54,729 - root - INFO - step: 41480 loss: 16.5760 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0154 +[titan] 2025-07-09 22:43:54,729 - root - INFO - lr: 2.0311e-04 gnorm: 0.79 [ 9:08:55<12:54:25] +[titan] 2025-07-09 22:43:58,652 - root - INFO - step: 41485 loss: 16.6653 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0979 +[titan] 2025-07-09 22:43:58,653 - root - INFO - lr: 2.0309e-04 gnorm: 0.79 [ 9:08:59<12:54:21] +[titan] 2025-07-09 22:44:02,579 - root - INFO - step: 41490 loss: 16.8465 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6005 global_avg_mtp_loss: 14.2460 +[titan] 2025-07-09 22:44:02,579 - root - INFO - lr: 2.0307e-04 gnorm: 0.73 [ 9:09:03<12:54:17] +[titan] 2025-07-09 22:44:06,526 - root - INFO - step: 41495 loss: 16.5332 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9815 +[titan] 2025-07-09 22:44:06,526 - root - INFO - lr: 2.0305e-04 gnorm: 0.77 [ 9:09:07<12:54:13] +[titan] 2025-07-09 22:44:09,677 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:44:10,475 - root - INFO - step: 41500 loss: 16.5778 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 14.0270 +[titan] 2025-07-09 22:44:10,475 - root - INFO - lr: 2.0303e-04 gnorm: 0.84 [ 9:09:11<12:54:09] +[titan] 2025-07-09 22:44:14,386 - root - INFO - step: 41505 loss: 16.5036 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9629 +[titan] 2025-07-09 22:44:14,386 - root - INFO - lr: 2.0301e-04 gnorm: 0.76 [ 9:09:15<12:54:05] +[titan] 2025-07-09 22:44:18,311 - root - INFO - step: 41510 loss: 16.5644 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-09 22:44:18,311 - root - INFO - lr: 2.0299e-04 gnorm: 0.76 [ 9:09:19<12:54:01] +[titan] 2025-07-09 22:44:22,228 - root - INFO - step: 41515 loss: 16.4863 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 13.9390 +[titan] 2025-07-09 22:44:22,228 - root - INFO - lr: 2.0297e-04 gnorm: 0.75 [ 9:09:23<12:53:57] +[titan] 2025-07-09 22:44:26,135 - root - INFO - step: 41520 loss: 16.6564 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0784 +[titan] 2025-07-09 22:44:26,135 - root - INFO - lr: 2.0295e-04 gnorm: 0.76 [ 9:09:26<12:53:53] +[titan] 2025-07-09 22:44:30,071 - root - INFO - step: 41525 loss: 16.5377 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.9936 +[titan] 2025-07-09 22:44:30,071 - root - INFO - lr: 2.0293e-04 gnorm: 0.78 [ 9:09:30<12:53:49] +[titan] 2025-07-09 22:44:33,973 - root - INFO - step: 41530 loss: 16.7719 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5957 global_avg_mtp_loss: 14.1762 +[titan] 2025-07-09 22:44:33,974 - root - INFO - lr: 2.0291e-04 gnorm: 0.78 [ 9:09:34<12:53:45] +[titan] 2025-07-09 22:44:37,886 - root - INFO - step: 41535 loss: 16.7910 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.1964 +[titan] 2025-07-09 22:44:37,887 - root - INFO - lr: 2.0289e-04 gnorm: 0.79 [ 9:09:38<12:53:41] +[titan] 2025-07-09 22:44:41,818 - root - INFO - step: 41540 loss: 16.5522 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 14.0083 +[titan] 2025-07-09 22:44:41,819 - root - INFO - lr: 2.0287e-04 gnorm: 0.78 [ 9:09:42<12:53:37] +[titan] 2025-07-09 22:44:45,728 - root - INFO - step: 41545 loss: 16.7193 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1347 +[titan] 2025-07-09 22:44:45,729 - root - INFO - lr: 2.0285e-04 gnorm: 0.74 [ 9:09:46<12:53:33] +[titan] 2025-07-09 22:44:48,877 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:44:49,684 - root - INFO - step: 41550 loss: 16.5211 memory: 44.58GiB(31.99%) tps: 82,858 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9885 +[titan] 2025-07-09 22:44:49,684 - root - INFO - lr: 2.0283e-04 gnorm: 0.80 [ 9:09:50<12:53:29] +[titan] 2025-07-09 22:44:53,626 - root - INFO - step: 41555 loss: 16.5987 memory: 44.58GiB(31.99%) tps: 83,113 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5583 global_avg_mtp_loss: 14.0404 +[titan] 2025-07-09 22:44:53,627 - root - INFO - lr: 2.0281e-04 gnorm: 0.78 [ 9:09:54<12:53:25] +[titan] 2025-07-09 22:44:57,531 - root - INFO - step: 41560 loss: 16.4728 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 13.9327 +[titan] 2025-07-09 22:44:57,532 - root - INFO - lr: 2.0278e-04 gnorm: 0.82 [ 9:09:58<12:53:20] +[titan] 2025-07-09 22:45:01,479 - root - INFO - step: 41565 loss: 16.4449 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 13.8982 +[titan] 2025-07-09 22:45:01,479 - root - INFO - lr: 2.0276e-04 gnorm: 0.90 [ 9:10:02<12:53:16] +[titan] 2025-07-09 22:45:05,470 - root - INFO - step: 41570 loss: 16.7042 memory: 44.58GiB(31.99%) tps: 82,112 tflops: 283.38 mfu: 28.65% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1327 +[titan] 2025-07-09 22:45:05,470 - root - INFO - lr: 2.0274e-04 gnorm: 0.79 [ 9:10:06<12:53:13] +[titan] 2025-07-09 22:45:09,391 - root - INFO - step: 41575 loss: 16.8297 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.6021 global_avg_mtp_loss: 14.2276 +[titan] 2025-07-09 22:45:09,391 - root - INFO - lr: 2.0272e-04 gnorm: 0.74 [ 9:10:10<12:53:08] +[titan] 2025-07-09 22:45:13,327 - root - INFO - step: 41580 loss: 16.4854 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9531 +[titan] 2025-07-09 22:45:13,327 - root - INFO - lr: 2.0270e-04 gnorm: 0.80 [ 9:10:14<12:53:04] +[titan] 2025-07-09 22:45:17,248 - root - INFO - step: 41585 loss: 16.5207 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 13.9784 +[titan] 2025-07-09 22:45:17,248 - root - INFO - lr: 2.0268e-04 gnorm: 0.82 [ 9:10:18<12:53:00] +[titan] 2025-07-09 22:45:21,214 - root - INFO - step: 41590 loss: 16.5155 memory: 44.58GiB(31.99%) tps: 82,635 tflops: 285.19 mfu: 28.84% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-09 22:45:21,214 - root - INFO - lr: 2.0266e-04 gnorm: 0.80 [ 9:10:22<12:52:56] +[titan] 2025-07-09 22:45:25,145 - root - INFO - step: 41595 loss: 16.5522 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 14.0016 +[titan] 2025-07-09 22:45:25,145 - root - INFO - lr: 2.0264e-04 gnorm: 0.74 [ 9:10:25<12:52:52] +[titan] 2025-07-09 22:45:28,290 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:45:29,079 - root - INFO - step: 41600 loss: 16.3778 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8697 +[titan] 2025-07-09 22:45:29,079 - root - INFO - lr: 2.0262e-04 gnorm: 0.79 [ 9:10:29<12:52:48] +[titan] 2025-07-09 22:45:32,993 - root - INFO - step: 41605 loss: 16.6325 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0651 +[titan] 2025-07-09 22:45:32,993 - root - INFO - lr: 2.0260e-04 gnorm: 0.79 [ 9:10:33<12:52:44] +[titan] 2025-07-09 22:45:36,914 - root - INFO - step: 41610 loss: 16.6261 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0675 +[titan] 2025-07-09 22:45:36,914 - root - INFO - lr: 2.0258e-04 gnorm: 0.80 [ 9:10:37<12:52:40] +[titan] 2025-07-09 22:45:40,872 - root - INFO - step: 41615 loss: 16.6281 memory: 44.58GiB(31.99%) tps: 82,797 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.0518 +[titan] 2025-07-09 22:45:40,872 - root - INFO - lr: 2.0256e-04 gnorm: 0.74 [ 9:10:41<12:52:36] +[titan] 2025-07-09 22:45:44,793 - root - INFO - step: 41620 loss: 16.7025 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5895 global_avg_mtp_loss: 14.1131 +[titan] 2025-07-09 22:45:44,793 - root - INFO - lr: 2.0254e-04 gnorm: 0.83 [ 9:10:45<12:52:32] +[titan] 2025-07-09 22:45:48,757 - root - INFO - step: 41625 loss: 16.7606 memory: 44.58GiB(31.99%) tps: 82,670 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1739 +[titan] 2025-07-09 22:45:48,757 - root - INFO - lr: 2.0252e-04 gnorm: 0.80 [ 9:10:49<12:52:28] +[titan] 2025-07-09 22:45:52,685 - root - INFO - step: 41630 loss: 16.5453 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 13.9915 +[titan] 2025-07-09 22:45:52,686 - root - INFO - lr: 2.0250e-04 gnorm: 0.88 [ 9:10:53<12:52:24] +[titan] 2025-07-09 22:45:56,662 - root - INFO - step: 41635 loss: 16.5993 memory: 44.58GiB(31.99%) tps: 82,400 tflops: 284.38 mfu: 28.75% global_avg_ntp_loss: 2.5636 global_avg_mtp_loss: 14.0357 +[titan] 2025-07-09 22:45:56,663 - root - INFO - lr: 2.0248e-04 gnorm: 0.78 [ 9:10:57<12:52:20] +[titan] 2025-07-09 22:46:00,579 - root - INFO - step: 41640 loss: 16.4474 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.9197 +[titan] 2025-07-09 22:46:00,579 - root - INFO - lr: 2.0246e-04 gnorm: 0.79 [ 9:11:01<12:52:16] +[titan] 2025-07-09 22:46:04,507 - root - INFO - step: 41645 loss: 16.3518 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8309 +[titan] 2025-07-09 22:46:04,507 - root - INFO - lr: 2.0244e-04 gnorm: 0.79 [ 9:11:05<12:52:12] +[titan] 2025-07-09 22:46:07,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:46:08,425 - root - INFO - step: 41650 loss: 16.5218 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9871 +[titan] 2025-07-09 22:46:08,425 - root - INFO - lr: 2.0241e-04 gnorm: 0.78 [ 9:11:09<12:52:08] +[titan] 2025-07-09 22:46:12,344 - root - INFO - step: 41655 loss: 16.7664 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6027 global_avg_mtp_loss: 14.1636 +[titan] 2025-07-09 22:46:12,344 - root - INFO - lr: 2.0239e-04 gnorm: 0.77 [ 9:11:13<12:52:04] +[titan] 2025-07-09 22:46:16,260 - root - INFO - step: 41660 loss: 16.6865 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1045 +[titan] 2025-07-09 22:46:16,261 - root - INFO - lr: 2.0237e-04 gnorm: 0.80 [ 9:11:17<12:52:00] +[titan] 2025-07-09 22:46:20,161 - root - INFO - step: 41665 loss: 16.7180 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5862 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 22:46:20,161 - root - INFO - lr: 2.0235e-04 gnorm: 0.81 [ 9:11:20<12:51:56] +[titan] 2025-07-09 22:46:24,080 - root - INFO - step: 41670 loss: 16.8229 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.2147 +[titan] 2025-07-09 22:46:24,080 - root - INFO - lr: 2.0233e-04 gnorm: 0.79 [ 9:11:24<12:51:52] +[titan] 2025-07-09 22:46:28,022 - root - INFO - step: 41675 loss: 16.2482 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.7430 +[titan] 2025-07-09 22:46:28,023 - root - INFO - lr: 2.0231e-04 gnorm: 0.80 [ 9:11:28<12:51:48] +[titan] 2025-07-09 22:46:31,935 - root - INFO - step: 41680 loss: 16.4817 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-09 22:46:31,935 - root - INFO - lr: 2.0229e-04 gnorm: 0.79 [ 9:11:32<12:51:44] +[titan] 2025-07-09 22:46:35,838 - root - INFO - step: 41685 loss: 16.3229 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8234 +[titan] 2025-07-09 22:46:35,839 - root - INFO - lr: 2.0227e-04 gnorm: 0.80 [ 9:11:36<12:51:40] +[titan] 2025-07-09 22:46:39,751 - root - INFO - step: 41690 loss: 16.5327 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5462 global_avg_mtp_loss: 13.9866 +[titan] 2025-07-09 22:46:39,752 - root - INFO - lr: 2.0225e-04 gnorm: 0.77 [ 9:11:40<12:51:36] +[titan] 2025-07-09 22:46:43,684 - root - INFO - step: 41695 loss: 16.6621 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.0917 +[titan] 2025-07-09 22:46:43,684 - root - INFO - lr: 2.0223e-04 gnorm: 0.76 [ 9:11:44<12:51:32] +[titan] 2025-07-09 22:46:46,821 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:46:47,610 - root - INFO - step: 41700 loss: 16.6084 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5595 global_avg_mtp_loss: 14.0489 +[titan] 2025-07-09 22:46:47,610 - root - INFO - lr: 2.0221e-04 gnorm: 0.71 [ 9:11:48<12:51:28] +[titan] 2025-07-09 22:46:51,510 - root - INFO - step: 41705 loss: 16.4435 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9187 +[titan] 2025-07-09 22:46:51,510 - root - INFO - lr: 2.0219e-04 gnorm: 0.77 [ 9:11:52<12:51:24] +[titan] 2025-07-09 22:46:55,409 - root - INFO - step: 41710 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.8037 +[titan] 2025-07-09 22:46:55,409 - root - INFO - lr: 2.0217e-04 gnorm: 0.77 [ 9:11:56<12:51:20] +[titan] 2025-07-09 22:46:59,329 - root - INFO - step: 41715 loss: 16.6279 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0635 +[titan] 2025-07-09 22:46:59,329 - root - INFO - lr: 2.0215e-04 gnorm: 0.75 [ 9:12:00<12:51:16] +[titan] 2025-07-09 22:47:03,239 - root - INFO - step: 41720 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-09 22:47:03,239 - root - INFO - lr: 2.0213e-04 gnorm: 0.77 [ 9:12:04<12:51:12] +[titan] 2025-07-09 22:47:07,149 - root - INFO - step: 41725 loss: 16.6098 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0396 +[titan] 2025-07-09 22:47:07,149 - root - INFO - lr: 2.0211e-04 gnorm: 0.75 [ 9:12:07<12:51:07] +[titan] 2025-07-09 22:47:11,058 - root - INFO - step: 41730 loss: 16.6576 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-09 22:47:11,059 - root - INFO - lr: 2.0209e-04 gnorm: 0.74 [ 9:12:11<12:51:03] +[titan] 2025-07-09 22:47:14,960 - root - INFO - step: 41735 loss: 16.6844 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5813 global_avg_mtp_loss: 14.1031 +[titan] 2025-07-09 22:47:14,961 - root - INFO - lr: 2.0206e-04 gnorm: 0.90 [ 9:12:15<12:50:59] +[titan] 2025-07-09 22:47:18,868 - root - INFO - step: 41740 loss: 16.5959 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0388 +[titan] 2025-07-09 22:47:18,869 - root - INFO - lr: 2.0204e-04 gnorm: 0.74 [ 9:12:19<12:50:55] +[titan] 2025-07-09 22:47:22,779 - root - INFO - step: 41745 loss: 16.4737 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9467 +[titan] 2025-07-09 22:47:22,779 - root - INFO - lr: 2.0202e-04 gnorm: 0.81 [ 9:12:23<12:50:51] +[titan] 2025-07-09 22:47:25,935 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:47:26,749 - root - INFO - step: 41750 loss: 16.4707 memory: 44.58GiB(31.99%) tps: 82,558 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 13.9312 +[titan] 2025-07-09 22:47:26,749 - root - INFO - lr: 2.0200e-04 gnorm: 0.81 [ 9:12:27<12:50:47] +[titan] 2025-07-09 22:47:30,678 - root - INFO - step: 41755 loss: 16.6539 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0911 +[titan] 2025-07-09 22:47:30,678 - root - INFO - lr: 2.0198e-04 gnorm: 0.77 [ 9:12:31<12:50:43] +[titan] 2025-07-09 22:47:34,593 - root - INFO - step: 41760 loss: 16.8680 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2615 +[titan] 2025-07-09 22:47:34,593 - root - INFO - lr: 2.0196e-04 gnorm: 0.83 [ 9:12:35<12:50:39] +[titan] 2025-07-09 22:47:38,494 - root - INFO - step: 41765 loss: 16.5202 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9738 +[titan] 2025-07-09 22:47:38,494 - root - INFO - lr: 2.0194e-04 gnorm: 0.75 [ 9:12:39<12:50:35] +[titan] 2025-07-09 22:47:42,411 - root - INFO - step: 41770 loss: 16.7990 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6238 global_avg_mtp_loss: 14.1752 +[titan] 2025-07-09 22:47:42,411 - root - INFO - lr: 2.0192e-04 gnorm: 0.80 [ 9:12:43<12:50:31] +[titan] 2025-07-09 22:47:46,324 - root - INFO - step: 41775 loss: 16.6560 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.0750 +[titan] 2025-07-09 22:47:46,324 - root - INFO - lr: 2.0190e-04 gnorm: 0.79 [ 9:12:47<12:50:27] +[titan] 2025-07-09 22:47:50,238 - root - INFO - step: 41780 loss: 16.5484 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-09 22:47:50,239 - root - INFO - lr: 2.0188e-04 gnorm: 0.77 [ 9:12:51<12:50:23] +[titan] 2025-07-09 22:47:54,134 - root - INFO - step: 41785 loss: 16.4058 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.8788 +[titan] 2025-07-09 22:47:54,134 - root - INFO - lr: 2.0186e-04 gnorm: 0.75 [ 9:12:54<12:50:19] +[titan] 2025-07-09 22:47:58,114 - root - INFO - step: 41790 loss: 16.5021 memory: 44.58GiB(31.99%) tps: 82,349 tflops: 284.20 mfu: 28.74% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9515 +[titan] 2025-07-09 22:47:58,114 - root - INFO - lr: 2.0184e-04 gnorm: 0.78 [ 9:12:58<12:50:15] +[titan] 2025-07-09 22:48:02,041 - root - INFO - step: 41795 loss: 16.4331 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9097 +[titan] 2025-07-09 22:48:02,042 - root - INFO - lr: 2.0182e-04 gnorm: 0.79 [ 9:13:02<12:50:11] +[titan] 2025-07-09 22:48:05,164 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:48:05,954 - root - INFO - step: 41800 loss: 16.5356 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9907 +[titan] 2025-07-09 22:48:05,954 - root - INFO - lr: 2.0180e-04 gnorm: 0.72 [ 9:13:06<12:50:07] +[titan] 2025-07-09 22:48:09,859 - root - INFO - step: 41805 loss: 16.5819 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5607 global_avg_mtp_loss: 14.0213 +[titan] 2025-07-09 22:48:09,859 - root - INFO - lr: 2.0178e-04 gnorm: 0.75 [ 9:13:10<12:50:03] +[titan] 2025-07-09 22:48:13,779 - root - INFO - step: 41810 loss: 16.4409 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9126 +[titan] 2025-07-09 22:48:13,780 - root - INFO - lr: 2.0176e-04 gnorm: 0.78 [ 9:13:14<12:49:59] +[titan] 2025-07-09 22:48:17,682 - root - INFO - step: 41815 loss: 16.6280 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5618 global_avg_mtp_loss: 14.0662 +[titan] 2025-07-09 22:48:17,683 - root - INFO - lr: 2.0173e-04 gnorm: 0.80 [ 9:13:18<12:49:55] +[titan] 2025-07-09 22:48:21,582 - root - INFO - step: 41820 loss: 16.6654 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.0933 +[titan] 2025-07-09 22:48:21,583 - root - INFO - lr: 2.0171e-04 gnorm: 0.76 [ 9:13:22<12:49:51] +[titan] 2025-07-09 22:48:25,549 - root - INFO - step: 41825 loss: 16.5545 memory: 44.58GiB(31.99%) tps: 82,607 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 14.0071 +[titan] 2025-07-09 22:48:25,550 - root - INFO - lr: 2.0169e-04 gnorm: 0.79 [ 9:13:26<12:49:47] +[titan] 2025-07-09 22:48:29,459 - root - INFO - step: 41830 loss: 16.1896 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7002 +[titan] 2025-07-09 22:48:29,459 - root - INFO - lr: 2.0167e-04 gnorm: 0.79 [ 9:13:30<12:49:43] +[titan] 2025-07-09 22:48:33,371 - root - INFO - step: 41835 loss: 16.5444 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 14.0001 +[titan] 2025-07-09 22:48:33,372 - root - INFO - lr: 2.0165e-04 gnorm: 0.85 [ 9:13:34<12:49:39] +[titan] 2025-07-09 22:48:37,293 - root - INFO - step: 41840 loss: 16.7977 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.2044 +[titan] 2025-07-09 22:48:37,294 - root - INFO - lr: 2.0163e-04 gnorm: 0.79 [ 9:13:38<12:49:35] +[titan] 2025-07-09 22:48:41,283 - root - INFO - step: 41845 loss: 16.7196 memory: 44.58GiB(31.99%) tps: 82,133 tflops: 283.45 mfu: 28.66% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.1444 +[titan] 2025-07-09 22:48:41,284 - root - INFO - lr: 2.0161e-04 gnorm: 0.72 [ 9:13:42<12:49:31] +[titan] 2025-07-09 22:48:44,418 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:48:45,233 - root - INFO - step: 41850 loss: 16.5888 memory: 44.58GiB(31.99%) tps: 82,963 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 14.0423 +[titan] 2025-07-09 22:48:45,234 - root - INFO - lr: 2.0159e-04 gnorm: 0.75 [ 9:13:46<12:49:27] +[titan] 2025-07-09 22:48:49,151 - root - INFO - step: 41855 loss: 16.5548 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 13.9998 +[titan] 2025-07-09 22:48:49,151 - root - INFO - lr: 2.0157e-04 gnorm: 0.76 [ 9:13:49<12:49:23] +[titan] 2025-07-09 22:48:53,066 - root - INFO - step: 41860 loss: 16.9604 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.6247 global_avg_mtp_loss: 14.3357 +[titan] 2025-07-09 22:48:53,066 - root - INFO - lr: 2.0155e-04 gnorm: 0.78 [ 9:13:53<12:49:19] +[titan] 2025-07-09 22:48:56,980 - root - INFO - step: 41865 loss: 16.7856 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5911 global_avg_mtp_loss: 14.1945 +[titan] 2025-07-09 22:48:56,980 - root - INFO - lr: 2.0153e-04 gnorm: 0.78 [ 9:13:57<12:49:14] +[titan] 2025-07-09 22:49:00,904 - root - INFO - step: 41870 loss: 16.9000 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.6244 global_avg_mtp_loss: 14.2756 +[titan] 2025-07-09 22:49:00,905 - root - INFO - lr: 2.0151e-04 gnorm: 0.82 [ 9:14:01<12:49:10] +[titan] 2025-07-09 22:49:04,811 - root - INFO - step: 41875 loss: 16.4019 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8799 +[titan] 2025-07-09 22:49:04,811 - root - INFO - lr: 2.0149e-04 gnorm: 0.77 [ 9:14:05<12:49:06] +[titan] 2025-07-09 22:49:08,729 - root - INFO - step: 41880 loss: 16.4080 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.8850 +[titan] 2025-07-09 22:49:08,729 - root - INFO - lr: 2.0147e-04 gnorm: 0.77 [ 9:14:09<12:49:02] +[titan] 2025-07-09 22:49:12,676 - root - INFO - step: 41885 loss: 16.7538 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6031 global_avg_mtp_loss: 14.1507 +[titan] 2025-07-09 22:49:12,676 - root - INFO - lr: 2.0145e-04 gnorm: 0.77 [ 9:14:13<12:48:58] +[titan] 2025-07-09 22:49:16,599 - root - INFO - step: 41890 loss: 16.5712 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 14.0156 +[titan] 2025-07-09 22:49:16,599 - root - INFO - lr: 2.0143e-04 gnorm: 0.82 [ 9:14:17<12:48:54] +[titan] 2025-07-09 22:49:20,523 - root - INFO - step: 41895 loss: 16.6870 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.1189 +[titan] 2025-07-09 22:49:20,524 - root - INFO - lr: 2.0140e-04 gnorm: 0.79 [ 9:14:21<12:48:50] +[titan] 2025-07-09 22:49:23,642 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:49:24,431 - root - INFO - step: 41900 loss: 16.6677 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.0831 +[titan] 2025-07-09 22:49:24,432 - root - INFO - lr: 2.0138e-04 gnorm: 0.74 [ 9:14:25<12:48:46] +[titan] 2025-07-09 22:49:28,364 - root - INFO - step: 41905 loss: 16.7650 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1777 +[titan] 2025-07-09 22:49:28,365 - root - INFO - lr: 2.0136e-04 gnorm: 0.73 [ 9:14:29<12:48:42] +[titan] 2025-07-09 22:49:32,259 - root - INFO - step: 41910 loss: 16.9244 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6188 global_avg_mtp_loss: 14.3056 +[titan] 2025-07-09 22:49:32,260 - root - INFO - lr: 2.0134e-04 gnorm: 0.82 [ 9:14:33<12:48:38] +[titan] 2025-07-09 22:49:36,191 - root - INFO - step: 41915 loss: 16.5694 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0105 +[titan] 2025-07-09 22:49:36,191 - root - INFO - lr: 2.0132e-04 gnorm: 0.80 [ 9:14:36<12:48:34] +[titan] 2025-07-09 22:49:40,103 - root - INFO - step: 41920 loss: 16.5159 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9705 +[titan] 2025-07-09 22:49:40,103 - root - INFO - lr: 2.0130e-04 gnorm: 0.82 [ 9:14:40<12:48:30] +[titan] 2025-07-09 22:49:44,059 - root - INFO - step: 41925 loss: 16.8704 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.6135 global_avg_mtp_loss: 14.2569 +[titan] 2025-07-09 22:49:44,059 - root - INFO - lr: 2.0128e-04 gnorm: 0.78 [ 9:14:44<12:48:26] +[titan] 2025-07-09 22:49:48,012 - root - INFO - step: 41930 loss: 16.7132 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1306 +[titan] 2025-07-09 22:49:48,012 - root - INFO - lr: 2.0126e-04 gnorm: 0.79 [ 9:14:48<12:48:22] +[titan] 2025-07-09 22:49:51,934 - root - INFO - step: 41935 loss: 16.5609 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 14.0025 +[titan] 2025-07-09 22:49:51,935 - root - INFO - lr: 2.0124e-04 gnorm: 0.77 [ 9:14:52<12:48:18] +[titan] 2025-07-09 22:49:55,840 - root - INFO - step: 41940 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-09 22:49:55,840 - root - INFO - lr: 2.0122e-04 gnorm: 0.75 [ 9:14:56<12:48:14] +[titan] 2025-07-09 22:49:59,755 - root - INFO - step: 41945 loss: 16.7487 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.1675 +[titan] 2025-07-09 22:49:59,755 - root - INFO - lr: 2.0120e-04 gnorm: 0.74 [ 9:15:00<12:48:10] +[titan] 2025-07-09 22:50:02,858 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:50:03,655 - root - INFO - step: 41950 loss: 16.6054 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0411 +[titan] 2025-07-09 22:50:03,656 - root - INFO - lr: 2.0118e-04 gnorm: 0.75 [ 9:15:04<12:48:06] +[titan] 2025-07-09 22:50:07,595 - root - INFO - step: 41955 loss: 16.5591 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 22:50:07,595 - root - INFO - lr: 2.0116e-04 gnorm: 0.82 [ 9:15:08<12:48:02] +[titan] 2025-07-09 22:50:11,550 - root - INFO - step: 41960 loss: 16.5288 memory: 44.58GiB(31.99%) tps: 82,845 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-09 22:50:11,551 - root - INFO - lr: 2.0114e-04 gnorm: 0.77 [ 9:15:12<12:47:58] +[titan] 2025-07-09 22:50:15,492 - root - INFO - step: 41965 loss: 16.7245 memory: 44.58GiB(31.99%) tps: 83,147 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.1452 +[titan] 2025-07-09 22:50:15,492 - root - INFO - lr: 2.0112e-04 gnorm: 0.76 [ 9:15:16<12:47:54] +[titan] 2025-07-09 22:50:19,406 - root - INFO - step: 41970 loss: 16.4852 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9374 +[titan] 2025-07-09 22:50:19,406 - root - INFO - lr: 2.0110e-04 gnorm: 0.77 [ 9:15:20<12:47:50] +[titan] 2025-07-09 22:50:23,324 - root - INFO - step: 41975 loss: 16.5478 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 14.0121 +[titan] 2025-07-09 22:50:23,325 - root - INFO - lr: 2.0107e-04 gnorm: 0.75 [ 9:15:24<12:47:46] +[titan] 2025-07-09 22:50:27,231 - root - INFO - step: 41980 loss: 16.7893 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1977 +[titan] 2025-07-09 22:50:27,232 - root - INFO - lr: 2.0105e-04 gnorm: 0.78 [ 9:15:28<12:47:42] +[titan] 2025-07-09 22:50:30,605 - root - INFO - Dumping profiler traces at step 41984 +[titan] 2025-07-09 22:50:30,638 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:50:31,435 - root - INFO - step: 41985 loss: 16.5633 memory: 44.58GiB(31.99%) tps: 77,964 tflops: 269.07 mfu: 27.21% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 14.0225 +[titan] 2025-07-09 22:50:31,435 - root - INFO - lr: 2.0103e-04 gnorm: 0.79 [ 9:15:32<12:47:38] +[titan] 2025-07-09 22:50:35,351 - root - INFO - step: 41990 loss: 16.3860 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-09 22:50:35,351 - root - INFO - lr: 2.0101e-04 gnorm: 0.81 [ 9:15:36<12:47:34] +[titan] 2025-07-09 22:50:39,270 - root - INFO - step: 41995 loss: 16.5511 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9937 +[titan] 2025-07-09 22:50:39,270 - root - INFO - lr: 2.0099e-04 gnorm: 0.79 [ 9:15:40<12:47:30] +[titan] 2025-07-09 22:50:42,413 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:50:43,213 - root - INFO - step: 42000 loss: 16.7402 memory: 44.58GiB(31.99%) tps: 83,113 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1455 +[titan] 2025-07-09 22:50:43,213 - root - INFO - lr: 2.0097e-04 gnorm: 0.81 [ 9:15:44<12:47:26] +[titan] 2025-07-09 22:50:47,113 - root - INFO - step: 42005 loss: 16.6327 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0654 +[titan] 2025-07-09 22:50:47,114 - root - INFO - lr: 2.0095e-04 gnorm: 0.76 [ 9:15:47<12:47:22] +[titan] 2025-07-09 22:50:51,061 - root - INFO - step: 42010 loss: 16.8131 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.6009 global_avg_mtp_loss: 14.2122 +[titan] 2025-07-09 22:50:51,061 - root - INFO - lr: 2.0093e-04 gnorm: 0.82 [ 9:15:51<12:47:18] +[titan] 2025-07-09 22:50:54,979 - root - INFO - step: 42015 loss: 16.5935 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0246 +[titan] 2025-07-09 22:50:54,979 - root - INFO - lr: 2.0091e-04 gnorm: 0.75 [ 9:15:55<12:47:14] +[titan] 2025-07-09 22:50:58,893 - root - INFO - step: 42020 loss: 16.3206 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8135 +[titan] 2025-07-09 22:50:58,893 - root - INFO - lr: 2.0089e-04 gnorm: 0.83 [ 9:15:59<12:47:10] +[titan] 2025-07-09 22:51:02,786 - root - INFO - step: 42025 loss: 16.7115 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.1394 +[titan] 2025-07-09 22:51:02,786 - root - INFO - lr: 2.0087e-04 gnorm: 0.79 [ 9:16:03<12:47:06] +[titan] 2025-07-09 22:51:06,709 - root - INFO - step: 42030 loss: 16.4136 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.8792 +[titan] 2025-07-09 22:51:06,709 - root - INFO - lr: 2.0085e-04 gnorm: 0.76 [ 9:16:07<12:47:02] +[titan] 2025-07-09 22:51:10,629 - root - INFO - step: 42035 loss: 16.6484 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0741 +[titan] 2025-07-09 22:51:10,629 - root - INFO - lr: 2.0083e-04 gnorm: 0.76 [ 9:16:11<12:46:58] +[titan] 2025-07-09 22:51:14,567 - root - INFO - step: 42040 loss: 16.9666 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.6419 global_avg_mtp_loss: 14.3247 +[titan] 2025-07-09 22:51:14,567 - root - INFO - lr: 2.0081e-04 gnorm: 0.81 [ 9:16:15<12:46:54] +[titan] 2025-07-09 22:51:18,481 - root - INFO - step: 42045 loss: 16.6307 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5598 global_avg_mtp_loss: 14.0709 +[titan] 2025-07-09 22:51:18,482 - root - INFO - lr: 2.0079e-04 gnorm: 0.78 [ 9:16:19<12:46:50] +[titan] 2025-07-09 22:51:21,593 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:51:22,380 - root - INFO - step: 42050 loss: 16.8680 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.6195 global_avg_mtp_loss: 14.2485 +[titan] 2025-07-09 22:51:22,380 - root - INFO - lr: 2.0077e-04 gnorm: 0.86 [ 9:16:23<12:46:46] +[titan] 2025-07-09 22:51:26,289 - root - INFO - step: 42055 loss: 16.6674 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.1000 +[titan] 2025-07-09 22:51:26,289 - root - INFO - lr: 2.0074e-04 gnorm: 0.79 [ 9:16:27<12:46:41] +[titan] 2025-07-09 22:51:30,215 - root - INFO - step: 42060 loss: 16.5731 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0138 +[titan] 2025-07-09 22:51:30,216 - root - INFO - lr: 2.0072e-04 gnorm: 0.84 [ 9:16:31<12:46:37] +[titan] 2025-07-09 22:51:34,123 - root - INFO - step: 42065 loss: 16.6646 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5745 global_avg_mtp_loss: 14.0901 +[titan] 2025-07-09 22:51:34,123 - root - INFO - lr: 2.0070e-04 gnorm: 0.81 [ 9:16:34<12:46:33] +[titan] 2025-07-09 22:51:38,024 - root - INFO - step: 42070 loss: 16.6316 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0631 +[titan] 2025-07-09 22:51:38,024 - root - INFO - lr: 2.0068e-04 gnorm: 0.77 [ 9:16:38<12:46:29] +[titan] 2025-07-09 22:51:41,925 - root - INFO - step: 42075 loss: 16.6827 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.1141 +[titan] 2025-07-09 22:51:41,925 - root - INFO - lr: 2.0066e-04 gnorm: 0.77 [ 9:16:42<12:46:25] +[titan] 2025-07-09 22:51:45,860 - root - INFO - step: 42080 loss: 16.7276 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1369 +[titan] 2025-07-09 22:51:45,860 - root - INFO - lr: 2.0064e-04 gnorm: 0.74 [ 9:16:46<12:46:21] +[titan] 2025-07-09 22:51:49,766 - root - INFO - step: 42085 loss: 16.4086 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8916 +[titan] 2025-07-09 22:51:49,767 - root - INFO - lr: 2.0062e-04 gnorm: 0.77 [ 9:16:50<12:46:17] +[titan] 2025-07-09 22:51:53,685 - root - INFO - step: 42090 loss: 16.5405 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 14.0053 +[titan] 2025-07-09 22:51:53,685 - root - INFO - lr: 2.0060e-04 gnorm: 0.81 [ 9:16:54<12:46:13] +[titan] 2025-07-09 22:51:57,616 - root - INFO - step: 42095 loss: 16.5884 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0207 +[titan] 2025-07-09 22:51:57,617 - root - INFO - lr: 2.0058e-04 gnorm: 0.77 [ 9:16:58<12:46:09] +[titan] 2025-07-09 22:52:00,740 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:52:01,533 - root - INFO - step: 42100 loss: 16.4214 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.8926 +[titan] 2025-07-09 22:52:01,533 - root - INFO - lr: 2.0056e-04 gnorm: 0.77 [ 9:17:02<12:46:05] +[titan] 2025-07-09 22:52:05,473 - root - INFO - step: 42105 loss: 16.6033 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0404 +[titan] 2025-07-09 22:52:05,473 - root - INFO - lr: 2.0054e-04 gnorm: 0.78 [ 9:17:06<12:46:01] +[titan] 2025-07-09 22:52:09,385 - root - INFO - step: 42110 loss: 16.6397 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.0565 +[titan] 2025-07-09 22:52:09,386 - root - INFO - lr: 2.0052e-04 gnorm: 0.78 [ 9:17:10<12:45:57] +[titan] 2025-07-09 22:52:13,319 - root - INFO - step: 42115 loss: 16.4376 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 13.8975 +[titan] 2025-07-09 22:52:13,319 - root - INFO - lr: 2.0050e-04 gnorm: 0.87 [ 9:17:14<12:45:53] +[titan] 2025-07-09 22:52:17,266 - root - INFO - step: 42120 loss: 16.6370 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.0713 +[titan] 2025-07-09 22:52:17,267 - root - INFO - lr: 2.0048e-04 gnorm: 0.82 [ 9:17:18<12:45:49] +[titan] 2025-07-09 22:52:21,217 - root - INFO - step: 42125 loss: 16.4309 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.8995 +[titan] 2025-07-09 22:52:21,218 - root - INFO - lr: 2.0046e-04 gnorm: 0.83 [ 9:17:21<12:45:45] +[titan] 2025-07-09 22:52:25,134 - root - INFO - step: 42130 loss: 16.5000 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9509 +[titan] 2025-07-09 22:52:25,134 - root - INFO - lr: 2.0043e-04 gnorm: 0.79 [ 9:17:25<12:45:41] +[titan] 2025-07-09 22:52:29,041 - root - INFO - step: 42135 loss: 16.7903 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1981 +[titan] 2025-07-09 22:52:29,042 - root - INFO - lr: 2.0041e-04 gnorm: 0.78 [ 9:17:29<12:45:37] +[titan] 2025-07-09 22:52:32,959 - root - INFO - step: 42140 loss: 16.4832 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9390 +[titan] 2025-07-09 22:52:32,959 - root - INFO - lr: 2.0039e-04 gnorm: 0.84 [ 9:17:33<12:45:33] +[titan] 2025-07-09 22:52:36,862 - root - INFO - step: 42145 loss: 16.7212 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.1492 +[titan] 2025-07-09 22:52:36,863 - root - INFO - lr: 2.0037e-04 gnorm: 0.71 [ 9:17:37<12:45:29] +[titan] 2025-07-09 22:52:39,983 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:52:40,770 - root - INFO - step: 42150 loss: 16.6000 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5657 global_avg_mtp_loss: 14.0343 +[titan] 2025-07-09 22:52:40,770 - root - INFO - lr: 2.0035e-04 gnorm: 0.75 [ 9:17:41<12:45:25] +[titan] 2025-07-09 22:52:44,724 - root - INFO - step: 42155 loss: 16.3895 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.8600 +[titan] 2025-07-09 22:52:44,725 - root - INFO - lr: 2.0033e-04 gnorm: 0.72 [ 9:17:45<12:45:21] +[titan] 2025-07-09 22:52:48,651 - root - INFO - step: 42160 loss: 16.7300 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5819 global_avg_mtp_loss: 14.1481 +[titan] 2025-07-09 22:52:48,652 - root - INFO - lr: 2.0031e-04 gnorm: 0.78 [ 9:17:49<12:45:17] +[titan] 2025-07-09 22:52:52,582 - root - INFO - step: 42165 loss: 16.6872 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-09 22:52:52,582 - root - INFO - lr: 2.0029e-04 gnorm: 0.76 [ 9:17:53<12:45:13] +[titan] 2025-07-09 22:52:56,515 - root - INFO - step: 42170 loss: 16.7310 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.1571 +[titan] 2025-07-09 22:52:56,515 - root - INFO - lr: 2.0027e-04 gnorm: 0.78 [ 9:17:57<12:45:09] +[titan] 2025-07-09 22:53:00,415 - root - INFO - step: 42175 loss: 16.6415 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0816 +[titan] 2025-07-09 22:53:00,416 - root - INFO - lr: 2.0025e-04 gnorm: 0.79 [ 9:18:01<12:45:05] +[titan] 2025-07-09 22:53:04,324 - root - INFO - step: 42180 loss: 16.7202 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.1425 +[titan] 2025-07-09 22:53:04,324 - root - INFO - lr: 2.0023e-04 gnorm: 0.79 [ 9:18:05<12:45:01] +[titan] 2025-07-09 22:53:08,238 - root - INFO - step: 42185 loss: 16.5545 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9984 +[titan] 2025-07-09 22:53:08,238 - root - INFO - lr: 2.0021e-04 gnorm: 0.74 [ 9:18:09<12:44:57] +[titan] 2025-07-09 22:53:12,131 - root - INFO - step: 42190 loss: 16.3550 memory: 44.58GiB(31.99%) tps: 84,175 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8460 +[titan] 2025-07-09 22:53:12,131 - root - INFO - lr: 2.0019e-04 gnorm: 0.79 [ 9:18:12<12:44:52] +[titan] 2025-07-09 22:53:16,046 - root - INFO - step: 42195 loss: 16.3148 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.7837 +[titan] 2025-07-09 22:53:16,047 - root - INFO - lr: 2.0017e-04 gnorm: 0.79 [ 9:18:16<12:44:48] +[titan] 2025-07-09 22:53:19,174 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:53:19,958 - root - INFO - step: 42200 loss: 16.8712 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.6440 global_avg_mtp_loss: 14.2272 +[titan] 2025-07-09 22:53:19,959 - root - INFO - lr: 2.0015e-04 gnorm: 0.74 [ 9:18:20<12:44:44] +[titan] 2025-07-09 22:53:23,849 - root - INFO - step: 42205 loss: 16.3732 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-09 22:53:23,849 - root - INFO - lr: 2.0012e-04 gnorm: 0.76 [ 9:18:24<12:44:40] +[titan] 2025-07-09 22:53:27,795 - root - INFO - step: 42210 loss: 16.5430 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9969 +[titan] 2025-07-09 22:53:27,795 - root - INFO - lr: 2.0010e-04 gnorm: 0.77 [ 9:18:28<12:44:36] +[titan] 2025-07-09 22:53:31,708 - root - INFO - step: 42215 loss: 16.5753 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0214 +[titan] 2025-07-09 22:53:31,709 - root - INFO - lr: 2.0008e-04 gnorm: 0.78 [ 9:18:32<12:44:32] +[titan] 2025-07-09 22:53:35,631 - root - INFO - step: 42220 loss: 16.5490 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 14.0024 +[titan] 2025-07-09 22:53:35,632 - root - INFO - lr: 2.0006e-04 gnorm: 0.83 [ 9:18:36<12:44:28] +[titan] 2025-07-09 22:53:39,552 - root - INFO - step: 42225 loss: 16.5619 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0127 +[titan] 2025-07-09 22:53:39,552 - root - INFO - lr: 2.0004e-04 gnorm: 0.74 [ 9:18:40<12:44:24] +[titan] 2025-07-09 22:53:43,499 - root - INFO - step: 42230 loss: 16.4093 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.8831 +[titan] 2025-07-09 22:53:43,499 - root - INFO - lr: 2.0002e-04 gnorm: 0.99 [ 9:18:44<12:44:20] +[titan] 2025-07-09 22:53:47,411 - root - INFO - step: 42235 loss: 16.5451 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5741 global_avg_mtp_loss: 13.9711 +[titan] 2025-07-09 22:53:47,411 - root - INFO - lr: 2.0000e-04 gnorm: 0.88 [ 9:18:48<12:44:16] +[titan] 2025-07-09 22:53:51,310 - root - INFO - step: 42240 loss: 16.5012 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 13.9543 +[titan] 2025-07-09 22:53:51,310 - root - INFO - lr: 1.9998e-04 gnorm: 0.80 [ 9:18:52<12:44:12] +[titan] 2025-07-09 22:53:55,221 - root - INFO - step: 42245 loss: 16.6495 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.0851 +[titan] 2025-07-09 22:53:55,221 - root - INFO - lr: 1.9996e-04 gnorm: 0.84 [ 9:18:55<12:44:08] +[titan] 2025-07-09 22:53:58,341 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:53:59,130 - root - INFO - step: 42250 loss: 16.7867 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.1708 +[titan] 2025-07-09 22:53:59,130 - root - INFO - lr: 1.9994e-04 gnorm: 0.80 [ 9:18:59<12:44:04] +[titan] 2025-07-09 22:54:03,035 - root - INFO - step: 42255 loss: 16.8956 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.6175 global_avg_mtp_loss: 14.2782 +[titan] 2025-07-09 22:54:03,036 - root - INFO - lr: 1.9992e-04 gnorm: 0.74 [ 9:19:03<12:44:00] +[titan] 2025-07-09 22:54:06,935 - root - INFO - step: 42260 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9486 +[titan] 2025-07-09 22:54:06,935 - root - INFO - lr: 1.9990e-04 gnorm: 0.74 [ 9:19:07<12:43:56] +[titan] 2025-07-09 22:54:10,858 - root - INFO - step: 42265 loss: 16.5142 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-09 22:54:10,858 - root - INFO - lr: 1.9988e-04 gnorm: 0.83 [ 9:19:11<12:43:52] +[titan] 2025-07-09 22:54:14,813 - root - INFO - step: 42270 loss: 16.4441 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.9191 +[titan] 2025-07-09 22:54:14,813 - root - INFO - lr: 1.9986e-04 gnorm: 0.75 [ 9:19:15<12:43:48] +[titan] 2025-07-09 22:54:18,707 - root - INFO - step: 42275 loss: 16.6478 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0819 +[titan] 2025-07-09 22:54:18,708 - root - INFO - lr: 1.9983e-04 gnorm: 0.81 [ 9:19:19<12:43:44] +[titan] 2025-07-09 22:54:22,620 - root - INFO - step: 42280 loss: 16.5125 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9729 +[titan] 2025-07-09 22:54:22,620 - root - INFO - lr: 1.9981e-04 gnorm: 0.77 [ 9:19:23<12:43:40] +[titan] 2025-07-09 22:54:26,549 - root - INFO - step: 42285 loss: 16.7652 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1806 +[titan] 2025-07-09 22:54:26,549 - root - INFO - lr: 1.9979e-04 gnorm: 0.78 [ 9:19:27<12:43:36] +[titan] 2025-07-09 22:54:30,466 - root - INFO - step: 42290 loss: 16.7422 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.1430 +[titan] 2025-07-09 22:54:30,467 - root - INFO - lr: 1.9977e-04 gnorm: 0.78 [ 9:19:31<12:43:32] +[titan] 2025-07-09 22:54:34,411 - root - INFO - step: 42295 loss: 16.6662 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0974 +[titan] 2025-07-09 22:54:34,411 - root - INFO - lr: 1.9975e-04 gnorm: 0.81 [ 9:19:35<12:43:28] +[titan] 2025-07-09 22:54:37,571 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:54:38,361 - root - INFO - step: 42300 loss: 16.5969 memory: 44.58GiB(31.99%) tps: 82,957 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5545 global_avg_mtp_loss: 14.0424 +[titan] 2025-07-09 22:54:38,361 - root - INFO - lr: 1.9973e-04 gnorm: 0.77 [ 9:19:39<12:43:24] +[titan] 2025-07-09 22:54:42,292 - root - INFO - step: 42305 loss: 16.6584 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-09 22:54:42,292 - root - INFO - lr: 1.9971e-04 gnorm: 0.76 [ 9:19:43<12:43:20] +[titan] 2025-07-09 22:54:46,224 - root - INFO - step: 42310 loss: 16.5590 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 14.0010 +[titan] 2025-07-09 22:54:46,224 - root - INFO - lr: 1.9969e-04 gnorm: 0.80 [ 9:19:46<12:43:16] +[titan] 2025-07-09 22:54:50,169 - root - INFO - step: 42315 loss: 16.7635 memory: 44.58GiB(31.99%) tps: 83,063 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.5976 global_avg_mtp_loss: 14.1659 +[titan] 2025-07-09 22:54:50,169 - root - INFO - lr: 1.9967e-04 gnorm: 0.79 [ 9:19:50<12:43:12] +[titan] 2025-07-09 22:54:54,087 - root - INFO - step: 42320 loss: 16.7903 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.1853 +[titan] 2025-07-09 22:54:54,088 - root - INFO - lr: 1.9965e-04 gnorm: 0.77 [ 9:19:54<12:43:08] +[titan] 2025-07-09 22:54:58,046 - root - INFO - step: 42325 loss: 16.7383 memory: 44.58GiB(31.99%) tps: 82,790 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5962 global_avg_mtp_loss: 14.1420 +[titan] 2025-07-09 22:54:58,046 - root - INFO - lr: 1.9963e-04 gnorm: 0.76 [ 9:19:58<12:43:04] +[titan] 2025-07-09 22:55:01,972 - root - INFO - step: 42330 loss: 16.6177 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5601 global_avg_mtp_loss: 14.0576 +[titan] 2025-07-09 22:55:01,972 - root - INFO - lr: 1.9961e-04 gnorm: 0.79 [ 9:20:02<12:43:00] +[titan] 2025-07-09 22:55:05,912 - root - INFO - step: 42335 loss: 16.5838 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0154 +[titan] 2025-07-09 22:55:05,912 - root - INFO - lr: 1.9959e-04 gnorm: 0.85 [ 9:20:06<12:42:56] +[titan] 2025-07-09 22:55:09,865 - root - INFO - step: 42340 loss: 16.7500 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.1658 +[titan] 2025-07-09 22:55:09,865 - root - INFO - lr: 1.9957e-04 gnorm: 0.78 [ 9:20:10<12:42:52] +[titan] 2025-07-09 22:55:13,779 - root - INFO - step: 42345 loss: 16.7105 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.1207 +[titan] 2025-07-09 22:55:13,780 - root - INFO - lr: 1.9954e-04 gnorm: 0.78 [ 9:20:14<12:42:48] +[titan] 2025-07-09 22:55:16,907 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:55:17,697 - root - INFO - step: 42350 loss: 16.7962 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6028 global_avg_mtp_loss: 14.1934 +[titan] 2025-07-09 22:55:17,697 - root - INFO - lr: 1.9952e-04 gnorm: 0.77 [ 9:20:18<12:42:43] +[titan] 2025-07-09 22:55:21,604 - root - INFO - step: 42355 loss: 16.1651 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.6875 +[titan] 2025-07-09 22:55:21,604 - root - INFO - lr: 1.9950e-04 gnorm: 0.76 [ 9:20:22<12:42:39] +[titan] 2025-07-09 22:55:25,565 - root - INFO - step: 42360 loss: 16.6922 memory: 44.58GiB(31.99%) tps: 82,747 tflops: 285.57 mfu: 28.87% global_avg_ntp_loss: 2.5862 global_avg_mtp_loss: 14.1060 +[titan] 2025-07-09 22:55:25,565 - root - INFO - lr: 1.9948e-04 gnorm: 0.81 [ 9:20:26<12:42:35] +[titan] 2025-07-09 22:55:29,536 - root - INFO - step: 42365 loss: 16.4534 memory: 44.58GiB(31.99%) tps: 82,524 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.9261 +[titan] 2025-07-09 22:55:29,536 - root - INFO - lr: 1.9946e-04 gnorm: 0.84 [ 9:20:30<12:42:31] +[titan] 2025-07-09 22:55:33,462 - root - INFO - step: 42370 loss: 16.6620 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5782 global_avg_mtp_loss: 14.0838 +[titan] 2025-07-09 22:55:33,462 - root - INFO - lr: 1.9944e-04 gnorm: 0.76 [ 9:20:34<12:42:27] +[titan] 2025-07-09 22:55:37,385 - root - INFO - step: 42375 loss: 16.5473 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 14.0068 +[titan] 2025-07-09 22:55:37,386 - root - INFO - lr: 1.9942e-04 gnorm: 0.79 [ 9:20:38<12:42:23] +[titan] 2025-07-09 22:55:41,303 - root - INFO - step: 42380 loss: 16.5801 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5531 global_avg_mtp_loss: 14.0269 +[titan] 2025-07-09 22:55:41,303 - root - INFO - lr: 1.9940e-04 gnorm: 0.79 [ 9:20:42<12:42:19] +[titan] 2025-07-09 22:55:45,202 - root - INFO - step: 42385 loss: 16.4888 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5431 global_avg_mtp_loss: 13.9457 +[titan] 2025-07-09 22:55:45,202 - root - INFO - lr: 1.9938e-04 gnorm: 0.80 [ 9:20:45<12:42:15] +[titan] 2025-07-09 22:55:49,111 - root - INFO - step: 42390 loss: 16.4275 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 13.8977 +[titan] 2025-07-09 22:55:49,111 - root - INFO - lr: 1.9936e-04 gnorm: 0.83 [ 9:20:49<12:42:11] +[titan] 2025-07-09 22:55:53,010 - root - INFO - step: 42395 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8477 +[titan] 2025-07-09 22:55:53,010 - root - INFO - lr: 1.9934e-04 gnorm: 0.82 [ 9:20:53<12:42:07] +[titan] 2025-07-09 22:55:56,168 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:55:56,979 - root - INFO - step: 42400 loss: 16.6064 memory: 44.58GiB(31.99%) tps: 82,568 tflops: 284.95 mfu: 28.81% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0520 +[titan] 2025-07-09 22:55:56,979 - root - INFO - lr: 1.9932e-04 gnorm: 0.78 [ 9:20:57<12:42:03] +[titan] 2025-07-09 22:56:00,889 - root - INFO - step: 42405 loss: 16.6082 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-09 22:56:00,890 - root - INFO - lr: 1.9930e-04 gnorm: 0.82 [ 9:21:01<12:41:59] +[titan] 2025-07-09 22:56:04,811 - root - INFO - step: 42410 loss: 16.4929 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5375 global_avg_mtp_loss: 13.9554 +[titan] 2025-07-09 22:56:04,812 - root - INFO - lr: 1.9928e-04 gnorm: 0.89 [ 9:21:05<12:41:55] +[titan] 2025-07-09 22:56:08,733 - root - INFO - step: 42415 loss: 16.5856 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0196 +[titan] 2025-07-09 22:56:08,733 - root - INFO - lr: 1.9925e-04 gnorm: 0.81 [ 9:21:09<12:41:51] +[titan] 2025-07-09 22:56:12,664 - root - INFO - step: 42420 loss: 16.6353 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 14.0859 +[titan] 2025-07-09 22:56:12,664 - root - INFO - lr: 1.9923e-04 gnorm: 0.75 [ 9:21:13<12:41:47] +[titan] 2025-07-09 22:56:16,600 - root - INFO - step: 42425 loss: 16.6269 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0625 +[titan] 2025-07-09 22:56:16,600 - root - INFO - lr: 1.9921e-04 gnorm: 0.77 [ 9:21:17<12:41:43] +[titan] 2025-07-09 22:56:20,531 - root - INFO - step: 42430 loss: 16.8802 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.6250 global_avg_mtp_loss: 14.2552 +[titan] 2025-07-09 22:56:20,531 - root - INFO - lr: 1.9919e-04 gnorm: 0.77 [ 9:21:21<12:41:39] +[titan] 2025-07-09 22:56:24,459 - root - INFO - step: 42435 loss: 16.3597 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8433 +[titan] 2025-07-09 22:56:24,460 - root - INFO - lr: 1.9917e-04 gnorm: 0.77 [ 9:21:25<12:41:35] +[titan] 2025-07-09 22:56:28,428 - root - INFO - step: 42440 loss: 16.5345 memory: 44.58GiB(31.99%) tps: 82,579 tflops: 284.99 mfu: 28.82% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 13.9839 +[titan] 2025-07-09 22:56:28,428 - root - INFO - lr: 1.9915e-04 gnorm: 0.78 [ 9:21:29<12:41:31] +[titan] 2025-07-09 22:56:32,383 - root - INFO - step: 42445 loss: 16.8103 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.2208 +[titan] 2025-07-09 22:56:32,383 - root - INFO - lr: 1.9913e-04 gnorm: 0.76 [ 9:21:33<12:41:27] +[titan] 2025-07-09 22:56:35,513 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:56:36,304 - root - INFO - step: 42450 loss: 16.5671 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0106 +[titan] 2025-07-09 22:56:36,304 - root - INFO - lr: 1.9911e-04 gnorm: 0.84 [ 9:21:37<12:41:23] +[titan] 2025-07-09 22:56:40,211 - root - INFO - step: 42455 loss: 16.4482 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-09 22:56:40,211 - root - INFO - lr: 1.9909e-04 gnorm: 0.82 [ 9:21:40<12:41:19] +[titan] 2025-07-09 22:56:44,133 - root - INFO - step: 42460 loss: 16.9976 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6541 global_avg_mtp_loss: 14.3435 +[titan] 2025-07-09 22:56:44,133 - root - INFO - lr: 1.9907e-04 gnorm: 0.81 [ 9:21:44<12:41:15] +[titan] 2025-07-09 22:56:48,061 - root - INFO - step: 42465 loss: 16.6761 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.1048 +[titan] 2025-07-09 22:56:48,061 - root - INFO - lr: 1.9905e-04 gnorm: 0.80 [ 9:21:48<12:41:11] +[titan] 2025-07-09 22:56:51,995 - root - INFO - step: 42470 loss: 16.7586 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1767 +[titan] 2025-07-09 22:56:51,995 - root - INFO - lr: 1.9903e-04 gnorm: 0.78 [ 9:21:52<12:41:07] +[titan] 2025-07-09 22:56:55,938 - root - INFO - step: 42475 loss: 16.5243 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-09 22:56:55,938 - root - INFO - lr: 1.9901e-04 gnorm: 0.86 [ 9:21:56<12:41:03] +[titan] 2025-07-09 22:56:59,888 - root - INFO - step: 42480 loss: 16.5384 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9930 +[titan] 2025-07-09 22:56:59,888 - root - INFO - lr: 1.9899e-04 gnorm: 0.79 [ 9:22:00<12:40:59] +[titan] 2025-07-09 22:57:03,801 - root - INFO - step: 42485 loss: 16.9469 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.6319 global_avg_mtp_loss: 14.3150 +[titan] 2025-07-09 22:57:03,801 - root - INFO - lr: 1.9896e-04 gnorm: 0.79 [ 9:22:04<12:40:55] +[titan] 2025-07-09 22:57:07,696 - root - INFO - step: 42490 loss: 16.5426 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 13.9751 +[titan] 2025-07-09 22:57:07,697 - root - INFO - lr: 1.9894e-04 gnorm: 0.80 [ 9:22:08<12:40:51] +[titan] 2025-07-09 22:57:11,705 - root - INFO - step: 42495 loss: 16.5011 memory: 44.58GiB(31.99%) tps: 81,747 tflops: 282.12 mfu: 28.53% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.9567 +[titan] 2025-07-09 22:57:11,706 - root - INFO - lr: 1.9892e-04 gnorm: 0.78 [ 9:22:12<12:40:47] +[titan] 2025-07-09 22:57:12,649 - root - INFO - Dumping profiler traces at step 42496 +[titan] 2025-07-09 22:57:12,680 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 22:57:15,054 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:57:15,843 - root - INFO - step: 42500 loss: 16.3346 memory: 44.58GiB(31.99%) tps: 79,208 tflops: 273.36 mfu: 27.64% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8182 +[titan] 2025-07-09 22:57:15,843 - root - INFO - lr: 1.9890e-04 gnorm: 0.77 [ 9:22:16<12:40:43] +[titan] 2025-07-09 22:57:19,747 - root - INFO - step: 42505 loss: 16.8249 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.6027 global_avg_mtp_loss: 14.2223 +[titan] 2025-07-09 22:57:19,748 - root - INFO - lr: 1.9888e-04 gnorm: 0.82 [ 9:22:20<12:40:39] +[titan] 2025-07-09 22:57:23,681 - root - INFO - step: 42510 loss: 16.5446 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 13.9853 +[titan] 2025-07-09 22:57:23,681 - root - INFO - lr: 1.9886e-04 gnorm: 0.76 [ 9:22:24<12:40:35] +[titan] 2025-07-09 22:57:27,618 - root - INFO - step: 42515 loss: 16.4260 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.9041 +[titan] 2025-07-09 22:57:27,618 - root - INFO - lr: 1.9884e-04 gnorm: 0.80 [ 9:22:28<12:40:31] +[titan] 2025-07-09 22:57:31,542 - root - INFO - step: 42520 loss: 16.7626 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.1834 +[titan] 2025-07-09 22:57:31,542 - root - INFO - lr: 1.9882e-04 gnorm: 0.72 [ 9:22:32<12:40:27] +[titan] 2025-07-09 22:57:35,481 - root - INFO - step: 42525 loss: 16.3396 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-09 22:57:35,482 - root - INFO - lr: 1.9880e-04 gnorm: 0.86 [ 9:22:36<12:40:23] +[titan] 2025-07-09 22:57:39,395 - root - INFO - step: 42530 loss: 16.5138 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 13.9672 +[titan] 2025-07-09 22:57:39,396 - root - INFO - lr: 1.9878e-04 gnorm: 0.89 [ 9:22:40<12:40:19] +[titan] 2025-07-09 22:57:43,348 - root - INFO - step: 42535 loss: 16.6755 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.1085 +[titan] 2025-07-09 22:57:43,348 - root - INFO - lr: 1.9876e-04 gnorm: 0.88 [ 9:22:44<12:40:15] +[titan] 2025-07-09 22:57:47,254 - root - INFO - step: 42540 loss: 16.4072 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.8643 +[titan] 2025-07-09 22:57:47,254 - root - INFO - lr: 1.9874e-04 gnorm: 0.79 [ 9:22:47<12:40:11] +[titan] 2025-07-09 22:57:51,156 - root - INFO - step: 42545 loss: 16.7204 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1409 +[titan] 2025-07-09 22:57:51,157 - root - INFO - lr: 1.9872e-04 gnorm: 0.76 [ 9:22:51<12:40:07] +[titan] 2025-07-09 22:57:54,301 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:57:55,089 - root - INFO - step: 42550 loss: 16.6562 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0892 +[titan] 2025-07-09 22:57:55,090 - root - INFO - lr: 1.9870e-04 gnorm: 0.77 [ 9:22:55<12:40:03] +[titan] 2025-07-09 22:57:59,010 - root - INFO - step: 42555 loss: 16.6114 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0520 +[titan] 2025-07-09 22:57:59,010 - root - INFO - lr: 1.9867e-04 gnorm: 0.72 [ 9:22:59<12:39:59] +[titan] 2025-07-09 22:58:02,951 - root - INFO - step: 42560 loss: 16.4211 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.8915 +[titan] 2025-07-09 22:58:02,951 - root - INFO - lr: 1.9865e-04 gnorm: 0.76 [ 9:23:03<12:39:55] +[titan] 2025-07-09 22:58:06,857 - root - INFO - step: 42565 loss: 16.7006 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.1284 +[titan] 2025-07-09 22:58:06,857 - root - INFO - lr: 1.9863e-04 gnorm: 0.77 [ 9:23:07<12:39:51] +[titan] 2025-07-09 22:58:10,775 - root - INFO - step: 42570 loss: 16.2942 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.7872 +[titan] 2025-07-09 22:58:10,776 - root - INFO - lr: 1.9861e-04 gnorm: 0.77 [ 9:23:11<12:39:47] +[titan] 2025-07-09 22:58:14,701 - root - INFO - step: 42575 loss: 16.7626 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1689 +[titan] 2025-07-09 22:58:14,701 - root - INFO - lr: 1.9859e-04 gnorm: 0.79 [ 9:23:15<12:39:43] +[titan] 2025-07-09 22:58:18,606 - root - INFO - step: 42580 loss: 16.4075 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8867 +[titan] 2025-07-09 22:58:18,606 - root - INFO - lr: 1.9857e-04 gnorm: 0.77 [ 9:23:19<12:39:39] +[titan] 2025-07-09 22:58:22,548 - root - INFO - step: 42585 loss: 16.5797 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 14.0288 +[titan] 2025-07-09 22:58:22,548 - root - INFO - lr: 1.9855e-04 gnorm: 0.74 [ 9:23:23<12:39:35] +[titan] 2025-07-09 22:58:26,460 - root - INFO - step: 42590 loss: 16.4994 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9603 +[titan] 2025-07-09 22:58:26,460 - root - INFO - lr: 1.9853e-04 gnorm: 0.78 [ 9:23:27<12:39:31] +[titan] 2025-07-09 22:58:30,391 - root - INFO - step: 42595 loss: 16.7011 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.1237 +[titan] 2025-07-09 22:58:30,391 - root - INFO - lr: 1.9851e-04 gnorm: 0.78 [ 9:23:31<12:39:27] +[titan] 2025-07-09 22:58:33,515 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:58:34,306 - root - INFO - step: 42600 loss: 16.7376 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6003 global_avg_mtp_loss: 14.1373 +[titan] 2025-07-09 22:58:34,306 - root - INFO - lr: 1.9849e-04 gnorm: 0.79 [ 9:23:35<12:39:22] +[titan] 2025-07-09 22:58:38,244 - root - INFO - step: 42605 loss: 16.5852 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0250 +[titan] 2025-07-09 22:58:38,244 - root - INFO - lr: 1.9847e-04 gnorm: 0.83 [ 9:23:38<12:39:18] +[titan] 2025-07-09 22:58:42,157 - root - INFO - step: 42610 loss: 16.7820 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5942 global_avg_mtp_loss: 14.1878 +[titan] 2025-07-09 22:58:42,157 - root - INFO - lr: 1.9845e-04 gnorm: 0.83 [ 9:23:42<12:39:14] +[titan] 2025-07-09 22:58:46,055 - root - INFO - step: 42615 loss: 16.8830 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.6122 global_avg_mtp_loss: 14.2708 +[titan] 2025-07-09 22:58:46,055 - root - INFO - lr: 1.9843e-04 gnorm: 0.75 [ 9:23:46<12:39:10] +[titan] 2025-07-09 22:58:49,972 - root - INFO - step: 42620 loss: 16.5959 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5492 global_avg_mtp_loss: 14.0468 +[titan] 2025-07-09 22:58:49,973 - root - INFO - lr: 1.9840e-04 gnorm: 0.79 [ 9:23:50<12:39:06] +[titan] 2025-07-09 22:58:53,920 - root - INFO - step: 42625 loss: 16.6165 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0492 +[titan] 2025-07-09 22:58:53,920 - root - INFO - lr: 1.9838e-04 gnorm: 0.73 [ 9:23:54<12:39:02] +[titan] 2025-07-09 22:58:57,861 - root - INFO - step: 42630 loss: 16.4610 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9341 +[titan] 2025-07-09 22:58:57,862 - root - INFO - lr: 1.9836e-04 gnorm: 0.75 [ 9:23:58<12:38:58] +[titan] 2025-07-09 22:59:01,766 - root - INFO - step: 42635 loss: 16.5611 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 13.9793 +[titan] 2025-07-09 22:59:01,766 - root - INFO - lr: 1.9834e-04 gnorm: 0.81 [ 9:24:02<12:38:54] +[titan] 2025-07-09 22:59:05,674 - root - INFO - step: 42640 loss: 16.8642 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.2716 +[titan] 2025-07-09 22:59:05,674 - root - INFO - lr: 1.9832e-04 gnorm: 0.78 [ 9:24:06<12:38:50] +[titan] 2025-07-09 22:59:09,578 - root - INFO - step: 42645 loss: 16.6561 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0825 +[titan] 2025-07-09 22:59:09,578 - root - INFO - lr: 1.9830e-04 gnorm: 0.77 [ 9:24:10<12:38:46] +[titan] 2025-07-09 22:59:12,706 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:59:13,495 - root - INFO - step: 42650 loss: 16.2901 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.7745 +[titan] 2025-07-09 22:59:13,496 - root - INFO - lr: 1.9828e-04 gnorm: 0.78 [ 9:24:14<12:38:42] +[titan] 2025-07-09 22:59:17,409 - root - INFO - step: 42655 loss: 16.6553 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5685 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-09 22:59:17,410 - root - INFO - lr: 1.9826e-04 gnorm: 0.77 [ 9:24:18<12:38:38] +[titan] 2025-07-09 22:59:21,309 - root - INFO - step: 42660 loss: 16.3969 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.8711 +[titan] 2025-07-09 22:59:21,309 - root - INFO - lr: 1.9824e-04 gnorm: 0.79 [ 9:24:22<12:38:34] +[titan] 2025-07-09 22:59:25,214 - root - INFO - step: 42665 loss: 16.6731 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0913 +[titan] 2025-07-09 22:59:25,214 - root - INFO - lr: 1.9822e-04 gnorm: 0.80 [ 9:24:25<12:38:30] +[titan] 2025-07-09 22:59:29,174 - root - INFO - step: 42670 loss: 16.9485 memory: 44.58GiB(31.99%) tps: 82,762 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.6331 global_avg_mtp_loss: 14.3154 +[titan] 2025-07-09 22:59:29,174 - root - INFO - lr: 1.9820e-04 gnorm: 0.86 [ 9:24:29<12:38:26] +[titan] 2025-07-09 22:59:33,084 - root - INFO - step: 42675 loss: 16.7285 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1440 +[titan] 2025-07-09 22:59:33,084 - root - INFO - lr: 1.9818e-04 gnorm: 0.80 [ 9:24:33<12:38:22] +[titan] 2025-07-09 22:59:37,002 - root - INFO - step: 42680 loss: 16.6181 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0459 +[titan] 2025-07-09 22:59:37,002 - root - INFO - lr: 1.9816e-04 gnorm: 0.76 [ 9:24:37<12:38:18] +[titan] 2025-07-09 22:59:40,901 - root - INFO - step: 42685 loss: 16.7120 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5825 global_avg_mtp_loss: 14.1295 +[titan] 2025-07-09 22:59:40,901 - root - INFO - lr: 1.9813e-04 gnorm: 0.79 [ 9:24:41<12:38:14] +[titan] 2025-07-09 22:59:44,851 - root - INFO - step: 42690 loss: 16.4909 memory: 44.58GiB(31.99%) tps: 82,965 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9544 +[titan] 2025-07-09 22:59:44,851 - root - INFO - lr: 1.9811e-04 gnorm: 0.77 [ 9:24:45<12:38:10] +[titan] 2025-07-09 22:59:48,745 - root - INFO - step: 42695 loss: 16.5449 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-09 22:59:48,745 - root - INFO - lr: 1.9809e-04 gnorm: 0.74 [ 9:24:49<12:38:06] +[titan] 2025-07-09 22:59:51,872 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 22:59:52,661 - root - INFO - step: 42700 loss: 16.5906 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0356 +[titan] 2025-07-09 22:59:52,661 - root - INFO - lr: 1.9807e-04 gnorm: 0.87 [ 9:24:53<12:38:02] +[titan] 2025-07-09 22:59:56,576 - root - INFO - step: 42705 loss: 16.6056 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0501 +[titan] 2025-07-09 22:59:56,576 - root - INFO - lr: 1.9805e-04 gnorm: 0.80 [ 9:24:57<12:37:58] +[titan] 2025-07-09 23:00:00,505 - root - INFO - step: 42710 loss: 16.4593 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-09 23:00:00,505 - root - INFO - lr: 1.9803e-04 gnorm: 0.87 [ 9:25:01<12:37:54] +[titan] 2025-07-09 23:00:04,431 - root - INFO - step: 42715 loss: 16.5195 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.9886 +[titan] 2025-07-09 23:00:04,431 - root - INFO - lr: 1.9801e-04 gnorm: 0.75 [ 9:25:05<12:37:50] +[titan] 2025-07-09 23:00:08,334 - root - INFO - step: 42720 loss: 16.5989 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 14.0522 +[titan] 2025-07-09 23:00:08,335 - root - INFO - lr: 1.9799e-04 gnorm: 0.74 [ 9:25:09<12:37:46] +[titan] 2025-07-09 23:00:12,237 - root - INFO - step: 42725 loss: 16.6644 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0969 +[titan] 2025-07-09 23:00:12,237 - root - INFO - lr: 1.9797e-04 gnorm: 0.77 [ 9:25:12<12:37:42] +[titan] 2025-07-09 23:00:16,141 - root - INFO - step: 42730 loss: 16.7022 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5869 global_avg_mtp_loss: 14.1153 +[titan] 2025-07-09 23:00:16,142 - root - INFO - lr: 1.9795e-04 gnorm: 0.80 [ 9:25:16<12:37:37] +[titan] 2025-07-09 23:00:20,053 - root - INFO - step: 42735 loss: 16.6511 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.0747 +[titan] 2025-07-09 23:00:20,053 - root - INFO - lr: 1.9793e-04 gnorm: 0.78 [ 9:25:20<12:37:33] +[titan] 2025-07-09 23:00:23,966 - root - INFO - step: 42740 loss: 16.5000 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9519 +[titan] 2025-07-09 23:00:23,966 - root - INFO - lr: 1.9791e-04 gnorm: 0.80 [ 9:25:24<12:37:29] +[titan] 2025-07-09 23:00:27,904 - root - INFO - step: 42745 loss: 16.6898 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.1017 +[titan] 2025-07-09 23:00:27,904 - root - INFO - lr: 1.9789e-04 gnorm: 0.77 [ 9:25:28<12:37:25] +[titan] 2025-07-09 23:00:31,060 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:00:31,858 - root - INFO - step: 42750 loss: 16.4762 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.9324 +[titan] 2025-07-09 23:00:31,858 - root - INFO - lr: 1.9786e-04 gnorm: 0.81 [ 9:25:32<12:37:21] +[titan] 2025-07-09 23:00:35,783 - root - INFO - step: 42755 loss: 16.6801 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.0909 +[titan] 2025-07-09 23:00:35,783 - root - INFO - lr: 1.9784e-04 gnorm: 0.80 [ 9:25:36<12:37:17] +[titan] 2025-07-09 23:00:39,710 - root - INFO - step: 42760 loss: 16.5644 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 14.0141 +[titan] 2025-07-09 23:00:39,711 - root - INFO - lr: 1.9782e-04 gnorm: 0.78 [ 9:25:40<12:37:13] +[titan] 2025-07-09 23:00:43,618 - root - INFO - step: 42765 loss: 16.2915 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.7884 +[titan] 2025-07-09 23:00:43,618 - root - INFO - lr: 1.9780e-04 gnorm: 0.79 [ 9:25:44<12:37:09] +[titan] 2025-07-09 23:00:47,517 - root - INFO - step: 42770 loss: 16.8990 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.6149 global_avg_mtp_loss: 14.2841 +[titan] 2025-07-09 23:00:47,518 - root - INFO - lr: 1.9778e-04 gnorm: 0.76 [ 9:25:48<12:37:05] +[titan] 2025-07-09 23:00:51,420 - root - INFO - step: 42775 loss: 16.7401 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1562 +[titan] 2025-07-09 23:00:51,420 - root - INFO - lr: 1.9776e-04 gnorm: 0.77 [ 9:25:52<12:37:01] +[titan] 2025-07-09 23:00:55,332 - root - INFO - step: 42780 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-09 23:00:55,333 - root - INFO - lr: 1.9774e-04 gnorm: 0.77 [ 9:25:56<12:36:57] +[titan] 2025-07-09 23:00:59,247 - root - INFO - step: 42785 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9356 +[titan] 2025-07-09 23:00:59,247 - root - INFO - lr: 1.9772e-04 gnorm: 0.80 [ 9:25:59<12:36:53] +[titan] 2025-07-09 23:01:03,182 - root - INFO - step: 42790 loss: 16.6237 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0444 +[titan] 2025-07-09 23:01:03,183 - root - INFO - lr: 1.9770e-04 gnorm: 0.78 [ 9:26:03<12:36:49] +[titan] 2025-07-09 23:01:07,079 - root - INFO - step: 42795 loss: 16.4684 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9202 +[titan] 2025-07-09 23:01:07,079 - root - INFO - lr: 1.9768e-04 gnorm: 0.77 [ 9:26:07<12:36:45] +[titan] 2025-07-09 23:01:10,179 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:01:10,971 - root - INFO - step: 42800 loss: 16.5719 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 14.0233 +[titan] 2025-07-09 23:01:10,972 - root - INFO - lr: 1.9766e-04 gnorm: 0.78 [ 9:26:11<12:36:41] +[titan] 2025-07-09 23:01:14,893 - root - INFO - step: 42805 loss: 16.5671 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0139 +[titan] 2025-07-09 23:01:14,894 - root - INFO - lr: 1.9764e-04 gnorm: 0.79 [ 9:26:15<12:36:37] +[titan] 2025-07-09 23:01:18,785 - root - INFO - step: 42810 loss: 16.3224 memory: 44.58GiB(31.99%) tps: 84,216 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.8004 +[titan] 2025-07-09 23:01:18,785 - root - INFO - lr: 1.9762e-04 gnorm: 0.77 [ 9:26:19<12:36:33] +[titan] 2025-07-09 23:01:22,691 - root - INFO - step: 42815 loss: 16.7004 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1289 +[titan] 2025-07-09 23:01:22,692 - root - INFO - lr: 1.9759e-04 gnorm: 0.77 [ 9:26:23<12:36:29] +[titan] 2025-07-09 23:01:26,616 - root - INFO - step: 42820 loss: 16.8562 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6053 global_avg_mtp_loss: 14.2509 +[titan] 2025-07-09 23:01:26,616 - root - INFO - lr: 1.9757e-04 gnorm: 0.76 [ 9:26:27<12:36:25] +[titan] 2025-07-09 23:01:30,534 - root - INFO - step: 42825 loss: 16.3916 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8742 +[titan] 2025-07-09 23:01:30,534 - root - INFO - lr: 1.9755e-04 gnorm: 0.77 [ 9:26:31<12:36:21] +[titan] 2025-07-09 23:01:34,473 - root - INFO - step: 42830 loss: 16.7816 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5990 global_avg_mtp_loss: 14.1826 +[titan] 2025-07-09 23:01:34,473 - root - INFO - lr: 1.9753e-04 gnorm: 0.80 [ 9:26:35<12:36:17] +[titan] 2025-07-09 23:01:38,426 - root - INFO - step: 42835 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 82,897 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 14.0531 +[titan] 2025-07-09 23:01:38,426 - root - INFO - lr: 1.9751e-04 gnorm: 0.80 [ 9:26:39<12:36:13] +[titan] 2025-07-09 23:01:42,363 - root - INFO - step: 42840 loss: 16.6849 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.1152 +[titan] 2025-07-09 23:01:42,363 - root - INFO - lr: 1.9749e-04 gnorm: 0.76 [ 9:26:43<12:36:09] +[titan] 2025-07-09 23:01:46,305 - root - INFO - step: 42845 loss: 16.8593 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2385 +[titan] 2025-07-09 23:01:46,305 - root - INFO - lr: 1.9747e-04 gnorm: 0.85 [ 9:26:47<12:36:05] +[titan] 2025-07-09 23:01:49,438 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:01:50,233 - root - INFO - step: 42850 loss: 16.5994 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0342 +[titan] 2025-07-09 23:01:50,233 - root - INFO - lr: 1.9745e-04 gnorm: 0.75 [ 9:26:50<12:36:01] +[titan] 2025-07-09 23:01:54,138 - root - INFO - step: 42855 loss: 16.6735 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.0934 +[titan] 2025-07-09 23:01:54,138 - root - INFO - lr: 1.9743e-04 gnorm: 0.88 [ 9:26:54<12:35:57] +[titan] 2025-07-09 23:01:58,040 - root - INFO - step: 42860 loss: 16.6247 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0517 +[titan] 2025-07-09 23:01:58,041 - root - INFO - lr: 1.9741e-04 gnorm: 0.75 [ 9:26:58<12:35:53] +[titan] 2025-07-09 23:02:01,956 - root - INFO - step: 42865 loss: 16.0876 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.6149 +[titan] 2025-07-09 23:02:01,956 - root - INFO - lr: 1.9739e-04 gnorm: 0.83 [ 9:27:02<12:35:49] +[titan] 2025-07-09 23:02:05,879 - root - INFO - step: 42870 loss: 16.8233 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6036 global_avg_mtp_loss: 14.2197 +[titan] 2025-07-09 23:02:05,879 - root - INFO - lr: 1.9737e-04 gnorm: 0.82 [ 9:27:06<12:35:44] +[titan] 2025-07-09 23:02:09,809 - root - INFO - step: 42875 loss: 16.5311 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5449 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-09 23:02:09,809 - root - INFO - lr: 1.9735e-04 gnorm: 0.80 [ 9:27:10<12:35:40] +[titan] 2025-07-09 23:02:13,711 - root - INFO - step: 42880 loss: 16.6352 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0676 +[titan] 2025-07-09 23:02:13,711 - root - INFO - lr: 1.9732e-04 gnorm: 0.78 [ 9:27:14<12:35:36] +[titan] 2025-07-09 23:02:17,634 - root - INFO - step: 42885 loss: 16.4281 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9005 +[titan] 2025-07-09 23:02:17,634 - root - INFO - lr: 1.9730e-04 gnorm: 0.81 [ 9:27:18<12:35:32] +[titan] 2025-07-09 23:02:21,525 - root - INFO - step: 42890 loss: 16.7927 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5938 global_avg_mtp_loss: 14.1989 +[titan] 2025-07-09 23:02:21,525 - root - INFO - lr: 1.9728e-04 gnorm: 0.79 [ 9:27:22<12:35:28] +[titan] 2025-07-09 23:02:25,436 - root - INFO - step: 42895 loss: 16.6516 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0785 +[titan] 2025-07-09 23:02:25,437 - root - INFO - lr: 1.9726e-04 gnorm: 0.85 [ 9:27:26<12:35:24] +[titan] 2025-07-09 23:02:28,593 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:02:29,385 - root - INFO - step: 42900 loss: 16.5647 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0072 +[titan] 2025-07-09 23:02:29,385 - root - INFO - lr: 1.9724e-04 gnorm: 0.76 [ 9:27:30<12:35:20] +[titan] 2025-07-09 23:02:33,310 - root - INFO - step: 42905 loss: 16.5007 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 13.9513 +[titan] 2025-07-09 23:02:33,310 - root - INFO - lr: 1.9722e-04 gnorm: 0.76 [ 9:27:34<12:35:16] +[titan] 2025-07-09 23:02:37,196 - root - INFO - step: 42910 loss: 16.7593 memory: 44.58GiB(31.99%) tps: 84,324 tflops: 291.02 mfu: 29.43% global_avg_ntp_loss: 2.5926 global_avg_mtp_loss: 14.1667 +[titan] 2025-07-09 23:02:37,196 - root - INFO - lr: 1.9720e-04 gnorm: 0.77 [ 9:27:37<12:35:12] +[titan] 2025-07-09 23:02:41,116 - root - INFO - step: 42915 loss: 16.6531 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.0849 +[titan] 2025-07-09 23:02:41,116 - root - INFO - lr: 1.9718e-04 gnorm: 0.77 [ 9:27:41<12:35:08] +[titan] 2025-07-09 23:02:45,044 - root - INFO - step: 42920 loss: 16.8502 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.6106 global_avg_mtp_loss: 14.2396 +[titan] 2025-07-09 23:02:45,044 - root - INFO - lr: 1.9716e-04 gnorm: 0.82 [ 9:27:45<12:35:04] +[titan] 2025-07-09 23:02:48,989 - root - INFO - step: 42925 loss: 16.5020 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 13.9532 +[titan] 2025-07-09 23:02:48,989 - root - INFO - lr: 1.9714e-04 gnorm: 0.79 [ 9:27:49<12:35:00] +[titan] 2025-07-09 23:02:52,944 - root - INFO - step: 42930 loss: 16.7792 memory: 44.58GiB(31.99%) tps: 82,850 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.1846 +[titan] 2025-07-09 23:02:52,945 - root - INFO - lr: 1.9712e-04 gnorm: 0.78 [ 9:27:53<12:34:56] +[titan] 2025-07-09 23:02:56,882 - root - INFO - step: 42935 loss: 16.7097 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5805 global_avg_mtp_loss: 14.1293 +[titan] 2025-07-09 23:02:56,882 - root - INFO - lr: 1.9710e-04 gnorm: 0.78 [ 9:27:57<12:34:52] +[titan] 2025-07-09 23:03:00,795 - root - INFO - step: 42940 loss: 16.6833 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.1147 +[titan] 2025-07-09 23:03:00,796 - root - INFO - lr: 1.9707e-04 gnorm: 0.82 [ 9:28:01<12:34:48] +[titan] 2025-07-09 23:03:04,730 - root - INFO - step: 42945 loss: 16.7429 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5785 global_avg_mtp_loss: 14.1644 +[titan] 2025-07-09 23:03:04,730 - root - INFO - lr: 1.9705e-04 gnorm: 0.80 [ 9:28:05<12:34:44] +[titan] 2025-07-09 23:03:07,837 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:03:08,625 - root - INFO - step: 42950 loss: 16.5716 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-09 23:03:08,625 - root - INFO - lr: 1.9703e-04 gnorm: 0.74 [ 9:28:09<12:34:40] +[titan] 2025-07-09 23:03:12,529 - root - INFO - step: 42955 loss: 16.6740 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.0975 +[titan] 2025-07-09 23:03:12,529 - root - INFO - lr: 1.9701e-04 gnorm: 0.79 [ 9:28:13<12:34:36] +[titan] 2025-07-09 23:03:16,428 - root - INFO - step: 42960 loss: 16.8449 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.6128 global_avg_mtp_loss: 14.2320 +[titan] 2025-07-09 23:03:16,428 - root - INFO - lr: 1.9699e-04 gnorm: 0.78 [ 9:28:17<12:34:32] +[titan] 2025-07-09 23:03:20,332 - root - INFO - step: 42965 loss: 16.4649 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9234 +[titan] 2025-07-09 23:03:20,332 - root - INFO - lr: 1.9697e-04 gnorm: 0.74 [ 9:28:21<12:34:28] +[titan] 2025-07-09 23:03:24,252 - root - INFO - step: 42970 loss: 16.6833 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 23:03:24,253 - root - INFO - lr: 1.9695e-04 gnorm: 0.77 [ 9:28:24<12:34:24] +[titan] 2025-07-09 23:03:28,160 - root - INFO - step: 42975 loss: 16.5506 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 13.9862 +[titan] 2025-07-09 23:03:28,160 - root - INFO - lr: 1.9693e-04 gnorm: 0.75 [ 9:28:28<12:34:20] +[titan] 2025-07-09 23:03:32,134 - root - INFO - step: 42980 loss: 16.6135 memory: 44.58GiB(31.99%) tps: 82,461 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0535 +[titan] 2025-07-09 23:03:32,135 - root - INFO - lr: 1.9691e-04 gnorm: 0.76 [ 9:28:32<12:34:16] +[titan] 2025-07-09 23:03:36,060 - root - INFO - step: 42985 loss: 16.9646 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.6312 global_avg_mtp_loss: 14.3333 +[titan] 2025-07-09 23:03:36,061 - root - INFO - lr: 1.9689e-04 gnorm: 0.82 [ 9:28:36<12:34:12] +[titan] 2025-07-09 23:03:40,030 - root - INFO - step: 42990 loss: 16.5563 memory: 44.58GiB(31.99%) tps: 82,558 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 14.0084 +[titan] 2025-07-09 23:03:40,030 - root - INFO - lr: 1.9687e-04 gnorm: 0.84 [ 9:28:40<12:34:08] +[titan] 2025-07-09 23:03:43,955 - root - INFO - step: 42995 loss: 16.6701 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.0886 +[titan] 2025-07-09 23:03:43,955 - root - INFO - lr: 1.9685e-04 gnorm: 0.79 [ 9:28:44<12:34:04] +[titan] 2025-07-09 23:03:47,100 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:03:47,912 - root - INFO - step: 43000 loss: 16.6501 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5784 global_avg_mtp_loss: 14.0717 +[titan] 2025-07-09 23:03:47,912 - root - INFO - lr: 1.9683e-04 gnorm: 0.76 [ 9:28:48<12:34:00] +[titan] 2025-07-09 23:03:51,892 - root - INFO - step: 43005 loss: 16.6439 memory: 44.58GiB(31.99%) tps: 82,337 tflops: 284.16 mfu: 28.73% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0780 +[titan] 2025-07-09 23:03:51,892 - root - INFO - lr: 1.9680e-04 gnorm: 0.83 [ 9:28:52<12:33:56] +[titan] 2025-07-09 23:03:54,393 - root - INFO - Dumping profiler traces at step 43008 +[titan] 2025-07-09 23:03:54,426 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:03:56,012 - root - INFO - step: 43010 loss: 16.5972 memory: 44.58GiB(31.99%) tps: 79,540 tflops: 274.51 mfu: 27.76% global_avg_ntp_loss: 2.5612 global_avg_mtp_loss: 14.0360 +[titan] 2025-07-09 23:03:56,012 - root - INFO - lr: 1.9678e-04 gnorm: 0.80 [ 9:28:56<12:33:52] +[titan] 2025-07-09 23:03:59,927 - root - INFO - step: 43015 loss: 16.5209 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 13.9729 +[titan] 2025-07-09 23:03:59,928 - root - INFO - lr: 1.9676e-04 gnorm: 0.79 [ 9:29:00<12:33:48] +[titan] 2025-07-09 23:04:03,830 - root - INFO - step: 43020 loss: 16.6741 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.0909 +[titan] 2025-07-09 23:04:03,830 - root - INFO - lr: 1.9674e-04 gnorm: 0.81 [ 9:29:04<12:33:44] +[titan] 2025-07-09 23:04:07,735 - root - INFO - step: 43025 loss: 16.6286 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0626 +[titan] 2025-07-09 23:04:07,736 - root - INFO - lr: 1.9672e-04 gnorm: 0.81 [ 9:29:08<12:33:40] +[titan] 2025-07-09 23:04:11,626 - root - INFO - step: 43030 loss: 16.3632 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-09 23:04:11,626 - root - INFO - lr: 1.9670e-04 gnorm: 0.76 [ 9:29:12<12:33:36] +[titan] 2025-07-09 23:04:15,530 - root - INFO - step: 43035 loss: 16.5043 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9667 +[titan] 2025-07-09 23:04:15,531 - root - INFO - lr: 1.9668e-04 gnorm: 0.77 [ 9:29:16<12:33:32] +[titan] 2025-07-09 23:04:19,426 - root - INFO - step: 43040 loss: 16.6680 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.0885 +[titan] 2025-07-09 23:04:19,426 - root - INFO - lr: 1.9666e-04 gnorm: 0.81 [ 9:29:20<12:33:28] +[titan] 2025-07-09 23:04:23,366 - root - INFO - step: 43045 loss: 16.5152 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 13.9715 +[titan] 2025-07-09 23:04:23,366 - root - INFO - lr: 1.9664e-04 gnorm: 0.74 [ 9:29:24<12:33:24] +[titan] 2025-07-09 23:04:26,488 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:04:27,278 - root - INFO - step: 43050 loss: 16.7228 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1394 +[titan] 2025-07-09 23:04:27,278 - root - INFO - lr: 1.9662e-04 gnorm: 0.80 [ 9:29:27<12:33:20] +[titan] 2025-07-09 23:04:31,172 - root - INFO - step: 43055 loss: 16.7590 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1661 +[titan] 2025-07-09 23:04:31,172 - root - INFO - lr: 1.9660e-04 gnorm: 0.78 [ 9:29:31<12:33:16] +[titan] 2025-07-09 23:04:35,087 - root - INFO - step: 43060 loss: 16.5250 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5509 global_avg_mtp_loss: 13.9741 +[titan] 2025-07-09 23:04:35,087 - root - INFO - lr: 1.9658e-04 gnorm: 0.80 [ 9:29:35<12:33:12] +[titan] 2025-07-09 23:04:38,989 - root - INFO - step: 43065 loss: 16.5253 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9826 +[titan] 2025-07-09 23:04:38,989 - root - INFO - lr: 1.9655e-04 gnorm: 0.77 [ 9:29:39<12:33:07] +[titan] 2025-07-09 23:04:42,946 - root - INFO - step: 43070 loss: 16.6166 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5649 global_avg_mtp_loss: 14.0517 +[titan] 2025-07-09 23:04:42,946 - root - INFO - lr: 1.9653e-04 gnorm: 0.76 [ 9:29:43<12:33:03] +[titan] 2025-07-09 23:04:46,880 - root - INFO - step: 43075 loss: 16.7860 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5974 global_avg_mtp_loss: 14.1886 +[titan] 2025-07-09 23:04:46,880 - root - INFO - lr: 1.9651e-04 gnorm: 0.75 [ 9:29:47<12:32:59] +[titan] 2025-07-09 23:04:50,799 - root - INFO - step: 43080 loss: 16.9825 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.6402 global_avg_mtp_loss: 14.3423 +[titan] 2025-07-09 23:04:50,799 - root - INFO - lr: 1.9649e-04 gnorm: 0.73 [ 9:29:51<12:32:55] +[titan] 2025-07-09 23:04:54,697 - root - INFO - step: 43085 loss: 16.5238 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.34% global_avg_ntp_loss: 2.5612 global_avg_mtp_loss: 13.9626 +[titan] 2025-07-09 23:04:54,697 - root - INFO - lr: 1.9647e-04 gnorm: 0.75 [ 9:29:55<12:32:51] +[titan] 2025-07-09 23:04:58,599 - root - INFO - step: 43090 loss: 16.2745 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.7709 +[titan] 2025-07-09 23:04:58,600 - root - INFO - lr: 1.9645e-04 gnorm: 0.78 [ 9:29:59<12:32:47] +[titan] 2025-07-09 23:05:02,498 - root - INFO - step: 43095 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 13.9864 +[titan] 2025-07-09 23:05:02,498 - root - INFO - lr: 1.9643e-04 gnorm: 0.79 [ 9:30:03<12:32:43] +[titan] 2025-07-09 23:05:05,616 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:05:06,412 - root - INFO - step: 43100 loss: 16.6860 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.1070 +[titan] 2025-07-09 23:05:06,412 - root - INFO - lr: 1.9641e-04 gnorm: 0.78 [ 9:30:07<12:32:39] +[titan] 2025-07-09 23:05:10,344 - root - INFO - step: 43105 loss: 16.3767 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8677 +[titan] 2025-07-09 23:05:10,344 - root - INFO - lr: 1.9639e-04 gnorm: 0.76 [ 9:30:11<12:32:35] +[titan] 2025-07-09 23:05:14,278 - root - INFO - step: 43110 loss: 16.7632 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5949 global_avg_mtp_loss: 14.1683 +[titan] 2025-07-09 23:05:14,278 - root - INFO - lr: 1.9637e-04 gnorm: 0.82 [ 9:30:14<12:32:31] +[titan] 2025-07-09 23:05:18,195 - root - INFO - step: 43115 loss: 16.8184 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.2151 +[titan] 2025-07-09 23:05:18,195 - root - INFO - lr: 1.9635e-04 gnorm: 0.76 [ 9:30:18<12:32:27] +[titan] 2025-07-09 23:05:22,124 - root - INFO - step: 43120 loss: 16.4706 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 13.9356 +[titan] 2025-07-09 23:05:22,124 - root - INFO - lr: 1.9633e-04 gnorm: 0.84 [ 9:30:22<12:32:23] +[titan] 2025-07-09 23:05:26,027 - root - INFO - step: 43125 loss: 16.4152 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.8940 +[titan] 2025-07-09 23:05:26,027 - root - INFO - lr: 1.9630e-04 gnorm: 0.72 [ 9:30:26<12:32:19] +[titan] 2025-07-09 23:05:29,982 - root - INFO - step: 43130 loss: 16.7614 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.1567 +[titan] 2025-07-09 23:05:29,982 - root - INFO - lr: 1.9628e-04 gnorm: 0.76 [ 9:30:30<12:32:15] +[titan] 2025-07-09 23:05:33,952 - root - INFO - step: 43135 loss: 16.7067 memory: 44.58GiB(31.99%) tps: 82,534 tflops: 284.84 mfu: 28.80% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.1278 +[titan] 2025-07-09 23:05:33,953 - root - INFO - lr: 1.9626e-04 gnorm: 0.87 [ 9:30:34<12:32:11] +[titan] 2025-07-09 23:05:37,876 - root - INFO - step: 43140 loss: 16.5968 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0436 +[titan] 2025-07-09 23:05:37,877 - root - INFO - lr: 1.9624e-04 gnorm: 0.86 [ 9:30:38<12:32:07] +[titan] 2025-07-09 23:05:41,791 - root - INFO - step: 43145 loss: 16.4794 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5342 global_avg_mtp_loss: 13.9451 +[titan] 2025-07-09 23:05:41,791 - root - INFO - lr: 1.9622e-04 gnorm: 0.75 [ 9:30:42<12:32:03] +[titan] 2025-07-09 23:05:44,901 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:05:45,693 - root - INFO - step: 43150 loss: 16.7272 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.1434 +[titan] 2025-07-09 23:05:45,693 - root - INFO - lr: 1.9620e-04 gnorm: 0.81 [ 9:30:46<12:31:59] +[titan] 2025-07-09 23:05:49,582 - root - INFO - step: 43155 loss: 16.5225 memory: 44.58GiB(31.99%) tps: 84,267 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9765 +[titan] 2025-07-09 23:05:49,582 - root - INFO - lr: 1.9618e-04 gnorm: 0.78 [ 9:30:50<12:31:55] +[titan] 2025-07-09 23:05:53,477 - root - INFO - step: 43160 loss: 16.6550 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0941 +[titan] 2025-07-09 23:05:53,478 - root - INFO - lr: 1.9616e-04 gnorm: 0.77 [ 9:30:54<12:31:51] +[titan] 2025-07-09 23:05:57,390 - root - INFO - step: 43165 loss: 16.5498 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 13.9977 +[titan] 2025-07-09 23:05:57,390 - root - INFO - lr: 1.9614e-04 gnorm: 0.78 [ 9:30:58<12:31:47] +[titan] 2025-07-09 23:06:01,310 - root - INFO - step: 43170 loss: 16.7710 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5956 global_avg_mtp_loss: 14.1755 +[titan] 2025-07-09 23:06:01,310 - root - INFO - lr: 1.9612e-04 gnorm: 0.78 [ 9:31:01<12:31:43] +[titan] 2025-07-09 23:06:05,222 - root - INFO - step: 43175 loss: 16.3302 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.8127 +[titan] 2025-07-09 23:06:05,223 - root - INFO - lr: 1.9610e-04 gnorm: 0.83 [ 9:31:05<12:31:39] +[titan] 2025-07-09 23:06:09,159 - root - INFO - step: 43180 loss: 16.6398 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.0698 +[titan] 2025-07-09 23:06:09,160 - root - INFO - lr: 1.9608e-04 gnorm: 0.77 [ 9:31:09<12:31:35] +[titan] 2025-07-09 23:06:13,093 - root - INFO - step: 43185 loss: 16.5713 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0043 +[titan] 2025-07-09 23:06:13,094 - root - INFO - lr: 1.9605e-04 gnorm: 0.79 [ 9:31:13<12:31:31] +[titan] 2025-07-09 23:06:16,987 - root - INFO - step: 43190 loss: 16.2600 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.7570 +[titan] 2025-07-09 23:06:16,988 - root - INFO - lr: 1.9603e-04 gnorm: 0.83 [ 9:31:17<12:31:27] +[titan] 2025-07-09 23:06:20,918 - root - INFO - step: 43195 loss: 16.6494 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0786 +[titan] 2025-07-09 23:06:20,918 - root - INFO - lr: 1.9601e-04 gnorm: 0.78 [ 9:31:21<12:31:23] +[titan] 2025-07-09 23:06:24,058 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:06:24,846 - root - INFO - step: 43200 loss: 16.4108 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-09 23:06:24,847 - root - INFO - lr: 1.9599e-04 gnorm: 0.78 [ 9:31:25<12:31:19] +[titan] 2025-07-09 23:06:28,771 - root - INFO - step: 43205 loss: 16.7080 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1231 +[titan] 2025-07-09 23:06:28,771 - root - INFO - lr: 1.9597e-04 gnorm: 0.74 [ 9:31:29<12:31:15] +[titan] 2025-07-09 23:06:32,684 - root - INFO - step: 43210 loss: 16.7236 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.1476 +[titan] 2025-07-09 23:06:32,685 - root - INFO - lr: 1.9595e-04 gnorm: 0.83 [ 9:31:33<12:31:11] +[titan] 2025-07-09 23:06:36,595 - root - INFO - step: 43215 loss: 16.5997 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0345 +[titan] 2025-07-09 23:06:36,596 - root - INFO - lr: 1.9593e-04 gnorm: 0.79 [ 9:31:37<12:31:07] +[titan] 2025-07-09 23:06:40,520 - root - INFO - step: 43220 loss: 16.8197 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5997 global_avg_mtp_loss: 14.2200 +[titan] 2025-07-09 23:06:40,521 - root - INFO - lr: 1.9591e-04 gnorm: 0.75 [ 9:31:41<12:31:02] +[titan] 2025-07-09 23:06:44,433 - root - INFO - step: 43225 loss: 16.5175 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9828 +[titan] 2025-07-09 23:06:44,433 - root - INFO - lr: 1.9589e-04 gnorm: 0.78 [ 9:31:45<12:30:58] +[titan] 2025-07-09 23:06:48,339 - root - INFO - step: 43230 loss: 16.4687 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9221 +[titan] 2025-07-09 23:06:48,339 - root - INFO - lr: 1.9587e-04 gnorm: 0.77 [ 9:31:49<12:30:54] +[titan] 2025-07-09 23:06:52,230 - root - INFO - step: 43235 loss: 16.4932 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9635 +[titan] 2025-07-09 23:06:52,231 - root - INFO - lr: 1.9585e-04 gnorm: 0.77 [ 9:31:52<12:30:50] +[titan] 2025-07-09 23:06:56,134 - root - INFO - step: 43240 loss: 16.5027 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 13.9522 +[titan] 2025-07-09 23:06:56,134 - root - INFO - lr: 1.9582e-04 gnorm: 0.73 [ 9:31:56<12:30:46] +[titan] 2025-07-09 23:07:00,051 - root - INFO - step: 43245 loss: 16.6043 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0371 +[titan] 2025-07-09 23:07:00,051 - root - INFO - lr: 1.9580e-04 gnorm: 0.80 [ 9:32:00<12:30:42] +[titan] 2025-07-09 23:07:03,158 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:07:03,944 - root - INFO - step: 43250 loss: 16.6092 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0516 +[titan] 2025-07-09 23:07:03,944 - root - INFO - lr: 1.9578e-04 gnorm: 0.84 [ 9:32:04<12:30:38] +[titan] 2025-07-09 23:07:07,862 - root - INFO - step: 43255 loss: 16.2471 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7495 +[titan] 2025-07-09 23:07:07,862 - root - INFO - lr: 1.9576e-04 gnorm: 0.81 [ 9:32:08<12:30:34] +[titan] 2025-07-09 23:07:11,773 - root - INFO - step: 43260 loss: 16.3752 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-09 23:07:11,773 - root - INFO - lr: 1.9574e-04 gnorm: 0.88 [ 9:32:12<12:30:30] +[titan] 2025-07-09 23:07:15,691 - root - INFO - step: 43265 loss: 16.6112 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0430 +[titan] 2025-07-09 23:07:15,691 - root - INFO - lr: 1.9572e-04 gnorm: 0.81 [ 9:32:16<12:30:26] +[titan] 2025-07-09 23:07:19,617 - root - INFO - step: 43270 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9619 +[titan] 2025-07-09 23:07:19,618 - root - INFO - lr: 1.9570e-04 gnorm: 0.81 [ 9:32:20<12:30:22] +[titan] 2025-07-09 23:07:23,530 - root - INFO - step: 43275 loss: 16.5273 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9879 +[titan] 2025-07-09 23:07:23,531 - root - INFO - lr: 1.9568e-04 gnorm: 0.88 [ 9:32:24<12:30:18] +[titan] 2025-07-09 23:07:27,435 - root - INFO - step: 43280 loss: 16.4567 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9254 +[titan] 2025-07-09 23:07:27,435 - root - INFO - lr: 1.9566e-04 gnorm: 0.74 [ 9:32:28<12:30:14] +[titan] 2025-07-09 23:07:31,350 - root - INFO - step: 43285 loss: 16.5560 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 14.0070 +[titan] 2025-07-09 23:07:31,350 - root - INFO - lr: 1.9564e-04 gnorm: 0.74 [ 9:32:32<12:30:10] +[titan] 2025-07-09 23:07:35,247 - root - INFO - step: 43290 loss: 16.4403 memory: 44.58GiB(31.99%) tps: 84,084 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9001 +[titan] 2025-07-09 23:07:35,247 - root - INFO - lr: 1.9562e-04 gnorm: 0.79 [ 9:32:35<12:30:06] +[titan] 2025-07-09 23:07:39,178 - root - INFO - step: 43295 loss: 16.3752 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-09 23:07:39,178 - root - INFO - lr: 1.9560e-04 gnorm: 0.76 [ 9:32:39<12:30:02] +[titan] 2025-07-09 23:07:42,302 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:07:43,085 - root - INFO - step: 43300 loss: 16.6985 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5724 global_avg_mtp_loss: 14.1261 +[titan] 2025-07-09 23:07:43,086 - root - INFO - lr: 1.9557e-04 gnorm: 0.81 [ 9:32:43<12:29:58] +[titan] 2025-07-09 23:07:46,980 - root - INFO - step: 43305 loss: 16.7046 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5949 global_avg_mtp_loss: 14.1097 +[titan] 2025-07-09 23:07:46,980 - root - INFO - lr: 1.9555e-04 gnorm: 0.80 [ 9:32:47<12:29:54] +[titan] 2025-07-09 23:07:50,926 - root - INFO - step: 43310 loss: 16.5991 memory: 44.58GiB(31.99%) tps: 83,045 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.0228 +[titan] 2025-07-09 23:07:50,926 - root - INFO - lr: 1.9553e-04 gnorm: 0.85 [ 9:32:51<12:29:50] +[titan] 2025-07-09 23:07:54,845 - root - INFO - step: 43315 loss: 16.5605 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 14.0122 +[titan] 2025-07-09 23:07:54,845 - root - INFO - lr: 1.9551e-04 gnorm: 0.86 [ 9:32:55<12:29:46] +[titan] 2025-07-09 23:07:58,795 - root - INFO - step: 43320 loss: 16.5794 memory: 44.58GiB(31.99%) tps: 82,957 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 14.0317 +[titan] 2025-07-09 23:07:58,795 - root - INFO - lr: 1.9549e-04 gnorm: 0.81 [ 9:32:59<12:29:42] +[titan] 2025-07-09 23:08:02,691 - root - INFO - step: 43325 loss: 16.7435 memory: 44.58GiB(31.99%) tps: 84,109 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5931 global_avg_mtp_loss: 14.1504 +[titan] 2025-07-09 23:08:02,692 - root - INFO - lr: 1.9547e-04 gnorm: 0.76 [ 9:33:03<12:29:38] +[titan] 2025-07-09 23:08:06,599 - root - INFO - step: 43330 loss: 16.5530 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 13.9972 +[titan] 2025-07-09 23:08:06,599 - root - INFO - lr: 1.9545e-04 gnorm: 0.82 [ 9:33:07<12:29:34] +[titan] 2025-07-09 23:08:10,506 - root - INFO - step: 43335 loss: 16.5448 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 13.9963 +[titan] 2025-07-09 23:08:10,506 - root - INFO - lr: 1.9543e-04 gnorm: 0.77 [ 9:33:11<12:29:30] +[titan] 2025-07-09 23:08:14,408 - root - INFO - step: 43340 loss: 16.3996 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.8652 +[titan] 2025-07-09 23:08:14,408 - root - INFO - lr: 1.9541e-04 gnorm: 0.84 [ 9:33:15<12:29:25] +[titan] 2025-07-09 23:08:18,315 - root - INFO - step: 43345 loss: 16.5797 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5610 global_avg_mtp_loss: 14.0187 +[titan] 2025-07-09 23:08:18,315 - root - INFO - lr: 1.9539e-04 gnorm: 0.78 [ 9:33:18<12:29:21] +[titan] 2025-07-09 23:08:21,467 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:08:22,256 - root - INFO - step: 43350 loss: 16.2117 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.7237 +[titan] 2025-07-09 23:08:22,257 - root - INFO - lr: 1.9537e-04 gnorm: 0.75 [ 9:33:22<12:29:17] +[titan] 2025-07-09 23:08:26,167 - root - INFO - step: 43355 loss: 16.6340 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0657 +[titan] 2025-07-09 23:08:26,167 - root - INFO - lr: 1.9535e-04 gnorm: 0.78 [ 9:33:26<12:29:13] +[titan] 2025-07-09 23:08:30,157 - root - INFO - step: 43360 loss: 16.6281 memory: 44.58GiB(31.99%) tps: 82,128 tflops: 283.44 mfu: 28.66% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0541 +[titan] 2025-07-09 23:08:30,157 - root - INFO - lr: 1.9532e-04 gnorm: 0.77 [ 9:33:30<12:29:09] +[titan] 2025-07-09 23:08:34,067 - root - INFO - step: 43365 loss: 16.7574 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.6004 global_avg_mtp_loss: 14.1570 +[titan] 2025-07-09 23:08:34,068 - root - INFO - lr: 1.9530e-04 gnorm: 0.77 [ 9:33:34<12:29:05] +[titan] 2025-07-09 23:08:37,977 - root - INFO - step: 43370 loss: 16.7570 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1679 +[titan] 2025-07-09 23:08:37,977 - root - INFO - lr: 1.9528e-04 gnorm: 0.79 [ 9:33:38<12:29:01] +[titan] 2025-07-09 23:08:41,907 - root - INFO - step: 43375 loss: 16.7523 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5826 global_avg_mtp_loss: 14.1697 +[titan] 2025-07-09 23:08:41,907 - root - INFO - lr: 1.9526e-04 gnorm: 0.80 [ 9:33:42<12:28:57] +[titan] 2025-07-09 23:08:45,801 - root - INFO - step: 43380 loss: 16.8217 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2081 +[titan] 2025-07-09 23:08:45,801 - root - INFO - lr: 1.9524e-04 gnorm: 0.81 [ 9:33:46<12:28:53] +[titan] 2025-07-09 23:08:49,715 - root - INFO - step: 43385 loss: 16.5899 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 14.0151 +[titan] 2025-07-09 23:08:49,715 - root - INFO - lr: 1.9522e-04 gnorm: 0.82 [ 9:33:50<12:28:49] +[titan] 2025-07-09 23:08:53,679 - root - INFO - step: 43390 loss: 16.5870 memory: 44.58GiB(31.99%) tps: 82,670 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0257 +[titan] 2025-07-09 23:08:53,680 - root - INFO - lr: 1.9520e-04 gnorm: 0.78 [ 9:33:54<12:28:45] +[titan] 2025-07-09 23:08:57,599 - root - INFO - step: 43395 loss: 16.4266 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9079 +[titan] 2025-07-09 23:08:57,599 - root - INFO - lr: 1.9518e-04 gnorm: 0.82 [ 9:33:58<12:28:41] +[titan] 2025-07-09 23:09:00,734 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:09:01,518 - root - INFO - step: 43400 loss: 16.6675 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0976 +[titan] 2025-07-09 23:09:01,519 - root - INFO - lr: 1.9516e-04 gnorm: 0.77 [ 9:34:02<12:28:37] +[titan] 2025-07-09 23:09:05,453 - root - INFO - step: 43405 loss: 16.7210 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.1449 +[titan] 2025-07-09 23:09:05,453 - root - INFO - lr: 1.9514e-04 gnorm: 0.79 [ 9:34:06<12:28:33] +[titan] 2025-07-09 23:09:09,375 - root - INFO - step: 43410 loss: 16.6126 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-09 23:09:09,375 - root - INFO - lr: 1.9512e-04 gnorm: 0.79 [ 9:34:10<12:28:29] +[titan] 2025-07-09 23:09:13,288 - root - INFO - step: 43415 loss: 16.6658 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0982 +[titan] 2025-07-09 23:09:13,288 - root - INFO - lr: 1.9509e-04 gnorm: 0.77 [ 9:34:13<12:28:25] +[titan] 2025-07-09 23:09:17,196 - root - INFO - step: 43420 loss: 16.6116 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0414 +[titan] 2025-07-09 23:09:17,196 - root - INFO - lr: 1.9507e-04 gnorm: 0.79 [ 9:34:17<12:28:21] +[titan] 2025-07-09 23:09:21,084 - root - INFO - step: 43425 loss: 16.4605 memory: 44.58GiB(31.99%) tps: 84,290 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9279 +[titan] 2025-07-09 23:09:21,084 - root - INFO - lr: 1.9505e-04 gnorm: 0.80 [ 9:34:21<12:28:17] +[titan] 2025-07-09 23:09:25,008 - root - INFO - step: 43430 loss: 16.8700 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6113 global_avg_mtp_loss: 14.2586 +[titan] 2025-07-09 23:09:25,008 - root - INFO - lr: 1.9503e-04 gnorm: 0.77 [ 9:34:25<12:28:13] +[titan] 2025-07-09 23:09:28,974 - root - INFO - step: 43435 loss: 16.4958 memory: 44.58GiB(31.99%) tps: 82,636 tflops: 285.19 mfu: 28.84% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9639 +[titan] 2025-07-09 23:09:28,974 - root - INFO - lr: 1.9501e-04 gnorm: 0.74 [ 9:34:29<12:28:09] +[titan] 2025-07-09 23:09:32,859 - root - INFO - step: 43440 loss: 16.6936 memory: 44.58GiB(31.99%) tps: 84,357 tflops: 291.13 mfu: 29.44% global_avg_ntp_loss: 2.5823 global_avg_mtp_loss: 14.1113 +[titan] 2025-07-09 23:09:32,859 - root - INFO - lr: 1.9499e-04 gnorm: 0.77 [ 9:34:33<12:28:05] +[titan] 2025-07-09 23:09:36,780 - root - INFO - step: 43445 loss: 16.6136 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.0419 +[titan] 2025-07-09 23:09:36,781 - root - INFO - lr: 1.9497e-04 gnorm: 0.75 [ 9:34:37<12:28:01] +[titan] 2025-07-09 23:09:39,891 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:09:40,682 - root - INFO - step: 43450 loss: 16.3256 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8178 +[titan] 2025-07-09 23:09:40,683 - root - INFO - lr: 1.9495e-04 gnorm: 0.75 [ 9:34:41<12:27:57] +[titan] 2025-07-09 23:09:44,596 - root - INFO - step: 43455 loss: 16.5173 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9733 +[titan] 2025-07-09 23:09:44,596 - root - INFO - lr: 1.9493e-04 gnorm: 0.76 [ 9:34:45<12:27:53] +[titan] 2025-07-09 23:09:48,543 - root - INFO - step: 43460 loss: 16.4269 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.9115 +[titan] 2025-07-09 23:09:48,543 - root - INFO - lr: 1.9491e-04 gnorm: 0.81 [ 9:34:49<12:27:49] +[titan] 2025-07-09 23:09:52,483 - root - INFO - step: 43465 loss: 16.6524 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0874 +[titan] 2025-07-09 23:09:52,483 - root - INFO - lr: 1.9489e-04 gnorm: 0.84 [ 9:34:53<12:27:45] +[titan] 2025-07-09 23:09:56,409 - root - INFO - step: 43470 loss: 16.8210 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.2203 +[titan] 2025-07-09 23:09:56,410 - root - INFO - lr: 1.9486e-04 gnorm: 0.77 [ 9:34:57<12:27:41] +[titan] 2025-07-09 23:10:00,318 - root - INFO - step: 43475 loss: 16.7277 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5860 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 23:10:00,318 - root - INFO - lr: 1.9484e-04 gnorm: 0.79 [ 9:35:00<12:27:37] +[titan] 2025-07-09 23:10:04,219 - root - INFO - step: 43480 loss: 16.6950 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.1129 +[titan] 2025-07-09 23:10:04,219 - root - INFO - lr: 1.9482e-04 gnorm: 0.79 [ 9:35:04<12:27:33] +[titan] 2025-07-09 23:10:08,113 - root - INFO - step: 43485 loss: 16.7259 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5825 global_avg_mtp_loss: 14.1434 +[titan] 2025-07-09 23:10:08,113 - root - INFO - lr: 1.9480e-04 gnorm: 0.80 [ 9:35:08<12:27:29] +[titan] 2025-07-09 23:10:12,017 - root - INFO - step: 43490 loss: 16.3452 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.8247 +[titan] 2025-07-09 23:10:12,017 - root - INFO - lr: 1.9478e-04 gnorm: 0.82 [ 9:35:12<12:27:25] +[titan] 2025-07-09 23:10:15,945 - root - INFO - step: 43495 loss: 16.7483 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5807 global_avg_mtp_loss: 14.1676 +[titan] 2025-07-09 23:10:15,946 - root - INFO - lr: 1.9476e-04 gnorm: 0.79 [ 9:35:16<12:27:21] +[titan] 2025-07-09 23:10:19,121 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:10:19,915 - root - INFO - step: 43500 loss: 16.5815 memory: 44.58GiB(31.99%) tps: 82,561 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0202 +[titan] 2025-07-09 23:10:19,915 - root - INFO - lr: 1.9474e-04 gnorm: 0.84 [ 9:35:20<12:27:17] +[titan] 2025-07-09 23:10:23,853 - root - INFO - step: 43505 loss: 16.4731 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9414 +[titan] 2025-07-09 23:10:23,854 - root - INFO - lr: 1.9472e-04 gnorm: 0.81 [ 9:35:24<12:27:13] +[titan] 2025-07-09 23:10:27,792 - root - INFO - step: 43510 loss: 16.5944 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0316 +[titan] 2025-07-09 23:10:27,793 - root - INFO - lr: 1.9470e-04 gnorm: 0.74 [ 9:35:28<12:27:09] +[titan] 2025-07-09 23:10:31,755 - root - INFO - step: 43515 loss: 16.5043 memory: 44.58GiB(31.99%) tps: 82,711 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 13.9535 +[titan] 2025-07-09 23:10:31,755 - root - INFO - lr: 1.9468e-04 gnorm: 0.77 [ 9:35:32<12:27:05] +[titan] 2025-07-09 23:10:35,804 - root - INFO - step: 43520 loss: 16.6220 memory: 44.58GiB(31.99%) tps: 80,919 tflops: 279.26 mfu: 28.24% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0480 +[titan] 2025-07-09 23:10:35,805 - root - INFO - lr: 1.9466e-04 gnorm: 0.87 [ 9:35:36<12:27:01] +[titan] 2025-07-09 23:10:35,959 - root - INFO - Dumping profiler traces at step 43520 +[titan] 2025-07-09 23:10:35,991 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:10:39,925 - root - INFO - step: 43525 loss: 16.4472 memory: 44.58GiB(31.99%) tps: 79,526 tflops: 274.46 mfu: 27.75% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9104 +[titan] 2025-07-09 23:10:39,925 - root - INFO - lr: 1.9464e-04 gnorm: 0.77 [ 9:35:40<12:26:57] +[titan] 2025-07-09 23:10:43,840 - root - INFO - step: 43530 loss: 16.6325 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0743 +[titan] 2025-07-09 23:10:43,841 - root - INFO - lr: 1.9461e-04 gnorm: 0.77 [ 9:35:44<12:26:53] +[titan] 2025-07-09 23:10:47,781 - root - INFO - step: 43535 loss: 16.5379 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9974 +[titan] 2025-07-09 23:10:47,781 - root - INFO - lr: 1.9459e-04 gnorm: 0.82 [ 9:35:48<12:26:49] +[titan] 2025-07-09 23:10:51,689 - root - INFO - step: 43540 loss: 16.6553 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 14.1037 +[titan] 2025-07-09 23:10:51,689 - root - INFO - lr: 1.9457e-04 gnorm: 0.79 [ 9:35:52<12:26:45] +[titan] 2025-07-09 23:10:55,576 - root - INFO - step: 43545 loss: 16.6944 memory: 44.58GiB(31.99%) tps: 84,308 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.1353 +[titan] 2025-07-09 23:10:55,576 - root - INFO - lr: 1.9455e-04 gnorm: 0.80 [ 9:35:56<12:26:41] +[titan] 2025-07-09 23:10:58,689 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:10:59,477 - root - INFO - step: 43550 loss: 16.6184 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0552 +[titan] 2025-07-09 23:10:59,478 - root - INFO - lr: 1.9453e-04 gnorm: 0.76 [ 9:36:00<12:26:37] +[titan] 2025-07-09 23:11:03,423 - root - INFO - step: 43555 loss: 16.5580 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-09 23:11:03,423 - root - INFO - lr: 1.9451e-04 gnorm: 0.80 [ 9:36:04<12:26:33] +[titan] 2025-07-09 23:11:07,349 - root - INFO - step: 43560 loss: 16.4973 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9563 +[titan] 2025-07-09 23:11:07,349 - root - INFO - lr: 1.9449e-04 gnorm: 0.79 [ 9:36:07<12:26:29] +[titan] 2025-07-09 23:11:11,283 - root - INFO - step: 43565 loss: 16.5028 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9685 +[titan] 2025-07-09 23:11:11,283 - root - INFO - lr: 1.9447e-04 gnorm: 0.76 [ 9:36:11<12:26:25] +[titan] 2025-07-09 23:11:15,212 - root - INFO - step: 43570 loss: 16.7973 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.1862 +[titan] 2025-07-09 23:11:15,212 - root - INFO - lr: 1.9445e-04 gnorm: 0.80 [ 9:36:15<12:26:21] +[titan] 2025-07-09 23:11:19,160 - root - INFO - step: 43575 loss: 16.6619 memory: 44.58GiB(31.99%) tps: 83,007 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0942 +[titan] 2025-07-09 23:11:19,160 - root - INFO - lr: 1.9443e-04 gnorm: 0.78 [ 9:36:19<12:26:17] +[titan] 2025-07-09 23:11:23,090 - root - INFO - step: 43580 loss: 16.8815 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6150 global_avg_mtp_loss: 14.2665 +[titan] 2025-07-09 23:11:23,090 - root - INFO - lr: 1.9441e-04 gnorm: 0.79 [ 9:36:23<12:26:13] +[titan] 2025-07-09 23:11:27,078 - root - INFO - step: 43585 loss: 16.7700 memory: 44.58GiB(31.99%) tps: 82,155 tflops: 283.53 mfu: 28.67% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1782 +[titan] 2025-07-09 23:11:27,079 - root - INFO - lr: 1.9438e-04 gnorm: 0.85 [ 9:36:27<12:26:09] +[titan] 2025-07-09 23:11:30,991 - root - INFO - step: 43590 loss: 16.8542 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.6152 global_avg_mtp_loss: 14.2390 +[titan] 2025-07-09 23:11:30,991 - root - INFO - lr: 1.9436e-04 gnorm: 0.80 [ 9:36:31<12:26:05] +[titan] 2025-07-09 23:11:34,908 - root - INFO - step: 43595 loss: 16.5524 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 14.0071 +[titan] 2025-07-09 23:11:34,908 - root - INFO - lr: 1.9434e-04 gnorm: 0.76 [ 9:36:35<12:26:01] +[titan] 2025-07-09 23:11:38,028 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:11:38,818 - root - INFO - step: 43600 loss: 16.5269 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 13.9732 +[titan] 2025-07-09 23:11:38,818 - root - INFO - lr: 1.9432e-04 gnorm: 0.74 [ 9:36:39<12:25:57] +[titan] 2025-07-09 23:11:42,762 - root - INFO - step: 43605 loss: 16.4847 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 13.9323 +[titan] 2025-07-09 23:11:42,762 - root - INFO - lr: 1.9430e-04 gnorm: 0.77 [ 9:36:43<12:25:53] +[titan] 2025-07-09 23:11:46,700 - root - INFO - step: 43610 loss: 16.5015 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9743 +[titan] 2025-07-09 23:11:46,700 - root - INFO - lr: 1.9428e-04 gnorm: 0.79 [ 9:36:47<12:25:49] +[titan] 2025-07-09 23:11:50,639 - root - INFO - step: 43615 loss: 16.6344 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5661 global_avg_mtp_loss: 14.0683 +[titan] 2025-07-09 23:11:50,639 - root - INFO - lr: 1.9426e-04 gnorm: 0.79 [ 9:36:51<12:25:45] +[titan] 2025-07-09 23:11:54,560 - root - INFO - step: 43620 loss: 16.8356 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6071 global_avg_mtp_loss: 14.2285 +[titan] 2025-07-09 23:11:54,560 - root - INFO - lr: 1.9424e-04 gnorm: 0.76 [ 9:36:55<12:25:41] +[titan] 2025-07-09 23:11:58,485 - root - INFO - step: 43625 loss: 16.5011 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9745 +[titan] 2025-07-09 23:11:58,485 - root - INFO - lr: 1.9422e-04 gnorm: 0.81 [ 9:36:59<12:25:37] +[titan] 2025-07-09 23:12:02,388 - root - INFO - step: 43630 loss: 16.5496 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 14.0027 +[titan] 2025-07-09 23:12:02,388 - root - INFO - lr: 1.9420e-04 gnorm: 0.78 [ 9:37:03<12:25:32] +[titan] 2025-07-09 23:12:06,279 - root - INFO - step: 43635 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0841 +[titan] 2025-07-09 23:12:06,279 - root - INFO - lr: 1.9418e-04 gnorm: 0.75 [ 9:37:06<12:25:28] +[titan] 2025-07-09 23:12:10,235 - root - INFO - step: 43640 loss: 16.4667 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9343 +[titan] 2025-07-09 23:12:10,235 - root - INFO - lr: 1.9415e-04 gnorm: 0.76 [ 9:37:10<12:25:24] +[titan] 2025-07-09 23:12:14,138 - root - INFO - step: 43645 loss: 16.4804 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9405 +[titan] 2025-07-09 23:12:14,138 - root - INFO - lr: 1.9413e-04 gnorm: 0.76 [ 9:37:14<12:25:20] +[titan] 2025-07-09 23:12:17,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:12:18,039 - root - INFO - step: 43650 loss: 16.6993 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5696 global_avg_mtp_loss: 14.1297 +[titan] 2025-07-09 23:12:18,040 - root - INFO - lr: 1.9411e-04 gnorm: 0.77 [ 9:37:18<12:25:16] +[titan] 2025-07-09 23:12:21,958 - root - INFO - step: 43655 loss: 16.4617 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9412 +[titan] 2025-07-09 23:12:21,958 - root - INFO - lr: 1.9409e-04 gnorm: 0.94 [ 9:37:22<12:25:12] +[titan] 2025-07-09 23:12:25,897 - root - INFO - step: 43660 loss: 16.6469 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.0561 +[titan] 2025-07-09 23:12:25,897 - root - INFO - lr: 1.9407e-04 gnorm: 0.82 [ 9:37:26<12:25:08] +[titan] 2025-07-09 23:12:29,845 - root - INFO - step: 43665 loss: 16.5198 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9814 +[titan] 2025-07-09 23:12:29,845 - root - INFO - lr: 1.9405e-04 gnorm: 0.83 [ 9:37:30<12:25:04] +[titan] 2025-07-09 23:12:33,756 - root - INFO - step: 43670 loss: 16.6205 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.0446 +[titan] 2025-07-09 23:12:33,757 - root - INFO - lr: 1.9403e-04 gnorm: 0.78 [ 9:37:34<12:25:00] +[titan] 2025-07-09 23:12:37,666 - root - INFO - step: 43675 loss: 16.5600 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0023 +[titan] 2025-07-09 23:12:37,666 - root - INFO - lr: 1.9401e-04 gnorm: 0.83 [ 9:37:38<12:24:56] +[titan] 2025-07-09 23:12:41,606 - root - INFO - step: 43680 loss: 16.4640 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 13.9220 +[titan] 2025-07-09 23:12:41,606 - root - INFO - lr: 1.9399e-04 gnorm: 0.80 [ 9:37:42<12:24:52] +[titan] 2025-07-09 23:12:45,517 - root - INFO - step: 43685 loss: 16.5546 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0026 +[titan] 2025-07-09 23:12:45,518 - root - INFO - lr: 1.9397e-04 gnorm: 0.76 [ 9:37:46<12:24:48] +[titan] 2025-07-09 23:12:49,435 - root - INFO - step: 43690 loss: 16.7339 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 14.1502 +[titan] 2025-07-09 23:12:49,435 - root - INFO - lr: 1.9395e-04 gnorm: 0.78 [ 9:37:50<12:24:44] +[titan] 2025-07-09 23:12:53,379 - root - INFO - step: 43695 loss: 16.8852 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.6136 global_avg_mtp_loss: 14.2716 +[titan] 2025-07-09 23:12:53,379 - root - INFO - lr: 1.9392e-04 gnorm: 0.80 [ 9:37:54<12:24:40] +[titan] 2025-07-09 23:12:56,549 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:12:57,342 - root - INFO - step: 43700 loss: 16.3513 memory: 44.58GiB(31.99%) tps: 82,692 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.8312 +[titan] 2025-07-09 23:12:57,342 - root - INFO - lr: 1.9390e-04 gnorm: 0.78 [ 9:37:57<12:24:36] +[titan] 2025-07-09 23:13:01,232 - root - INFO - step: 43705 loss: 16.4228 memory: 44.58GiB(31.99%) tps: 84,250 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8989 +[titan] 2025-07-09 23:13:01,232 - root - INFO - lr: 1.9388e-04 gnorm: 0.77 [ 9:38:01<12:24:32] +[titan] 2025-07-09 23:13:05,167 - root - INFO - step: 43710 loss: 16.5683 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0023 +[titan] 2025-07-09 23:13:05,167 - root - INFO - lr: 1.9386e-04 gnorm: 0.78 [ 9:38:05<12:24:28] +[titan] 2025-07-09 23:13:09,068 - root - INFO - step: 43715 loss: 16.8589 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6050 global_avg_mtp_loss: 14.2539 +[titan] 2025-07-09 23:13:09,068 - root - INFO - lr: 1.9384e-04 gnorm: 0.79 [ 9:38:09<12:24:24] +[titan] 2025-07-09 23:13:12,968 - root - INFO - step: 43720 loss: 16.4927 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9485 +[titan] 2025-07-09 23:13:12,968 - root - INFO - lr: 1.9382e-04 gnorm: 0.76 [ 9:38:13<12:24:20] +[titan] 2025-07-09 23:13:16,896 - root - INFO - step: 43725 loss: 16.3198 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8127 +[titan] 2025-07-09 23:13:16,896 - root - INFO - lr: 1.9380e-04 gnorm: 0.78 [ 9:38:17<12:24:16] +[titan] 2025-07-09 23:13:20,799 - root - INFO - step: 43730 loss: 16.6860 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1040 +[titan] 2025-07-09 23:13:20,800 - root - INFO - lr: 1.9378e-04 gnorm: 0.77 [ 9:38:21<12:24:12] +[titan] 2025-07-09 23:13:24,738 - root - INFO - step: 43735 loss: 16.8818 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.6101 global_avg_mtp_loss: 14.2717 +[titan] 2025-07-09 23:13:24,739 - root - INFO - lr: 1.9376e-04 gnorm: 0.78 [ 9:38:25<12:24:08] +[titan] 2025-07-09 23:13:28,670 - root - INFO - step: 43740 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 14.1244 +[titan] 2025-07-09 23:13:28,670 - root - INFO - lr: 1.9374e-04 gnorm: 0.76 [ 9:38:29<12:24:04] +[titan] 2025-07-09 23:13:32,588 - root - INFO - step: 43745 loss: 16.4756 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9328 +[titan] 2025-07-09 23:13:32,588 - root - INFO - lr: 1.9371e-04 gnorm: 0.78 [ 9:38:33<12:24:00] +[titan] 2025-07-09 23:13:35,699 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:13:36,489 - root - INFO - step: 43750 loss: 16.4547 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9163 +[titan] 2025-07-09 23:13:36,489 - root - INFO - lr: 1.9369e-04 gnorm: 0.78 [ 9:38:37<12:23:56] +[titan] 2025-07-09 23:13:40,434 - root - INFO - step: 43755 loss: 16.6556 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0890 +[titan] 2025-07-09 23:13:40,434 - root - INFO - lr: 1.9367e-04 gnorm: 0.74 [ 9:38:41<12:23:52] +[titan] 2025-07-09 23:13:44,373 - root - INFO - step: 43760 loss: 16.4121 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.8866 +[titan] 2025-07-09 23:13:44,373 - root - INFO - lr: 1.9365e-04 gnorm: 0.76 [ 9:38:44<12:23:48] +[titan] 2025-07-09 23:13:48,285 - root - INFO - step: 43765 loss: 16.3152 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8008 +[titan] 2025-07-09 23:13:48,285 - root - INFO - lr: 1.9363e-04 gnorm: 0.79 [ 9:38:48<12:23:44] +[titan] 2025-07-09 23:13:52,180 - root - INFO - step: 43770 loss: 16.5288 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9836 +[titan] 2025-07-09 23:13:52,180 - root - INFO - lr: 1.9361e-04 gnorm: 0.80 [ 9:38:52<12:23:40] +[titan] 2025-07-09 23:13:56,098 - root - INFO - step: 43775 loss: 16.7035 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.1256 +[titan] 2025-07-09 23:13:56,098 - root - INFO - lr: 1.9359e-04 gnorm: 0.81 [ 9:38:56<12:23:36] +[titan] 2025-07-09 23:14:00,026 - root - INFO - step: 43780 loss: 16.4736 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.9354 +[titan] 2025-07-09 23:14:00,026 - root - INFO - lr: 1.9357e-04 gnorm: 0.73 [ 9:39:00<12:23:32] +[titan] 2025-07-09 23:14:03,921 - root - INFO - step: 43785 loss: 16.6527 memory: 44.58GiB(31.99%) tps: 84,143 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0805 +[titan] 2025-07-09 23:14:03,921 - root - INFO - lr: 1.9355e-04 gnorm: 0.87 [ 9:39:04<12:23:28] +[titan] 2025-07-09 23:14:07,842 - root - INFO - step: 43790 loss: 16.6591 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.1001 +[titan] 2025-07-09 23:14:07,842 - root - INFO - lr: 1.9353e-04 gnorm: 0.79 [ 9:39:08<12:23:24] +[titan] 2025-07-09 23:14:11,733 - root - INFO - step: 43795 loss: 16.9132 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.6159 global_avg_mtp_loss: 14.2974 +[titan] 2025-07-09 23:14:11,733 - root - INFO - lr: 1.9351e-04 gnorm: 0.80 [ 9:39:12<12:23:19] +[titan] 2025-07-09 23:14:14,858 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:14:15,649 - root - INFO - step: 43800 loss: 16.5405 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9927 +[titan] 2025-07-09 23:14:15,650 - root - INFO - lr: 1.9348e-04 gnorm: 0.76 [ 9:39:16<12:23:15] +[titan] 2025-07-09 23:14:19,551 - root - INFO - step: 43805 loss: 16.5324 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 13.9771 +[titan] 2025-07-09 23:14:19,551 - root - INFO - lr: 1.9346e-04 gnorm: 0.76 [ 9:39:20<12:23:11] +[titan] 2025-07-09 23:14:23,458 - root - INFO - step: 43810 loss: 16.6157 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0485 +[titan] 2025-07-09 23:14:23,459 - root - INFO - lr: 1.9344e-04 gnorm: 0.78 [ 9:39:24<12:23:07] +[titan] 2025-07-09 23:14:27,386 - root - INFO - step: 43815 loss: 16.6228 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0632 +[titan] 2025-07-09 23:14:27,386 - root - INFO - lr: 1.9342e-04 gnorm: 0.82 [ 9:39:28<12:23:03] +[titan] 2025-07-09 23:14:31,286 - root - INFO - step: 43820 loss: 16.5937 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0362 +[titan] 2025-07-09 23:14:31,286 - root - INFO - lr: 1.9340e-04 gnorm: 0.80 [ 9:39:31<12:22:59] +[titan] 2025-07-09 23:14:35,195 - root - INFO - step: 43825 loss: 16.8030 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.6025 global_avg_mtp_loss: 14.2006 +[titan] 2025-07-09 23:14:35,196 - root - INFO - lr: 1.9338e-04 gnorm: 0.75 [ 9:39:35<12:22:55] +[titan] 2025-07-09 23:14:39,096 - root - INFO - step: 43830 loss: 16.6398 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.0642 +[titan] 2025-07-09 23:14:39,097 - root - INFO - lr: 1.9336e-04 gnorm: 0.77 [ 9:39:39<12:22:51] +[titan] 2025-07-09 23:14:42,993 - root - INFO - step: 43835 loss: 16.6740 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.0964 +[titan] 2025-07-09 23:14:42,993 - root - INFO - lr: 1.9334e-04 gnorm: 0.84 [ 9:39:43<12:22:47] +[titan] 2025-07-09 23:14:46,900 - root - INFO - step: 43840 loss: 16.4808 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9544 +[titan] 2025-07-09 23:14:46,900 - root - INFO - lr: 1.9332e-04 gnorm: 0.77 [ 9:39:47<12:22:43] +[titan] 2025-07-09 23:14:50,803 - root - INFO - step: 43845 loss: 16.9143 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2951 +[titan] 2025-07-09 23:14:50,804 - root - INFO - lr: 1.9330e-04 gnorm: 0.84 [ 9:39:51<12:22:39] +[titan] 2025-07-09 23:14:53,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:14:54,698 - root - INFO - step: 43850 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9336 +[titan] 2025-07-09 23:14:54,699 - root - INFO - lr: 1.9328e-04 gnorm: 0.76 [ 9:39:55<12:22:35] +[titan] 2025-07-09 23:14:58,599 - root - INFO - step: 43855 loss: 16.7239 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5945 global_avg_mtp_loss: 14.1293 +[titan] 2025-07-09 23:14:58,600 - root - INFO - lr: 1.9325e-04 gnorm: 0.82 [ 9:39:59<12:22:31] +[titan] 2025-07-09 23:15:02,501 - root - INFO - step: 43860 loss: 16.7473 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.1674 +[titan] 2025-07-09 23:15:02,501 - root - INFO - lr: 1.9323e-04 gnorm: 0.79 [ 9:40:03<12:22:27] +[titan] 2025-07-09 23:15:06,409 - root - INFO - step: 43865 loss: 16.6462 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0769 +[titan] 2025-07-09 23:15:06,409 - root - INFO - lr: 1.9321e-04 gnorm: 0.84 [ 9:40:07<12:22:23] +[titan] 2025-07-09 23:15:10,300 - root - INFO - step: 43870 loss: 16.3660 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-09 23:15:10,300 - root - INFO - lr: 1.9319e-04 gnorm: 0.80 [ 9:40:10<12:22:19] +[titan] 2025-07-09 23:15:14,204 - root - INFO - step: 43875 loss: 16.4814 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 13.9263 +[titan] 2025-07-09 23:15:14,205 - root - INFO - lr: 1.9317e-04 gnorm: 1.36 [ 9:40:14<12:22:15] +[titan] 2025-07-09 23:15:18,106 - root - INFO - step: 43880 loss: 16.4737 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9340 +[titan] 2025-07-09 23:15:18,107 - root - INFO - lr: 1.9315e-04 gnorm: 0.87 [ 9:40:18<12:22:11] +[titan] 2025-07-09 23:15:22,009 - root - INFO - step: 43885 loss: 16.5073 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9684 +[titan] 2025-07-09 23:15:22,010 - root - INFO - lr: 1.9313e-04 gnorm: 0.87 [ 9:40:22<12:22:07] +[titan] 2025-07-09 23:15:25,909 - root - INFO - step: 43890 loss: 16.7686 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-09 23:15:25,909 - root - INFO - lr: 1.9311e-04 gnorm: 0.76 [ 9:40:26<12:22:03] +[titan] 2025-07-09 23:15:29,808 - root - INFO - step: 43895 loss: 16.4958 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9496 +[titan] 2025-07-09 23:15:29,808 - root - INFO - lr: 1.9309e-04 gnorm: 0.82 [ 9:40:30<12:21:58] +[titan] 2025-07-09 23:15:32,918 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:15:33,706 - root - INFO - step: 43900 loss: 16.6501 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0847 +[titan] 2025-07-09 23:15:33,706 - root - INFO - lr: 1.9307e-04 gnorm: 0.80 [ 9:40:34<12:21:54] +[titan] 2025-07-09 23:15:37,623 - root - INFO - step: 43905 loss: 16.5884 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0291 +[titan] 2025-07-09 23:15:37,623 - root - INFO - lr: 1.9304e-04 gnorm: 0.81 [ 9:40:38<12:21:50] +[titan] 2025-07-09 23:15:41,527 - root - INFO - step: 43910 loss: 16.5625 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 14.0150 +[titan] 2025-07-09 23:15:41,528 - root - INFO - lr: 1.9302e-04 gnorm: 0.78 [ 9:40:42<12:21:46] +[titan] 2025-07-09 23:15:45,422 - root - INFO - step: 43915 loss: 16.5094 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9647 +[titan] 2025-07-09 23:15:45,422 - root - INFO - lr: 1.9300e-04 gnorm: 0.82 [ 9:40:46<12:21:42] +[titan] 2025-07-09 23:15:49,328 - root - INFO - step: 43920 loss: 16.5533 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 14.0066 +[titan] 2025-07-09 23:15:49,328 - root - INFO - lr: 1.9298e-04 gnorm: 0.80 [ 9:40:49<12:21:38] +[titan] 2025-07-09 23:15:53,249 - root - INFO - step: 43925 loss: 16.4764 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9505 +[titan] 2025-07-09 23:15:53,249 - root - INFO - lr: 1.9296e-04 gnorm: 0.82 [ 9:40:53<12:21:34] +[titan] 2025-07-09 23:15:57,172 - root - INFO - step: 43930 loss: 16.4173 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.8863 +[titan] 2025-07-09 23:15:57,172 - root - INFO - lr: 1.9294e-04 gnorm: 0.82 [ 9:40:57<12:21:30] +[titan] 2025-07-09 23:16:01,104 - root - INFO - step: 43935 loss: 16.7269 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1434 +[titan] 2025-07-09 23:16:01,104 - root - INFO - lr: 1.9292e-04 gnorm: 0.76 [ 9:41:01<12:21:26] +[titan] 2025-07-09 23:16:05,028 - root - INFO - step: 43940 loss: 16.8299 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.6034 global_avg_mtp_loss: 14.2265 +[titan] 2025-07-09 23:16:05,029 - root - INFO - lr: 1.9290e-04 gnorm: 0.78 [ 9:41:05<12:21:22] +[titan] 2025-07-09 23:16:08,997 - root - INFO - step: 43945 loss: 16.8148 memory: 44.58GiB(31.99%) tps: 82,571 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.2204 +[titan] 2025-07-09 23:16:08,998 - root - INFO - lr: 1.9288e-04 gnorm: 0.77 [ 9:41:09<12:21:18] +[titan] 2025-07-09 23:16:12,162 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:16:12,961 - root - INFO - step: 43950 loss: 16.3876 memory: 44.58GiB(31.99%) tps: 82,670 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8686 +[titan] 2025-07-09 23:16:12,962 - root - INFO - lr: 1.9286e-04 gnorm: 0.76 [ 9:41:13<12:21:14] +[titan] 2025-07-09 23:16:16,883 - root - INFO - step: 43955 loss: 16.5426 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 13.9890 +[titan] 2025-07-09 23:16:16,884 - root - INFO - lr: 1.9284e-04 gnorm: 0.79 [ 9:41:17<12:21:10] +[titan] 2025-07-09 23:16:20,792 - root - INFO - step: 43960 loss: 16.5228 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 13.9791 +[titan] 2025-07-09 23:16:20,792 - root - INFO - lr: 1.9281e-04 gnorm: 0.78 [ 9:41:21<12:21:06] +[titan] 2025-07-09 23:16:24,734 - root - INFO - step: 43965 loss: 16.5907 memory: 44.58GiB(31.99%) tps: 83,134 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0315 +[titan] 2025-07-09 23:16:24,734 - root - INFO - lr: 1.9279e-04 gnorm: 0.76 [ 9:41:25<12:21:02] +[titan] 2025-07-09 23:16:28,649 - root - INFO - step: 43970 loss: 16.5653 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0119 +[titan] 2025-07-09 23:16:28,649 - root - INFO - lr: 1.9277e-04 gnorm: 0.75 [ 9:41:29<12:20:58] +[titan] 2025-07-09 23:16:32,562 - root - INFO - step: 43975 loss: 16.7088 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5856 global_avg_mtp_loss: 14.1232 +[titan] 2025-07-09 23:16:32,562 - root - INFO - lr: 1.9275e-04 gnorm: 0.81 [ 9:41:33<12:20:54] +[titan] 2025-07-09 23:16:36,494 - root - INFO - step: 43980 loss: 16.4195 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5431 global_avg_mtp_loss: 13.8764 +[titan] 2025-07-09 23:16:36,494 - root - INFO - lr: 1.9273e-04 gnorm: 0.80 [ 9:41:37<12:20:50] +[titan] 2025-07-09 23:16:40,392 - root - INFO - step: 43985 loss: 16.6950 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.1243 +[titan] 2025-07-09 23:16:40,392 - root - INFO - lr: 1.9271e-04 gnorm: 0.77 [ 9:41:40<12:20:46] +[titan] 2025-07-09 23:16:44,313 - root - INFO - step: 43990 loss: 16.4889 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5499 global_avg_mtp_loss: 13.9390 +[titan] 2025-07-09 23:16:44,313 - root - INFO - lr: 1.9269e-04 gnorm: 0.76 [ 9:41:44<12:20:42] +[titan] 2025-07-09 23:16:48,225 - root - INFO - step: 43995 loss: 16.3578 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-09 23:16:48,225 - root - INFO - lr: 1.9267e-04 gnorm: 0.79 [ 9:41:48<12:20:38] +[titan] 2025-07-09 23:16:51,402 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:16:52,192 - root - INFO - step: 44000 loss: 16.4788 memory: 44.58GiB(31.99%) tps: 82,605 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-09 23:16:52,192 - root - INFO - lr: 1.9265e-04 gnorm: 0.78 [ 9:41:52<12:20:34] +[titan] 2025-07-09 23:16:56,131 - root - INFO - step: 44005 loss: 16.7964 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5914 global_avg_mtp_loss: 14.2050 +[titan] 2025-07-09 23:16:56,131 - root - INFO - lr: 1.9263e-04 gnorm: 0.80 [ 9:41:56<12:20:30] +[titan] 2025-07-09 23:17:00,033 - root - INFO - step: 44010 loss: 16.6237 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0542 +[titan] 2025-07-09 23:17:00,033 - root - INFO - lr: 1.9260e-04 gnorm: 0.79 [ 9:42:00<12:20:26] +[titan] 2025-07-09 23:17:03,939 - root - INFO - step: 44015 loss: 16.6824 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5768 global_avg_mtp_loss: 14.1056 +[titan] 2025-07-09 23:17:03,939 - root - INFO - lr: 1.9258e-04 gnorm: 0.87 [ 9:42:04<12:20:22] +[titan] 2025-07-09 23:17:07,846 - root - INFO - step: 44020 loss: 16.7159 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.1369 +[titan] 2025-07-09 23:17:07,847 - root - INFO - lr: 1.9256e-04 gnorm: 0.81 [ 9:42:08<12:20:18] +[titan] 2025-07-09 23:17:11,734 - root - INFO - step: 44025 loss: 16.5924 memory: 44.58GiB(31.99%) tps: 84,288 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0303 +[titan] 2025-07-09 23:17:11,735 - root - INFO - lr: 1.9254e-04 gnorm: 0.76 [ 9:42:12<12:20:14] +[titan] 2025-07-09 23:17:15,748 - root - INFO - step: 44030 loss: 16.3469 memory: 44.58GiB(31.99%) tps: 81,645 tflops: 281.77 mfu: 28.49% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8242 +[titan] 2025-07-09 23:17:15,749 - root - INFO - lr: 1.9252e-04 gnorm: 0.84 [ 9:42:16<12:20:10] +[titan] 2025-07-09 23:17:17,490 - root - INFO - Dumping profiler traces at step 44032 +[titan] 2025-07-09 23:17:17,521 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:17:19,905 - root - INFO - step: 44035 loss: 16.4674 memory: 44.58GiB(31.99%) tps: 78,841 tflops: 272.09 mfu: 27.51% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9256 +[titan] 2025-07-09 23:17:19,905 - root - INFO - lr: 1.9250e-04 gnorm: 0.82 [ 9:42:20<12:20:06] +[titan] 2025-07-09 23:17:23,837 - root - INFO - step: 44040 loss: 16.4595 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9245 +[titan] 2025-07-09 23:17:23,837 - root - INFO - lr: 1.9248e-04 gnorm: 0.83 [ 9:42:24<12:20:02] +[titan] 2025-07-09 23:17:27,749 - root - INFO - step: 44045 loss: 16.7250 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1399 +[titan] 2025-07-09 23:17:27,750 - root - INFO - lr: 1.9246e-04 gnorm: 0.78 [ 9:42:28<12:19:58] +[titan] 2025-07-09 23:17:30,867 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:17:31,654 - root - INFO - step: 44050 loss: 16.5653 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0047 +[titan] 2025-07-09 23:17:31,654 - root - INFO - lr: 1.9244e-04 gnorm: 0.83 [ 9:42:32<12:19:54] +[titan] 2025-07-09 23:17:35,595 - root - INFO - step: 44055 loss: 16.5928 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0264 +[titan] 2025-07-09 23:17:35,596 - root - INFO - lr: 1.9242e-04 gnorm: 1.32 [ 9:42:36<12:19:50] +[titan] 2025-07-09 23:17:39,556 - root - INFO - step: 44060 loss: 16.5163 memory: 44.58GiB(31.99%) tps: 82,734 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.9724 +[titan] 2025-07-09 23:17:39,557 - root - INFO - lr: 1.9239e-04 gnorm: 0.75 [ 9:42:40<12:19:46] +[titan] 2025-07-09 23:17:43,477 - root - INFO - step: 44065 loss: 16.5529 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 23:17:43,477 - root - INFO - lr: 1.9237e-04 gnorm: 0.77 [ 9:42:44<12:19:42] +[titan] 2025-07-09 23:17:47,370 - root - INFO - step: 44070 loss: 16.6232 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0650 +[titan] 2025-07-09 23:17:47,370 - root - INFO - lr: 1.9235e-04 gnorm: 0.87 [ 9:42:47<12:19:38] +[titan] 2025-07-09 23:17:51,303 - root - INFO - step: 44075 loss: 16.4625 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9217 +[titan] 2025-07-09 23:17:51,303 - root - INFO - lr: 1.9233e-04 gnorm: 0.78 [ 9:42:51<12:19:34] +[titan] 2025-07-09 23:17:55,301 - root - INFO - step: 44080 loss: 16.6464 memory: 44.58GiB(31.99%) tps: 81,968 tflops: 282.88 mfu: 28.60% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.0726 +[titan] 2025-07-09 23:17:55,301 - root - INFO - lr: 1.9231e-04 gnorm: 0.79 [ 9:42:55<12:19:30] +[titan] 2025-07-09 23:17:59,221 - root - INFO - step: 44085 loss: 16.7507 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5858 global_avg_mtp_loss: 14.1649 +[titan] 2025-07-09 23:17:59,222 - root - INFO - lr: 1.9229e-04 gnorm: 0.82 [ 9:42:59<12:19:26] +[titan] 2025-07-09 23:18:03,120 - root - INFO - step: 44090 loss: 16.6064 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 14.0503 +[titan] 2025-07-09 23:18:03,120 - root - INFO - lr: 1.9227e-04 gnorm: 0.79 [ 9:43:03<12:19:22] +[titan] 2025-07-09 23:18:07,026 - root - INFO - step: 44095 loss: 16.8373 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.6000 global_avg_mtp_loss: 14.2373 +[titan] 2025-07-09 23:18:07,026 - root - INFO - lr: 1.9225e-04 gnorm: 0.81 [ 9:43:07<12:19:18] +[titan] 2025-07-09 23:18:10,153 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:18:10,946 - root - INFO - step: 44100 loss: 16.6425 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0652 +[titan] 2025-07-09 23:18:10,946 - root - INFO - lr: 1.9223e-04 gnorm: 0.75 [ 9:43:11<12:19:14] +[titan] 2025-07-09 23:18:14,847 - root - INFO - step: 44105 loss: 16.3445 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8345 +[titan] 2025-07-09 23:18:14,848 - root - INFO - lr: 1.9221e-04 gnorm: 0.77 [ 9:43:15<12:19:10] +[titan] 2025-07-09 23:18:18,742 - root - INFO - step: 44110 loss: 16.5783 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5603 global_avg_mtp_loss: 14.0180 +[titan] 2025-07-09 23:18:18,742 - root - INFO - lr: 1.9219e-04 gnorm: 0.74 [ 9:43:19<12:19:06] +[titan] 2025-07-09 23:18:22,669 - root - INFO - step: 44115 loss: 16.7264 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.1539 +[titan] 2025-07-09 23:18:22,669 - root - INFO - lr: 1.9216e-04 gnorm: 0.73 [ 9:43:23<12:19:02] +[titan] 2025-07-09 23:18:26,610 - root - INFO - step: 44120 loss: 16.7412 memory: 44.58GiB(31.99%) tps: 83,140 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5959 global_avg_mtp_loss: 14.1453 +[titan] 2025-07-09 23:18:26,610 - root - INFO - lr: 1.9214e-04 gnorm: 0.80 [ 9:43:27<12:18:58] +[titan] 2025-07-09 23:18:30,550 - root - INFO - step: 44125 loss: 16.5029 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9607 +[titan] 2025-07-09 23:18:30,550 - root - INFO - lr: 1.9212e-04 gnorm: 0.79 [ 9:43:31<12:18:54] +[titan] 2025-07-09 23:18:34,469 - root - INFO - step: 44130 loss: 16.5992 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-09 23:18:34,470 - root - INFO - lr: 1.9210e-04 gnorm: 0.80 [ 9:43:35<12:18:50] +[titan] 2025-07-09 23:18:38,415 - root - INFO - step: 44135 loss: 16.8316 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.6038 global_avg_mtp_loss: 14.2278 +[titan] 2025-07-09 23:18:38,415 - root - INFO - lr: 1.9208e-04 gnorm: 0.78 [ 9:43:39<12:18:46] +[titan] 2025-07-09 23:18:42,326 - root - INFO - step: 44140 loss: 16.4567 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9250 +[titan] 2025-07-09 23:18:42,326 - root - INFO - lr: 1.9206e-04 gnorm: 0.85 [ 9:43:42<12:18:42] +[titan] 2025-07-09 23:18:46,251 - root - INFO - step: 44145 loss: 16.6886 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.1117 +[titan] 2025-07-09 23:18:46,251 - root - INFO - lr: 1.9204e-04 gnorm: 0.78 [ 9:43:46<12:18:38] +[titan] 2025-07-09 23:18:49,373 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:18:50,159 - root - INFO - step: 44150 loss: 16.6805 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5766 global_avg_mtp_loss: 14.1039 +[titan] 2025-07-09 23:18:50,159 - root - INFO - lr: 1.9202e-04 gnorm: 0.78 [ 9:43:50<12:18:34] +[titan] 2025-07-09 23:18:54,076 - root - INFO - step: 44155 loss: 16.5155 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 13.9599 +[titan] 2025-07-09 23:18:54,076 - root - INFO - lr: 1.9200e-04 gnorm: 0.80 [ 9:43:54<12:18:30] +[titan] 2025-07-09 23:18:57,971 - root - INFO - step: 44160 loss: 16.4833 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9403 +[titan] 2025-07-09 23:18:57,971 - root - INFO - lr: 1.9198e-04 gnorm: 0.75 [ 9:43:58<12:18:26] +[titan] 2025-07-09 23:19:01,909 - root - INFO - step: 44165 loss: 16.5607 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 13.9948 +[titan] 2025-07-09 23:19:01,909 - root - INFO - lr: 1.9195e-04 gnorm: 0.82 [ 9:44:02<12:18:22] +[titan] 2025-07-09 23:19:05,863 - root - INFO - step: 44170 loss: 16.6407 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0913 +[titan] 2025-07-09 23:19:05,864 - root - INFO - lr: 1.9193e-04 gnorm: 0.79 [ 9:44:06<12:18:18] +[titan] 2025-07-09 23:19:09,775 - root - INFO - step: 44175 loss: 16.3983 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8798 +[titan] 2025-07-09 23:19:09,775 - root - INFO - lr: 1.9191e-04 gnorm: 0.77 [ 9:44:10<12:18:13] +[titan] 2025-07-09 23:19:13,739 - root - INFO - step: 44180 loss: 16.5999 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.0283 +[titan] 2025-07-09 23:19:13,739 - root - INFO - lr: 1.9189e-04 gnorm: 0.81 [ 9:44:14<12:18:10] +[titan] 2025-07-09 23:19:17,681 - root - INFO - step: 44185 loss: 16.5923 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.0338 +[titan] 2025-07-09 23:19:17,682 - root - INFO - lr: 1.9187e-04 gnorm: 0.78 [ 9:44:18<12:18:06] +[titan] 2025-07-09 23:19:21,613 - root - INFO - step: 44190 loss: 16.5846 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 14.0238 +[titan] 2025-07-09 23:19:21,613 - root - INFO - lr: 1.9185e-04 gnorm: 0.85 [ 9:44:22<12:18:01] +[titan] 2025-07-09 23:19:25,519 - root - INFO - step: 44195 loss: 16.4348 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.8986 +[titan] 2025-07-09 23:19:25,520 - root - INFO - lr: 1.9183e-04 gnorm: 0.78 [ 9:44:26<12:17:57] +[titan] 2025-07-09 23:19:28,652 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:19:29,441 - root - INFO - step: 44200 loss: 16.5956 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0435 +[titan] 2025-07-09 23:19:29,441 - root - INFO - lr: 1.9181e-04 gnorm: 0.81 [ 9:44:30<12:17:53] +[titan] 2025-07-09 23:19:33,362 - root - INFO - step: 44205 loss: 16.2889 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.7828 +[titan] 2025-07-09 23:19:33,362 - root - INFO - lr: 1.9179e-04 gnorm: 0.77 [ 9:44:33<12:17:49] +[titan] 2025-07-09 23:19:37,273 - root - INFO - step: 44210 loss: 16.8069 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.2123 +[titan] 2025-07-09 23:19:37,274 - root - INFO - lr: 1.9177e-04 gnorm: 0.75 [ 9:44:37<12:17:45] +[titan] 2025-07-09 23:19:41,173 - root - INFO - step: 44215 loss: 16.6933 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.1257 +[titan] 2025-07-09 23:19:41,174 - root - INFO - lr: 1.9174e-04 gnorm: 0.77 [ 9:44:41<12:17:41] +[titan] 2025-07-09 23:19:45,081 - root - INFO - step: 44220 loss: 16.5397 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9933 +[titan] 2025-07-09 23:19:45,081 - root - INFO - lr: 1.9172e-04 gnorm: 0.81 [ 9:44:45<12:17:37] +[titan] 2025-07-09 23:19:48,997 - root - INFO - step: 44225 loss: 16.5793 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-09 23:19:48,997 - root - INFO - lr: 1.9170e-04 gnorm: 0.81 [ 9:44:49<12:17:33] +[titan] 2025-07-09 23:19:52,899 - root - INFO - step: 44230 loss: 16.6115 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5633 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-09 23:19:52,900 - root - INFO - lr: 1.9168e-04 gnorm: 0.80 [ 9:44:53<12:17:29] +[titan] 2025-07-09 23:19:56,834 - root - INFO - step: 44235 loss: 16.8686 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.6163 global_avg_mtp_loss: 14.2523 +[titan] 2025-07-09 23:19:56,834 - root - INFO - lr: 1.9166e-04 gnorm: 0.84 [ 9:44:57<12:17:25] +[titan] 2025-07-09 23:20:00,763 - root - INFO - step: 44240 loss: 16.8133 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.6103 global_avg_mtp_loss: 14.2030 +[titan] 2025-07-09 23:20:00,763 - root - INFO - lr: 1.9164e-04 gnorm: 0.81 [ 9:45:01<12:17:21] +[titan] 2025-07-09 23:20:04,723 - root - INFO - step: 44245 loss: 17.0211 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.6662 global_avg_mtp_loss: 14.3549 +[titan] 2025-07-09 23:20:04,724 - root - INFO - lr: 1.9162e-04 gnorm: 0.80 [ 9:45:05<12:17:17] +[titan] 2025-07-09 23:20:07,842 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:20:08,631 - root - INFO - step: 44250 loss: 16.4815 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9408 +[titan] 2025-07-09 23:20:08,631 - root - INFO - lr: 1.9160e-04 gnorm: 0.78 [ 9:45:09<12:17:13] +[titan] 2025-07-09 23:20:12,550 - root - INFO - step: 44255 loss: 16.4757 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9310 +[titan] 2025-07-09 23:20:12,550 - root - INFO - lr: 1.9158e-04 gnorm: 0.85 [ 9:45:13<12:17:09] +[titan] 2025-07-09 23:20:16,463 - root - INFO - step: 44260 loss: 16.7452 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.1735 +[titan] 2025-07-09 23:20:16,464 - root - INFO - lr: 1.9156e-04 gnorm: 0.82 [ 9:45:17<12:17:05] +[titan] 2025-07-09 23:20:20,379 - root - INFO - step: 44265 loss: 16.7072 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.1320 +[titan] 2025-07-09 23:20:20,380 - root - INFO - lr: 1.9153e-04 gnorm: 0.76 [ 9:45:20<12:17:01] +[titan] 2025-07-09 23:20:24,274 - root - INFO - step: 44270 loss: 16.6248 memory: 44.58GiB(31.99%) tps: 84,151 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5724 global_avg_mtp_loss: 14.0524 +[titan] 2025-07-09 23:20:24,274 - root - INFO - lr: 1.9151e-04 gnorm: 0.73 [ 9:45:24<12:16:57] +[titan] 2025-07-09 23:20:28,180 - root - INFO - step: 44275 loss: 16.5927 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 14.0366 +[titan] 2025-07-09 23:20:28,180 - root - INFO - lr: 1.9149e-04 gnorm: 0.74 [ 9:45:28<12:16:53] +[titan] 2025-07-09 23:20:32,099 - root - INFO - step: 44280 loss: 16.5889 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0350 +[titan] 2025-07-09 23:20:32,100 - root - INFO - lr: 1.9147e-04 gnorm: 0.77 [ 9:45:32<12:16:49] +[titan] 2025-07-09 23:20:35,993 - root - INFO - step: 44285 loss: 16.8033 memory: 44.58GiB(31.99%) tps: 84,169 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.2148 +[titan] 2025-07-09 23:20:35,993 - root - INFO - lr: 1.9145e-04 gnorm: 0.76 [ 9:45:36<12:16:45] +[titan] 2025-07-09 23:20:39,889 - root - INFO - step: 44290 loss: 16.4376 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-09 23:20:39,890 - root - INFO - lr: 1.9143e-04 gnorm: 0.88 [ 9:45:40<12:16:41] +[titan] 2025-07-09 23:20:43,823 - root - INFO - step: 44295 loss: 16.5957 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-09 23:20:43,823 - root - INFO - lr: 1.9141e-04 gnorm: 0.80 [ 9:45:44<12:16:37] +[titan] 2025-07-09 23:20:46,974 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:20:47,755 - root - INFO - step: 44300 loss: 16.6294 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-09 23:20:47,756 - root - INFO - lr: 1.9139e-04 gnorm: 0.76 [ 9:45:48<12:16:33] +[titan] 2025-07-09 23:20:51,706 - root - INFO - step: 44305 loss: 16.4764 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5462 global_avg_mtp_loss: 13.9302 +[titan] 2025-07-09 23:20:51,707 - root - INFO - lr: 1.9137e-04 gnorm: 0.74 [ 9:45:52<12:16:29] +[titan] 2025-07-09 23:20:55,623 - root - INFO - step: 44310 loss: 16.1025 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.6220 +[titan] 2025-07-09 23:20:55,623 - root - INFO - lr: 1.9135e-04 gnorm: 0.78 [ 9:45:56<12:16:25] +[titan] 2025-07-09 23:20:59,557 - root - INFO - step: 44315 loss: 16.6227 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0597 +[titan] 2025-07-09 23:20:59,557 - root - INFO - lr: 1.9132e-04 gnorm: 0.77 [ 9:46:00<12:16:21] +[titan] 2025-07-09 23:21:03,456 - root - INFO - step: 44320 loss: 16.5034 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 13.9519 +[titan] 2025-07-09 23:21:03,457 - root - INFO - lr: 1.9130e-04 gnorm: 0.89 [ 9:46:04<12:16:17] +[titan] 2025-07-09 23:21:07,363 - root - INFO - step: 44325 loss: 16.7047 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5886 global_avg_mtp_loss: 14.1161 +[titan] 2025-07-09 23:21:07,363 - root - INFO - lr: 1.9128e-04 gnorm: 0.83 [ 9:46:07<12:16:13] +[titan] 2025-07-09 23:21:11,284 - root - INFO - step: 44330 loss: 16.6707 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.0934 +[titan] 2025-07-09 23:21:11,285 - root - INFO - lr: 1.9126e-04 gnorm: 0.81 [ 9:46:11<12:16:09] +[titan] 2025-07-09 23:21:15,190 - root - INFO - step: 44335 loss: 16.2443 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7527 +[titan] 2025-07-09 23:21:15,191 - root - INFO - lr: 1.9124e-04 gnorm: 0.75 [ 9:46:15<12:16:05] +[titan] 2025-07-09 23:21:19,097 - root - INFO - step: 44340 loss: 16.4609 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9281 +[titan] 2025-07-09 23:21:19,097 - root - INFO - lr: 1.9122e-04 gnorm: 0.79 [ 9:46:19<12:16:01] +[titan] 2025-07-09 23:21:23,001 - root - INFO - step: 44345 loss: 16.6395 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5727 global_avg_mtp_loss: 14.0668 +[titan] 2025-07-09 23:21:23,001 - root - INFO - lr: 1.9120e-04 gnorm: 0.75 [ 9:46:23<12:15:56] +[titan] 2025-07-09 23:21:26,122 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:21:26,916 - root - INFO - step: 44350 loss: 16.8274 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.6056 global_avg_mtp_loss: 14.2218 +[titan] 2025-07-09 23:21:26,916 - root - INFO - lr: 1.9118e-04 gnorm: 0.81 [ 9:46:27<12:15:52] +[titan] 2025-07-09 23:21:30,836 - root - INFO - step: 44355 loss: 16.6956 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.1155 +[titan] 2025-07-09 23:21:30,836 - root - INFO - lr: 1.9116e-04 gnorm: 0.82 [ 9:46:31<12:15:48] +[titan] 2025-07-09 23:21:34,743 - root - INFO - step: 44360 loss: 16.5134 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 13.9584 +[titan] 2025-07-09 23:21:34,744 - root - INFO - lr: 1.9114e-04 gnorm: 0.79 [ 9:46:35<12:15:44] +[titan] 2025-07-09 23:21:38,649 - root - INFO - step: 44365 loss: 16.4355 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9010 +[titan] 2025-07-09 23:21:38,649 - root - INFO - lr: 1.9111e-04 gnorm: 0.79 [ 9:46:39<12:15:40] +[titan] 2025-07-09 23:21:42,602 - root - INFO - step: 44370 loss: 16.5176 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9709 +[titan] 2025-07-09 23:21:42,603 - root - INFO - lr: 1.9109e-04 gnorm: 0.86 [ 9:46:43<12:15:36] +[titan] 2025-07-09 23:21:46,521 - root - INFO - step: 44375 loss: 16.5795 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0287 +[titan] 2025-07-09 23:21:46,521 - root - INFO - lr: 1.9107e-04 gnorm: 0.85 [ 9:46:47<12:15:32] +[titan] 2025-07-09 23:21:50,458 - root - INFO - step: 44380 loss: 16.6784 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5854 global_avg_mtp_loss: 14.0930 +[titan] 2025-07-09 23:21:50,458 - root - INFO - lr: 1.9105e-04 gnorm: 0.82 [ 9:46:51<12:15:28] +[titan] 2025-07-09 23:21:54,366 - root - INFO - step: 44385 loss: 16.5925 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0320 +[titan] 2025-07-09 23:21:54,367 - root - INFO - lr: 1.9103e-04 gnorm: 0.79 [ 9:46:54<12:15:24] +[titan] 2025-07-09 23:21:58,299 - root - INFO - step: 44390 loss: 16.6593 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-09 23:21:58,300 - root - INFO - lr: 1.9101e-04 gnorm: 0.82 [ 9:46:58<12:15:20] +[titan] 2025-07-09 23:22:02,233 - root - INFO - step: 44395 loss: 16.4139 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.8881 +[titan] 2025-07-09 23:22:02,234 - root - INFO - lr: 1.9099e-04 gnorm: 0.80 [ 9:47:02<12:15:16] +[titan] 2025-07-09 23:22:05,369 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:22:06,184 - root - INFO - step: 44400 loss: 16.7059 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.1257 +[titan] 2025-07-09 23:22:06,184 - root - INFO - lr: 1.9097e-04 gnorm: 0.80 [ 9:47:06<12:15:12] +[titan] 2025-07-09 23:22:10,090 - root - INFO - step: 44405 loss: 16.5846 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0260 +[titan] 2025-07-09 23:22:10,091 - root - INFO - lr: 1.9095e-04 gnorm: 0.84 [ 9:47:10<12:15:08] +[titan] 2025-07-09 23:22:14,018 - root - INFO - step: 44410 loss: 16.5890 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-09 23:22:14,018 - root - INFO - lr: 1.9093e-04 gnorm: 0.77 [ 9:47:14<12:15:04] +[titan] 2025-07-09 23:22:17,978 - root - INFO - step: 44415 loss: 16.5810 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0196 +[titan] 2025-07-09 23:22:17,978 - root - INFO - lr: 1.9090e-04 gnorm: 0.80 [ 9:47:18<12:15:00] +[titan] 2025-07-09 23:22:21,877 - root - INFO - step: 44420 loss: 16.6391 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-09 23:22:21,878 - root - INFO - lr: 1.9088e-04 gnorm: 0.81 [ 9:47:22<12:14:56] +[titan] 2025-07-09 23:22:25,803 - root - INFO - step: 44425 loss: 16.5406 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 23:22:25,803 - root - INFO - lr: 1.9086e-04 gnorm: 0.80 [ 9:47:26<12:14:52] +[titan] 2025-07-09 23:22:29,709 - root - INFO - step: 44430 loss: 16.5662 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0143 +[titan] 2025-07-09 23:22:29,710 - root - INFO - lr: 1.9084e-04 gnorm: 0.80 [ 9:47:30<12:14:48] +[titan] 2025-07-09 23:22:33,651 - root - INFO - step: 44435 loss: 16.5115 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9696 +[titan] 2025-07-09 23:22:33,651 - root - INFO - lr: 1.9082e-04 gnorm: 0.82 [ 9:47:34<12:14:44] +[titan] 2025-07-09 23:22:37,603 - root - INFO - step: 44440 loss: 16.4023 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.8688 +[titan] 2025-07-09 23:22:37,603 - root - INFO - lr: 1.9080e-04 gnorm: 0.78 [ 9:47:38<12:14:40] +[titan] 2025-07-09 23:22:41,517 - root - INFO - step: 44445 loss: 16.4853 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5412 global_avg_mtp_loss: 13.9440 +[titan] 2025-07-09 23:22:41,517 - root - INFO - lr: 1.9078e-04 gnorm: 0.80 [ 9:47:42<12:14:36] +[titan] 2025-07-09 23:22:44,625 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:22:45,414 - root - INFO - step: 44450 loss: 16.7685 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5817 global_avg_mtp_loss: 14.1868 +[titan] 2025-07-09 23:22:45,415 - root - INFO - lr: 1.9076e-04 gnorm: 0.75 [ 9:47:45<12:14:32] +[titan] 2025-07-09 23:22:49,328 - root - INFO - step: 44455 loss: 16.6788 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.0946 +[titan] 2025-07-09 23:22:49,329 - root - INFO - lr: 1.9074e-04 gnorm: 0.76 [ 9:47:49<12:14:28] +[titan] 2025-07-09 23:22:53,227 - root - INFO - step: 44460 loss: 16.5677 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5595 global_avg_mtp_loss: 14.0081 +[titan] 2025-07-09 23:22:53,227 - root - INFO - lr: 1.9071e-04 gnorm: 0.79 [ 9:47:53<12:14:24] +[titan] 2025-07-09 23:22:57,144 - root - INFO - step: 44465 loss: 16.6980 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1180 +[titan] 2025-07-09 23:22:57,144 - root - INFO - lr: 1.9069e-04 gnorm: 0.77 [ 9:47:57<12:14:20] +[titan] 2025-07-09 23:23:01,066 - root - INFO - step: 44470 loss: 16.7821 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.1813 +[titan] 2025-07-09 23:23:01,066 - root - INFO - lr: 1.9067e-04 gnorm: 0.82 [ 9:48:01<12:14:16] +[titan] 2025-07-09 23:23:04,993 - root - INFO - step: 44475 loss: 16.6061 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0379 +[titan] 2025-07-09 23:23:04,993 - root - INFO - lr: 1.9065e-04 gnorm: 0.86 [ 9:48:05<12:14:12] +[titan] 2025-07-09 23:23:08,891 - root - INFO - step: 44480 loss: 16.8568 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.6094 global_avg_mtp_loss: 14.2474 +[titan] 2025-07-09 23:23:08,891 - root - INFO - lr: 1.9063e-04 gnorm: 0.79 [ 9:48:09<12:14:08] +[titan] 2025-07-09 23:23:12,809 - root - INFO - step: 44485 loss: 16.6689 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.1119 +[titan] 2025-07-09 23:23:12,809 - root - INFO - lr: 1.9061e-04 gnorm: 0.75 [ 9:48:13<12:14:04] +[titan] 2025-07-09 23:23:16,716 - root - INFO - step: 44490 loss: 16.4042 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8940 +[titan] 2025-07-09 23:23:16,716 - root - INFO - lr: 1.9059e-04 gnorm: 0.77 [ 9:48:17<12:14:00] +[titan] 2025-07-09 23:23:20,643 - root - INFO - step: 44495 loss: 16.7118 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.1197 +[titan] 2025-07-09 23:23:20,643 - root - INFO - lr: 1.9057e-04 gnorm: 0.77 [ 9:48:21<12:13:56] +[titan] 2025-07-09 23:23:23,747 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:23:24,537 - root - INFO - step: 44500 loss: 16.5596 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 14.0101 +[titan] 2025-07-09 23:23:24,537 - root - INFO - lr: 1.9055e-04 gnorm: 0.78 [ 9:48:25<12:13:52] +[titan] 2025-07-09 23:23:28,491 - root - INFO - step: 44505 loss: 16.2472 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.7322 +[titan] 2025-07-09 23:23:28,491 - root - INFO - lr: 1.9053e-04 gnorm: 0.79 [ 9:48:29<12:13:48] +[titan] 2025-07-09 23:23:32,401 - root - INFO - step: 44510 loss: 16.6214 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5671 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-09 23:23:32,401 - root - INFO - lr: 1.9050e-04 gnorm: 0.77 [ 9:48:32<12:13:44] +[titan] 2025-07-09 23:23:36,295 - root - INFO - step: 44515 loss: 16.3807 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8647 +[titan] 2025-07-09 23:23:36,295 - root - INFO - lr: 1.9048e-04 gnorm: 0.76 [ 9:48:36<12:13:40] +[titan] 2025-07-09 23:23:40,201 - root - INFO - step: 44520 loss: 16.4833 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9422 +[titan] 2025-07-09 23:23:40,202 - root - INFO - lr: 1.9046e-04 gnorm: 0.77 [ 9:48:40<12:13:36] +[titan] 2025-07-09 23:23:44,151 - root - INFO - step: 44525 loss: 16.7288 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.1416 +[titan] 2025-07-09 23:23:44,151 - root - INFO - lr: 1.9044e-04 gnorm: 0.80 [ 9:48:44<12:13:32] +[titan] 2025-07-09 23:23:48,057 - root - INFO - step: 44530 loss: 16.3495 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.8320 +[titan] 2025-07-09 23:23:48,057 - root - INFO - lr: 1.9042e-04 gnorm: 0.81 [ 9:48:48<12:13:28] +[titan] 2025-07-09 23:23:51,975 - root - INFO - step: 44535 loss: 16.6422 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0840 +[titan] 2025-07-09 23:23:51,975 - root - INFO - lr: 1.9040e-04 gnorm: 0.82 [ 9:48:52<12:13:24] +[titan] 2025-07-09 23:23:55,874 - root - INFO - step: 44540 loss: 16.5431 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9972 +[titan] 2025-07-09 23:23:55,874 - root - INFO - lr: 1.9038e-04 gnorm: 0.80 [ 9:48:56<12:13:19] +[titan] 2025-07-09 23:23:59,226 - root - INFO - Dumping profiler traces at step 44544 +[titan] 2025-07-09 23:23:59,260 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:24:00,052 - root - INFO - step: 44545 loss: 16.6414 memory: 44.58GiB(31.99%) tps: 78,432 tflops: 270.68 mfu: 27.37% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0750 +[titan] 2025-07-09 23:24:00,052 - root - INFO - lr: 1.9036e-04 gnorm: 0.82 [ 9:49:00<12:13:16] +[titan] 2025-07-09 23:24:03,168 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:24:03,965 - root - INFO - step: 44550 loss: 16.3592 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-09 23:24:03,965 - root - INFO - lr: 1.9034e-04 gnorm: 0.80 [ 9:49:04<12:13:12] +[titan] 2025-07-09 23:24:07,899 - root - INFO - step: 44555 loss: 16.5050 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9667 +[titan] 2025-07-09 23:24:07,899 - root - INFO - lr: 1.9032e-04 gnorm: 0.79 [ 9:49:08<12:13:08] +[titan] 2025-07-09 23:24:11,814 - root - INFO - step: 44560 loss: 16.4805 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 13.9440 +[titan] 2025-07-09 23:24:11,815 - root - INFO - lr: 1.9029e-04 gnorm: 0.75 [ 9:49:12<12:13:04] +[titan] 2025-07-09 23:24:15,734 - root - INFO - step: 44565 loss: 16.4155 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-09 23:24:15,734 - root - INFO - lr: 1.9027e-04 gnorm: 0.81 [ 9:49:16<12:13:00] +[titan] 2025-07-09 23:24:19,654 - root - INFO - step: 44570 loss: 16.4932 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9539 +[titan] 2025-07-09 23:24:19,654 - root - INFO - lr: 1.9025e-04 gnorm: 0.83 [ 9:49:20<12:12:56] +[titan] 2025-07-09 23:24:23,578 - root - INFO - step: 44575 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8622 +[titan] 2025-07-09 23:24:23,578 - root - INFO - lr: 1.9023e-04 gnorm: 0.81 [ 9:49:24<12:12:52] +[titan] 2025-07-09 23:24:27,494 - root - INFO - step: 44580 loss: 16.7348 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.1591 +[titan] 2025-07-09 23:24:27,494 - root - INFO - lr: 1.9021e-04 gnorm: 0.83 [ 9:49:28<12:12:48] +[titan] 2025-07-09 23:24:31,400 - root - INFO - step: 44585 loss: 16.7272 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1472 +[titan] 2025-07-09 23:24:31,400 - root - INFO - lr: 1.9019e-04 gnorm: 0.84 [ 9:49:31<12:12:44] +[titan] 2025-07-09 23:24:35,313 - root - INFO - step: 44590 loss: 16.7320 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5837 global_avg_mtp_loss: 14.1483 +[titan] 2025-07-09 23:24:35,313 - root - INFO - lr: 1.9017e-04 gnorm: 0.78 [ 9:49:35<12:12:40] +[titan] 2025-07-09 23:24:39,264 - root - INFO - step: 44595 loss: 16.7324 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5912 global_avg_mtp_loss: 14.1412 +[titan] 2025-07-09 23:24:39,264 - root - INFO - lr: 1.9015e-04 gnorm: 0.79 [ 9:49:39<12:12:36] +[titan] 2025-07-09 23:24:42,414 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:24:43,198 - root - INFO - step: 44600 loss: 16.4573 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5318 global_avg_mtp_loss: 13.9256 +[titan] 2025-07-09 23:24:43,198 - root - INFO - lr: 1.9013e-04 gnorm: 0.78 [ 9:49:43<12:12:32] +[titan] 2025-07-09 23:24:47,122 - root - INFO - step: 44605 loss: 16.3485 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.8436 +[titan] 2025-07-09 23:24:47,122 - root - INFO - lr: 1.9010e-04 gnorm: 0.76 [ 9:49:47<12:12:27] +[titan] 2025-07-09 23:24:51,040 - root - INFO - step: 44610 loss: 16.6789 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.1051 +[titan] 2025-07-09 23:24:51,040 - root - INFO - lr: 1.9008e-04 gnorm: 0.75 [ 9:49:51<12:12:23] +[titan] 2025-07-09 23:24:54,947 - root - INFO - step: 44615 loss: 16.5496 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 13.9903 +[titan] 2025-07-09 23:24:54,947 - root - INFO - lr: 1.9006e-04 gnorm: 0.76 [ 9:49:55<12:12:19] +[titan] 2025-07-09 23:24:58,900 - root - INFO - step: 44620 loss: 16.5566 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5662 global_avg_mtp_loss: 13.9904 +[titan] 2025-07-09 23:24:58,900 - root - INFO - lr: 1.9004e-04 gnorm: 0.77 [ 9:49:59<12:12:15] +[titan] 2025-07-09 23:25:02,808 - root - INFO - step: 44625 loss: 16.5703 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 14.0203 +[titan] 2025-07-09 23:25:02,808 - root - INFO - lr: 1.9002e-04 gnorm: 0.79 [ 9:50:03<12:12:11] +[titan] 2025-07-09 23:25:06,707 - root - INFO - step: 44630 loss: 16.8440 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.2448 +[titan] 2025-07-09 23:25:06,707 - root - INFO - lr: 1.9000e-04 gnorm: 0.84 [ 9:50:07<12:12:07] +[titan] 2025-07-09 23:25:10,622 - root - INFO - step: 44635 loss: 16.4032 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8986 +[titan] 2025-07-09 23:25:10,622 - root - INFO - lr: 1.8998e-04 gnorm: 0.80 [ 9:50:11<12:12:03] +[titan] 2025-07-09 23:25:14,579 - root - INFO - step: 44640 loss: 16.7948 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.6040 global_avg_mtp_loss: 14.1908 +[titan] 2025-07-09 23:25:14,579 - root - INFO - lr: 1.8996e-04 gnorm: 0.77 [ 9:50:15<12:11:59] +[titan] 2025-07-09 23:25:18,497 - root - INFO - step: 44645 loss: 16.6925 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-09 23:25:18,497 - root - INFO - lr: 1.8994e-04 gnorm: 0.78 [ 9:50:19<12:11:55] +[titan] 2025-07-09 23:25:21,616 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:25:22,408 - root - INFO - step: 44650 loss: 16.5530 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-09 23:25:22,408 - root - INFO - lr: 1.8992e-04 gnorm: 0.78 [ 9:50:22<12:11:51] +[titan] 2025-07-09 23:25:26,307 - root - INFO - step: 44655 loss: 16.5797 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 14.0318 +[titan] 2025-07-09 23:25:26,308 - root - INFO - lr: 1.8989e-04 gnorm: 0.75 [ 9:50:26<12:11:47] +[titan] 2025-07-09 23:25:30,250 - root - INFO - step: 44660 loss: 16.4862 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-09 23:25:30,250 - root - INFO - lr: 1.8987e-04 gnorm: 0.79 [ 9:50:30<12:11:43] +[titan] 2025-07-09 23:25:34,171 - root - INFO - step: 44665 loss: 16.6278 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.0623 +[titan] 2025-07-09 23:25:34,171 - root - INFO - lr: 1.8985e-04 gnorm: 0.81 [ 9:50:34<12:11:39] +[titan] 2025-07-09 23:25:38,091 - root - INFO - step: 44670 loss: 16.5813 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.0285 +[titan] 2025-07-09 23:25:38,091 - root - INFO - lr: 1.8983e-04 gnorm: 0.77 [ 9:50:38<12:11:35] +[titan] 2025-07-09 23:25:42,030 - root - INFO - step: 44675 loss: 16.5374 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9915 +[titan] 2025-07-09 23:25:42,030 - root - INFO - lr: 1.8981e-04 gnorm: 0.80 [ 9:50:42<12:11:31] +[titan] 2025-07-09 23:25:45,939 - root - INFO - step: 44680 loss: 16.5546 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5542 global_avg_mtp_loss: 14.0004 +[titan] 2025-07-09 23:25:45,939 - root - INFO - lr: 1.8979e-04 gnorm: 0.85 [ 9:50:46<12:11:27] +[titan] 2025-07-09 23:25:49,850 - root - INFO - step: 44685 loss: 16.3118 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8025 +[titan] 2025-07-09 23:25:49,850 - root - INFO - lr: 1.8977e-04 gnorm: 0.79 [ 9:50:50<12:11:23] +[titan] 2025-07-09 23:25:53,769 - root - INFO - step: 44690 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.9925 +[titan] 2025-07-09 23:25:53,769 - root - INFO - lr: 1.8975e-04 gnorm: 0.91 [ 9:50:54<12:11:19] +[titan] 2025-07-09 23:25:57,692 - root - INFO - step: 44695 loss: 16.5166 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-09 23:25:57,692 - root - INFO - lr: 1.8973e-04 gnorm: 0.88 [ 9:50:58<12:11:15] +[titan] 2025-07-09 23:26:00,804 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:26:01,599 - root - INFO - step: 44700 loss: 16.5252 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9830 +[titan] 2025-07-09 23:26:01,600 - root - INFO - lr: 1.8970e-04 gnorm: 0.76 [ 9:51:02<12:11:11] +[titan] 2025-07-09 23:26:05,511 - root - INFO - step: 44705 loss: 16.7145 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1345 +[titan] 2025-07-09 23:26:05,511 - root - INFO - lr: 1.8968e-04 gnorm: 1.11 [ 9:51:06<12:11:07] +[titan] 2025-07-09 23:26:09,430 - root - INFO - step: 44710 loss: 16.7685 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.6079 global_avg_mtp_loss: 14.1606 +[titan] 2025-07-09 23:26:09,430 - root - INFO - lr: 1.8966e-04 gnorm: 0.81 [ 9:51:09<12:11:03] +[titan] 2025-07-09 23:26:13,352 - root - INFO - step: 44715 loss: 16.2744 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-09 23:26:13,352 - root - INFO - lr: 1.8964e-04 gnorm: 0.83 [ 9:51:13<12:10:59] +[titan] 2025-07-09 23:26:17,261 - root - INFO - step: 44720 loss: 16.5584 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 14.0144 +[titan] 2025-07-09 23:26:17,262 - root - INFO - lr: 1.8962e-04 gnorm: 0.77 [ 9:51:17<12:10:55] +[titan] 2025-07-09 23:26:21,236 - root - INFO - step: 44725 loss: 16.5439 memory: 44.58GiB(31.99%) tps: 82,462 tflops: 284.59 mfu: 28.78% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 14.0070 +[titan] 2025-07-09 23:26:21,236 - root - INFO - lr: 1.8960e-04 gnorm: 0.81 [ 9:51:21<12:10:51] +[titan] 2025-07-09 23:26:25,140 - root - INFO - step: 44730 loss: 16.6608 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.0830 +[titan] 2025-07-09 23:26:25,140 - root - INFO - lr: 1.8958e-04 gnorm: 0.84 [ 9:51:25<12:10:47] +[titan] 2025-07-09 23:26:29,064 - root - INFO - step: 44735 loss: 16.7799 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5948 global_avg_mtp_loss: 14.1851 +[titan] 2025-07-09 23:26:29,065 - root - INFO - lr: 1.8956e-04 gnorm: 0.83 [ 9:51:29<12:10:43] +[titan] 2025-07-09 23:26:32,996 - root - INFO - step: 44740 loss: 16.8884 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.6161 global_avg_mtp_loss: 14.2723 +[titan] 2025-07-09 23:26:32,996 - root - INFO - lr: 1.8954e-04 gnorm: 0.76 [ 9:51:33<12:10:39] +[titan] 2025-07-09 23:26:36,910 - root - INFO - step: 44745 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-09 23:26:36,910 - root - INFO - lr: 1.8952e-04 gnorm: 0.86 [ 9:51:37<12:10:35] +[titan] 2025-07-09 23:26:40,033 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:26:40,871 - root - INFO - step: 44750 loss: 16.4295 memory: 44.58GiB(31.99%) tps: 82,730 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.5470 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-09 23:26:40,871 - root - INFO - lr: 1.8949e-04 gnorm: 0.90 [ 9:51:41<12:10:31] +[titan] 2025-07-09 23:26:44,785 - root - INFO - step: 44755 loss: 16.1813 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.6876 +[titan] 2025-07-09 23:26:44,785 - root - INFO - lr: 1.8947e-04 gnorm: 0.79 [ 9:51:45<12:10:27] +[titan] 2025-07-09 23:26:48,721 - root - INFO - step: 44760 loss: 16.6659 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.0917 +[titan] 2025-07-09 23:26:48,721 - root - INFO - lr: 1.8945e-04 gnorm: 0.78 [ 9:51:49<12:10:23] +[titan] 2025-07-09 23:26:52,611 - root - INFO - step: 44765 loss: 16.6250 memory: 44.58GiB(31.99%) tps: 84,240 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0557 +[titan] 2025-07-09 23:26:52,611 - root - INFO - lr: 1.8943e-04 gnorm: 0.79 [ 9:51:53<12:10:19] +[titan] 2025-07-09 23:26:56,529 - root - INFO - step: 44770 loss: 16.8512 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5967 global_avg_mtp_loss: 14.2545 +[titan] 2025-07-09 23:26:56,529 - root - INFO - lr: 1.8941e-04 gnorm: 0.80 [ 9:51:57<12:10:15] +[titan] 2025-07-09 23:27:00,449 - root - INFO - step: 44775 loss: 16.5760 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 14.0330 +[titan] 2025-07-09 23:27:00,450 - root - INFO - lr: 1.8939e-04 gnorm: 0.78 [ 9:52:00<12:10:11] +[titan] 2025-07-09 23:27:04,367 - root - INFO - step: 44780 loss: 16.7709 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.6038 global_avg_mtp_loss: 14.1672 +[titan] 2025-07-09 23:27:04,368 - root - INFO - lr: 1.8937e-04 gnorm: 0.80 [ 9:52:04<12:10:07] +[titan] 2025-07-09 23:27:08,270 - root - INFO - step: 44785 loss: 16.6416 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0762 +[titan] 2025-07-09 23:27:08,270 - root - INFO - lr: 1.8935e-04 gnorm: 0.82 [ 9:52:08<12:10:03] +[titan] 2025-07-09 23:27:12,200 - root - INFO - step: 44790 loss: 16.6652 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0987 +[titan] 2025-07-09 23:27:12,201 - root - INFO - lr: 1.8933e-04 gnorm: 0.87 [ 9:52:12<12:09:59] +[titan] 2025-07-09 23:27:16,100 - root - INFO - step: 44795 loss: 16.7583 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1694 +[titan] 2025-07-09 23:27:16,100 - root - INFO - lr: 1.8930e-04 gnorm: 0.79 [ 9:52:16<12:09:55] +[titan] 2025-07-09 23:27:19,217 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:27:20,021 - root - INFO - step: 44800 loss: 16.4328 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.8988 +[titan] 2025-07-09 23:27:20,021 - root - INFO - lr: 1.8928e-04 gnorm: 0.78 [ 9:52:20<12:09:51] +[titan] 2025-07-09 23:27:23,918 - root - INFO - step: 44805 loss: 16.6778 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.0989 +[titan] 2025-07-09 23:27:23,918 - root - INFO - lr: 1.8926e-04 gnorm: 0.80 [ 9:52:24<12:09:46] +[titan] 2025-07-09 23:27:27,855 - root - INFO - step: 44810 loss: 16.6141 memory: 44.58GiB(31.99%) tps: 83,229 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.0345 +[titan] 2025-07-09 23:27:27,855 - root - INFO - lr: 1.8924e-04 gnorm: 0.81 [ 9:52:28<12:09:42] +[titan] 2025-07-09 23:27:31,797 - root - INFO - step: 44815 loss: 16.6438 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0850 +[titan] 2025-07-09 23:27:31,798 - root - INFO - lr: 1.8922e-04 gnorm: 0.77 [ 9:52:32<12:09:38] +[titan] 2025-07-09 23:27:35,724 - root - INFO - step: 44820 loss: 16.6415 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0750 +[titan] 2025-07-09 23:27:35,725 - root - INFO - lr: 1.8920e-04 gnorm: 0.80 [ 9:52:36<12:09:34] +[titan] 2025-07-09 23:27:39,631 - root - INFO - step: 44825 loss: 16.6534 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.0847 +[titan] 2025-07-09 23:27:39,631 - root - INFO - lr: 1.8918e-04 gnorm: 0.77 [ 9:52:40<12:09:30] +[titan] 2025-07-09 23:27:43,594 - root - INFO - step: 44830 loss: 16.6590 memory: 44.58GiB(31.99%) tps: 82,698 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.5653 global_avg_mtp_loss: 14.0937 +[titan] 2025-07-09 23:27:43,594 - root - INFO - lr: 1.8916e-04 gnorm: 0.82 [ 9:52:44<12:09:26] +[titan] 2025-07-09 23:27:47,501 - root - INFO - step: 44835 loss: 16.6551 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5669 global_avg_mtp_loss: 14.0882 +[titan] 2025-07-09 23:27:47,501 - root - INFO - lr: 1.8914e-04 gnorm: 0.80 [ 9:52:48<12:09:22] +[titan] 2025-07-09 23:27:51,417 - root - INFO - step: 44840 loss: 16.6044 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0468 +[titan] 2025-07-09 23:27:51,417 - root - INFO - lr: 1.8911e-04 gnorm: 0.82 [ 9:52:51<12:09:18] +[titan] 2025-07-09 23:27:55,315 - root - INFO - step: 44845 loss: 16.5593 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 14.0020 +[titan] 2025-07-09 23:27:55,315 - root - INFO - lr: 1.8909e-04 gnorm: 0.75 [ 9:52:55<12:09:14] +[titan] 2025-07-09 23:27:58,482 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:27:59,268 - root - INFO - step: 44850 loss: 16.2889 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.7689 +[titan] 2025-07-09 23:27:59,268 - root - INFO - lr: 1.8907e-04 gnorm: 0.82 [ 9:52:59<12:09:10] +[titan] 2025-07-09 23:28:03,201 - root - INFO - step: 44855 loss: 16.5673 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 14.0221 +[titan] 2025-07-09 23:28:03,201 - root - INFO - lr: 1.8905e-04 gnorm: 0.79 [ 9:53:03<12:09:06] +[titan] 2025-07-09 23:28:07,095 - root - INFO - step: 44860 loss: 16.6188 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0495 +[titan] 2025-07-09 23:28:07,095 - root - INFO - lr: 1.8903e-04 gnorm: 0.76 [ 9:53:07<12:09:02] +[titan] 2025-07-09 23:28:10,997 - root - INFO - step: 44865 loss: 16.5006 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9553 +[titan] 2025-07-09 23:28:10,997 - root - INFO - lr: 1.8901e-04 gnorm: 0.78 [ 9:53:11<12:08:58] +[titan] 2025-07-09 23:28:14,926 - root - INFO - step: 44870 loss: 16.6942 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.1009 +[titan] 2025-07-09 23:28:14,926 - root - INFO - lr: 1.8899e-04 gnorm: 0.80 [ 9:53:15<12:08:54] +[titan] 2025-07-09 23:28:18,834 - root - INFO - step: 44875 loss: 16.5282 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9871 +[titan] 2025-07-09 23:28:18,835 - root - INFO - lr: 1.8897e-04 gnorm: 0.82 [ 9:53:19<12:08:50] +[titan] 2025-07-09 23:28:22,727 - root - INFO - step: 44880 loss: 16.5356 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9952 +[titan] 2025-07-09 23:28:22,727 - root - INFO - lr: 1.8895e-04 gnorm: 0.77 [ 9:53:23<12:08:46] +[titan] 2025-07-09 23:28:26,624 - root - INFO - step: 44885 loss: 16.4707 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9399 +[titan] 2025-07-09 23:28:26,625 - root - INFO - lr: 1.8893e-04 gnorm: 0.79 [ 9:53:27<12:08:42] +[titan] 2025-07-09 23:28:30,523 - root - INFO - step: 44890 loss: 16.5623 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0033 +[titan] 2025-07-09 23:28:30,523 - root - INFO - lr: 1.8890e-04 gnorm: 0.77 [ 9:53:31<12:08:38] +[titan] 2025-07-09 23:28:34,444 - root - INFO - step: 44895 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5360 global_avg_mtp_loss: 13.9706 +[titan] 2025-07-09 23:28:34,444 - root - INFO - lr: 1.8888e-04 gnorm: 0.88 [ 9:53:34<12:08:34] +[titan] 2025-07-09 23:28:37,546 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:28:38,334 - root - INFO - step: 44900 loss: 16.6037 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0497 +[titan] 2025-07-09 23:28:38,335 - root - INFO - lr: 1.8886e-04 gnorm: 0.84 [ 9:53:38<12:08:30] +[titan] 2025-07-09 23:28:42,236 - root - INFO - step: 44905 loss: 16.6701 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.0996 +[titan] 2025-07-09 23:28:42,236 - root - INFO - lr: 1.8884e-04 gnorm: 0.81 [ 9:53:42<12:08:26] +[titan] 2025-07-09 23:28:46,151 - root - INFO - step: 44910 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5484 global_avg_mtp_loss: 13.9642 +[titan] 2025-07-09 23:28:46,151 - root - INFO - lr: 1.8882e-04 gnorm: 0.78 [ 9:53:46<12:08:22] +[titan] 2025-07-09 23:28:50,052 - root - INFO - step: 44915 loss: 16.8572 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.2619 +[titan] 2025-07-09 23:28:50,052 - root - INFO - lr: 1.8880e-04 gnorm: 0.81 [ 9:53:50<12:08:18] +[titan] 2025-07-09 23:28:53,977 - root - INFO - step: 44920 loss: 16.5830 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0296 +[titan] 2025-07-09 23:28:53,977 - root - INFO - lr: 1.8878e-04 gnorm: 0.73 [ 9:53:54<12:08:14] +[titan] 2025-07-09 23:28:57,905 - root - INFO - step: 44925 loss: 16.3548 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8420 +[titan] 2025-07-09 23:28:57,905 - root - INFO - lr: 1.8876e-04 gnorm: 0.78 [ 9:53:58<12:08:10] +[titan] 2025-07-09 23:29:01,839 - root - INFO - step: 44930 loss: 16.8108 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.6077 global_avg_mtp_loss: 14.2030 +[titan] 2025-07-09 23:29:01,839 - root - INFO - lr: 1.8874e-04 gnorm: 0.80 [ 9:54:02<12:08:06] +[titan] 2025-07-09 23:29:05,744 - root - INFO - step: 44935 loss: 16.6782 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.1131 +[titan] 2025-07-09 23:29:05,744 - root - INFO - lr: 1.8871e-04 gnorm: 0.82 [ 9:54:06<12:08:02] +[titan] 2025-07-09 23:29:09,639 - root - INFO - step: 44940 loss: 16.6977 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1124 +[titan] 2025-07-09 23:29:09,639 - root - INFO - lr: 1.8869e-04 gnorm: 0.81 [ 9:54:10<12:07:58] +[titan] 2025-07-09 23:29:13,550 - root - INFO - step: 44945 loss: 16.6860 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-09 23:29:13,550 - root - INFO - lr: 1.8867e-04 gnorm: 0.76 [ 9:54:14<12:07:54] +[titan] 2025-07-09 23:29:16,658 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:29:17,445 - root - INFO - step: 44950 loss: 16.5158 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9750 +[titan] 2025-07-09 23:29:17,445 - root - INFO - lr: 1.8865e-04 gnorm: 0.77 [ 9:54:17<12:07:50] +[titan] 2025-07-09 23:29:21,368 - root - INFO - step: 44955 loss: 16.6522 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0864 +[titan] 2025-07-09 23:29:21,368 - root - INFO - lr: 1.8863e-04 gnorm: 0.78 [ 9:54:21<12:07:46] +[titan] 2025-07-09 23:29:25,287 - root - INFO - step: 44960 loss: 16.7177 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.1349 +[titan] 2025-07-09 23:29:25,288 - root - INFO - lr: 1.8861e-04 gnorm: 0.76 [ 9:54:25<12:07:42] +[titan] 2025-07-09 23:29:29,206 - root - INFO - step: 44965 loss: 16.6105 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-09 23:29:29,207 - root - INFO - lr: 1.8859e-04 gnorm: 0.74 [ 9:54:29<12:07:38] +[titan] 2025-07-09 23:29:33,136 - root - INFO - step: 44970 loss: 16.4552 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9216 +[titan] 2025-07-09 23:29:33,136 - root - INFO - lr: 1.8857e-04 gnorm: 0.75 [ 9:54:33<12:07:34] +[titan] 2025-07-09 23:29:37,050 - root - INFO - step: 44975 loss: 16.3286 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8177 +[titan] 2025-07-09 23:29:37,051 - root - INFO - lr: 1.8855e-04 gnorm: 0.78 [ 9:54:37<12:07:29] +[titan] 2025-07-09 23:29:40,968 - root - INFO - step: 44980 loss: 16.3738 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-09 23:29:40,968 - root - INFO - lr: 1.8852e-04 gnorm: 0.75 [ 9:54:41<12:07:25] +[titan] 2025-07-09 23:29:44,901 - root - INFO - step: 44985 loss: 16.4195 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9027 +[titan] 2025-07-09 23:29:44,902 - root - INFO - lr: 1.8850e-04 gnorm: 0.80 [ 9:54:45<12:07:21] +[titan] 2025-07-09 23:29:48,817 - root - INFO - step: 44990 loss: 16.4777 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9375 +[titan] 2025-07-09 23:29:48,817 - root - INFO - lr: 1.8848e-04 gnorm: 0.87 [ 9:54:49<12:07:17] +[titan] 2025-07-09 23:29:52,725 - root - INFO - step: 44995 loss: 16.5518 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0043 +[titan] 2025-07-09 23:29:52,725 - root - INFO - lr: 1.8846e-04 gnorm: 0.74 [ 9:54:53<12:07:13] +[titan] 2025-07-09 23:29:55,837 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:29:56,633 - root - INFO - step: 45000 loss: 16.7732 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5992 global_avg_mtp_loss: 14.1740 +[titan] 2025-07-09 23:29:56,633 - root - INFO - lr: 1.8844e-04 gnorm: 0.75 [ 9:54:57<12:07:09] +[titan] 2025-07-09 23:30:00,572 - root - INFO - step: 45005 loss: 16.5428 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-09 23:30:00,572 - root - INFO - lr: 1.8842e-04 gnorm: 0.78 [ 9:55:01<12:07:05] +[titan] 2025-07-09 23:30:04,475 - root - INFO - step: 45010 loss: 16.3547 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.8212 +[titan] 2025-07-09 23:30:04,475 - root - INFO - lr: 1.8840e-04 gnorm: 0.86 [ 9:55:04<12:07:01] +[titan] 2025-07-09 23:30:08,393 - root - INFO - step: 45015 loss: 16.6633 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.0881 +[titan] 2025-07-09 23:30:08,394 - root - INFO - lr: 1.8838e-04 gnorm: 0.77 [ 9:55:08<12:06:57] +[titan] 2025-07-09 23:30:12,305 - root - INFO - step: 45020 loss: 16.6655 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.0932 +[titan] 2025-07-09 23:30:12,305 - root - INFO - lr: 1.8836e-04 gnorm: 0.79 [ 9:55:12<12:06:53] +[titan] 2025-07-09 23:30:16,210 - root - INFO - step: 45025 loss: 16.6784 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5854 global_avg_mtp_loss: 14.0930 +[titan] 2025-07-09 23:30:16,211 - root - INFO - lr: 1.8833e-04 gnorm: 0.81 [ 9:55:16<12:06:49] +[titan] 2025-07-09 23:30:20,174 - root - INFO - step: 45030 loss: 16.5422 memory: 44.58GiB(31.99%) tps: 82,674 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9969 +[titan] 2025-07-09 23:30:20,174 - root - INFO - lr: 1.8831e-04 gnorm: 0.75 [ 9:55:20<12:06:45] +[titan] 2025-07-09 23:30:24,085 - root - INFO - step: 45035 loss: 16.6295 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5573 global_avg_mtp_loss: 14.0722 +[titan] 2025-07-09 23:30:24,085 - root - INFO - lr: 1.8829e-04 gnorm: 0.78 [ 9:55:24<12:06:41] +[titan] 2025-07-09 23:30:28,038 - root - INFO - step: 45040 loss: 16.6155 memory: 44.58GiB(31.99%) tps: 82,897 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0621 +[titan] 2025-07-09 23:30:28,039 - root - INFO - lr: 1.8827e-04 gnorm: 0.80 [ 9:55:28<12:06:37] +[titan] 2025-07-09 23:30:31,966 - root - INFO - step: 45045 loss: 16.5565 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 14.0054 +[titan] 2025-07-09 23:30:31,966 - root - INFO - lr: 1.8825e-04 gnorm: 0.82 [ 9:55:32<12:06:33] +[titan] 2025-07-09 23:30:35,100 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:30:35,894 - root - INFO - step: 45050 loss: 16.8825 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6289 global_avg_mtp_loss: 14.2535 +[titan] 2025-07-09 23:30:35,894 - root - INFO - lr: 1.8823e-04 gnorm: 0.78 [ 9:55:36<12:06:29] +[titan] 2025-07-09 23:30:39,870 - root - INFO - step: 45055 loss: 16.5287 memory: 44.58GiB(31.99%) tps: 82,420 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9787 +[titan] 2025-07-09 23:30:39,870 - root - INFO - lr: 1.8821e-04 gnorm: 0.86 [ 9:55:40<12:06:25] +[titan] 2025-07-09 23:30:40,804 - root - INFO - Dumping profiler traces at step 45056 +[titan] 2025-07-09 23:30:40,835 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:30:43,991 - root - INFO - step: 45060 loss: 16.1552 memory: 44.58GiB(31.99%) tps: 79,512 tflops: 274.41 mfu: 27.75% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.6620 +[titan] 2025-07-09 23:30:43,992 - root - INFO - lr: 1.8819e-04 gnorm: 0.80 [ 9:55:44<12:06:21] +[titan] 2025-07-09 23:30:47,892 - root - INFO - step: 45065 loss: 16.1502 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.6729 +[titan] 2025-07-09 23:30:47,892 - root - INFO - lr: 1.8817e-04 gnorm: 0.79 [ 9:55:48<12:06:17] +[titan] 2025-07-09 23:30:51,804 - root - INFO - step: 45070 loss: 16.5593 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 14.0115 +[titan] 2025-07-09 23:30:51,804 - root - INFO - lr: 1.8814e-04 gnorm: 0.83 [ 9:55:52<12:06:13] +[titan] 2025-07-09 23:30:55,745 - root - INFO - step: 45075 loss: 16.6641 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0971 +[titan] 2025-07-09 23:30:55,745 - root - INFO - lr: 1.8812e-04 gnorm: 0.84 [ 9:55:56<12:06:09] +[titan] 2025-07-09 23:30:59,680 - root - INFO - step: 45080 loss: 16.8504 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.6021 global_avg_mtp_loss: 14.2484 +[titan] 2025-07-09 23:30:59,680 - root - INFO - lr: 1.8810e-04 gnorm: 0.82 [ 9:56:00<12:06:05] +[titan] 2025-07-09 23:31:03,622 - root - INFO - step: 45085 loss: 16.6765 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5759 global_avg_mtp_loss: 14.1007 +[titan] 2025-07-09 23:31:03,623 - root - INFO - lr: 1.8808e-04 gnorm: 0.83 [ 9:56:04<12:06:01] +[titan] 2025-07-09 23:31:07,529 - root - INFO - step: 45090 loss: 16.2414 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.7405 +[titan] 2025-07-09 23:31:07,529 - root - INFO - lr: 1.8806e-04 gnorm: 0.87 [ 9:56:08<12:05:57] +[titan] 2025-07-09 23:31:11,431 - root - INFO - step: 45095 loss: 16.5585 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 13.9890 +[titan] 2025-07-09 23:31:11,431 - root - INFO - lr: 1.8804e-04 gnorm: 0.80 [ 9:56:11<12:05:53] +[titan] 2025-07-09 23:31:14,571 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:31:15,411 - root - INFO - step: 45100 loss: 16.5128 memory: 44.58GiB(31.99%) tps: 82,333 tflops: 284.15 mfu: 28.73% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 13.9614 +[titan] 2025-07-09 23:31:15,411 - root - INFO - lr: 1.8802e-04 gnorm: 0.75 [ 9:56:15<12:05:49] +[titan] 2025-07-09 23:31:19,343 - root - INFO - step: 45105 loss: 16.5760 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0182 +[titan] 2025-07-09 23:31:19,343 - root - INFO - lr: 1.8800e-04 gnorm: 0.79 [ 9:56:19<12:05:45] +[titan] 2025-07-09 23:31:23,260 - root - INFO - step: 45110 loss: 16.9537 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6383 global_avg_mtp_loss: 14.3154 +[titan] 2025-07-09 23:31:23,261 - root - INFO - lr: 1.8798e-04 gnorm: 0.84 [ 9:56:23<12:05:41] +[titan] 2025-07-09 23:31:27,189 - root - INFO - step: 45115 loss: 16.4334 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.8983 +[titan] 2025-07-09 23:31:27,189 - root - INFO - lr: 1.8795e-04 gnorm: 0.77 [ 9:56:27<12:05:37] +[titan] 2025-07-09 23:31:31,097 - root - INFO - step: 45120 loss: 16.6109 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0355 +[titan] 2025-07-09 23:31:31,097 - root - INFO - lr: 1.8793e-04 gnorm: 0.81 [ 9:56:31<12:05:33] +[titan] 2025-07-09 23:31:35,000 - root - INFO - step: 45125 loss: 16.7776 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.1653 +[titan] 2025-07-09 23:31:35,000 - root - INFO - lr: 1.8791e-04 gnorm: 0.75 [ 9:56:35<12:05:29] +[titan] 2025-07-09 23:31:38,938 - root - INFO - step: 45130 loss: 16.3715 memory: 44.58GiB(31.99%) tps: 83,199 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8664 +[titan] 2025-07-09 23:31:38,939 - root - INFO - lr: 1.8789e-04 gnorm: 0.80 [ 9:56:39<12:05:25] +[titan] 2025-07-09 23:31:42,851 - root - INFO - step: 45135 loss: 16.5237 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 13.9774 +[titan] 2025-07-09 23:31:42,851 - root - INFO - lr: 1.8787e-04 gnorm: 0.79 [ 9:56:43<12:05:21] +[titan] 2025-07-09 23:31:46,738 - root - INFO - step: 45140 loss: 16.6903 memory: 44.58GiB(31.99%) tps: 84,319 tflops: 291.00 mfu: 29.42% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1055 +[titan] 2025-07-09 23:31:46,738 - root - INFO - lr: 1.8785e-04 gnorm: 0.78 [ 9:56:47<12:05:17] +[titan] 2025-07-09 23:31:50,666 - root - INFO - step: 45145 loss: 16.5998 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0388 +[titan] 2025-07-09 23:31:50,666 - root - INFO - lr: 1.8783e-04 gnorm: 0.82 [ 9:56:51<12:05:13] +[titan] 2025-07-09 23:31:53,778 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:31:54,570 - root - INFO - step: 45150 loss: 16.6980 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1085 +[titan] 2025-07-09 23:31:54,570 - root - INFO - lr: 1.8781e-04 gnorm: 0.80 [ 9:56:55<12:05:09] +[titan] 2025-07-09 23:31:58,499 - root - INFO - step: 45155 loss: 16.4788 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 13.9367 +[titan] 2025-07-09 23:31:58,500 - root - INFO - lr: 1.8779e-04 gnorm: 0.79 [ 9:56:58<12:05:05] +[titan] 2025-07-09 23:32:02,407 - root - INFO - step: 45160 loss: 16.5956 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0474 +[titan] 2025-07-09 23:32:02,407 - root - INFO - lr: 1.8776e-04 gnorm: 0.83 [ 9:57:02<12:05:01] +[titan] 2025-07-09 23:32:06,342 - root - INFO - step: 45165 loss: 16.5312 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 13.9760 +[titan] 2025-07-09 23:32:06,342 - root - INFO - lr: 1.8774e-04 gnorm: 0.79 [ 9:57:06<12:04:57] +[titan] 2025-07-09 23:32:10,260 - root - INFO - step: 45170 loss: 16.8144 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5895 global_avg_mtp_loss: 14.2249 +[titan] 2025-07-09 23:32:10,260 - root - INFO - lr: 1.8772e-04 gnorm: 0.78 [ 9:57:10<12:04:53] +[titan] 2025-07-09 23:32:14,164 - root - INFO - step: 45175 loss: 16.4788 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9423 +[titan] 2025-07-09 23:32:14,165 - root - INFO - lr: 1.8770e-04 gnorm: 0.79 [ 9:57:14<12:04:49] +[titan] 2025-07-09 23:32:18,090 - root - INFO - step: 45180 loss: 16.4826 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 13.9406 +[titan] 2025-07-09 23:32:18,090 - root - INFO - lr: 1.8768e-04 gnorm: 0.84 [ 9:57:18<12:04:45] +[titan] 2025-07-09 23:32:22,037 - root - INFO - step: 45185 loss: 16.4729 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5378 global_avg_mtp_loss: 13.9351 +[titan] 2025-07-09 23:32:22,038 - root - INFO - lr: 1.8766e-04 gnorm: 0.80 [ 9:57:22<12:04:41] +[titan] 2025-07-09 23:32:25,965 - root - INFO - step: 45190 loss: 16.5649 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0144 +[titan] 2025-07-09 23:32:25,965 - root - INFO - lr: 1.8764e-04 gnorm: 0.81 [ 9:57:26<12:04:37] +[titan] 2025-07-09 23:32:29,882 - root - INFO - step: 45195 loss: 16.5983 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0418 +[titan] 2025-07-09 23:32:29,883 - root - INFO - lr: 1.8762e-04 gnorm: 0.81 [ 9:57:30<12:04:33] +[titan] 2025-07-09 23:32:32,994 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:32:33,783 - root - INFO - step: 45200 loss: 16.5184 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9766 +[titan] 2025-07-09 23:32:33,783 - root - INFO - lr: 1.8760e-04 gnorm: 0.80 [ 9:57:34<12:04:29] +[titan] 2025-07-09 23:32:37,681 - root - INFO - step: 45205 loss: 16.9382 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6260 global_avg_mtp_loss: 14.3122 +[titan] 2025-07-09 23:32:37,681 - root - INFO - lr: 1.8757e-04 gnorm: 0.83 [ 9:57:38<12:04:25] +[titan] 2025-07-09 23:32:41,577 - root - INFO - step: 45210 loss: 16.5999 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0462 +[titan] 2025-07-09 23:32:41,577 - root - INFO - lr: 1.8755e-04 gnorm: 0.82 [ 9:57:42<12:04:21] +[titan] 2025-07-09 23:32:45,518 - root - INFO - step: 45215 loss: 16.5062 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.9678 +[titan] 2025-07-09 23:32:45,518 - root - INFO - lr: 1.8753e-04 gnorm: 0.76 [ 9:57:46<12:04:17] +[titan] 2025-07-09 23:32:49,457 - root - INFO - step: 45220 loss: 16.6675 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.1084 +[titan] 2025-07-09 23:32:49,457 - root - INFO - lr: 1.8751e-04 gnorm: 0.91 [ 9:57:49<12:04:13] +[titan] 2025-07-09 23:32:53,406 - root - INFO - step: 45225 loss: 16.5715 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 14.0250 +[titan] 2025-07-09 23:32:53,406 - root - INFO - lr: 1.8749e-04 gnorm: 0.82 [ 9:57:53<12:04:09] +[titan] 2025-07-09 23:32:57,354 - root - INFO - step: 45230 loss: 16.6510 memory: 44.58GiB(31.99%) tps: 83,004 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-09 23:32:57,355 - root - INFO - lr: 1.8747e-04 gnorm: 0.85 [ 9:57:57<12:04:05] +[titan] 2025-07-09 23:33:01,279 - root - INFO - step: 45235 loss: 16.5068 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 13.9570 +[titan] 2025-07-09 23:33:01,279 - root - INFO - lr: 1.8745e-04 gnorm: 0.83 [ 9:58:01<12:04:01] +[titan] 2025-07-09 23:33:05,199 - root - INFO - step: 45240 loss: 16.6427 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0790 +[titan] 2025-07-09 23:33:05,200 - root - INFO - lr: 1.8743e-04 gnorm: 0.83 [ 9:58:05<12:03:57] +[titan] 2025-07-09 23:33:09,093 - root - INFO - step: 45245 loss: 16.6170 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5653 global_avg_mtp_loss: 14.0518 +[titan] 2025-07-09 23:33:09,094 - root - INFO - lr: 1.8741e-04 gnorm: 0.86 [ 9:58:09<12:03:53] +[titan] 2025-07-09 23:33:12,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:33:13,042 - root - INFO - step: 45250 loss: 16.7717 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.5940 global_avg_mtp_loss: 14.1777 +[titan] 2025-07-09 23:33:13,043 - root - INFO - lr: 1.8738e-04 gnorm: 0.82 [ 9:58:13<12:03:49] +[titan] 2025-07-09 23:33:16,945 - root - INFO - step: 45255 loss: 16.6322 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0652 +[titan] 2025-07-09 23:33:16,945 - root - INFO - lr: 1.8736e-04 gnorm: 0.85 [ 9:58:17<12:03:45] +[titan] 2025-07-09 23:33:20,846 - root - INFO - step: 45260 loss: 16.8191 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6046 global_avg_mtp_loss: 14.2145 +[titan] 2025-07-09 23:33:20,846 - root - INFO - lr: 1.8734e-04 gnorm: 0.79 [ 9:58:21<12:03:41] +[titan] 2025-07-09 23:33:24,767 - root - INFO - step: 45265 loss: 16.5383 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5484 global_avg_mtp_loss: 13.9898 +[titan] 2025-07-09 23:33:24,767 - root - INFO - lr: 1.8732e-04 gnorm: 0.87 [ 9:58:25<12:03:37] +[titan] 2025-07-09 23:33:28,675 - root - INFO - step: 45270 loss: 16.6931 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5784 global_avg_mtp_loss: 14.1147 +[titan] 2025-07-09 23:33:28,675 - root - INFO - lr: 1.8730e-04 gnorm: 0.81 [ 9:58:29<12:03:33] +[titan] 2025-07-09 23:33:32,600 - root - INFO - step: 45275 loss: 16.7372 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.1639 +[titan] 2025-07-09 23:33:32,600 - root - INFO - lr: 1.8728e-04 gnorm: 0.80 [ 9:58:33<12:03:29] +[titan] 2025-07-09 23:33:36,503 - root - INFO - step: 45280 loss: 16.6437 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.0668 +[titan] 2025-07-09 23:33:36,504 - root - INFO - lr: 1.8726e-04 gnorm: 0.86 [ 9:58:36<12:03:24] +[titan] 2025-07-09 23:33:40,401 - root - INFO - step: 45285 loss: 16.6795 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.0949 +[titan] 2025-07-09 23:33:40,401 - root - INFO - lr: 1.8724e-04 gnorm: 0.84 [ 9:58:40<12:03:20] +[titan] 2025-07-09 23:33:44,313 - root - INFO - step: 45290 loss: 16.5758 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0121 +[titan] 2025-07-09 23:33:44,313 - root - INFO - lr: 1.8722e-04 gnorm: 0.90 [ 9:58:44<12:03:16] +[titan] 2025-07-09 23:33:48,217 - root - INFO - step: 45295 loss: 16.5867 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0320 +[titan] 2025-07-09 23:33:48,217 - root - INFO - lr: 1.8719e-04 gnorm: 0.87 [ 9:58:48<12:03:12] +[titan] 2025-07-09 23:33:51,347 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:33:52,136 - root - INFO - step: 45300 loss: 16.7239 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5845 global_avg_mtp_loss: 14.1394 +[titan] 2025-07-09 23:33:52,136 - root - INFO - lr: 1.8717e-04 gnorm: 0.81 [ 9:58:52<12:03:08] +[titan] 2025-07-09 23:33:56,056 - root - INFO - step: 45305 loss: 16.7655 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1792 +[titan] 2025-07-09 23:33:56,057 - root - INFO - lr: 1.8715e-04 gnorm: 0.85 [ 9:58:56<12:03:04] +[titan] 2025-07-09 23:34:00,022 - root - INFO - step: 45310 loss: 16.2923 memory: 44.58GiB(31.99%) tps: 82,644 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.7931 +[titan] 2025-07-09 23:34:00,022 - root - INFO - lr: 1.8713e-04 gnorm: 0.85 [ 9:59:00<12:03:00] +[titan] 2025-07-09 23:34:03,945 - root - INFO - step: 45315 loss: 16.5133 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9872 +[titan] 2025-07-09 23:34:03,945 - root - INFO - lr: 1.8711e-04 gnorm: 0.80 [ 9:59:04<12:02:56] +[titan] 2025-07-09 23:34:07,884 - root - INFO - step: 45320 loss: 16.8324 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.6110 global_avg_mtp_loss: 14.2215 +[titan] 2025-07-09 23:34:07,884 - root - INFO - lr: 1.8709e-04 gnorm: 0.75 [ 9:59:08<12:02:52] +[titan] 2025-07-09 23:34:11,797 - root - INFO - step: 45325 loss: 16.6502 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.0815 +[titan] 2025-07-09 23:34:11,798 - root - INFO - lr: 1.8707e-04 gnorm: 0.76 [ 9:59:12<12:02:48] +[titan] 2025-07-09 23:34:15,707 - root - INFO - step: 45330 loss: 16.5462 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 13.9923 +[titan] 2025-07-09 23:34:15,707 - root - INFO - lr: 1.8705e-04 gnorm: 0.78 [ 9:59:16<12:02:44] +[titan] 2025-07-09 23:34:19,632 - root - INFO - step: 45335 loss: 16.7024 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.1314 +[titan] 2025-07-09 23:34:19,632 - root - INFO - lr: 1.8703e-04 gnorm: 0.83 [ 9:59:20<12:02:40] +[titan] 2025-07-09 23:34:23,559 - root - INFO - step: 45340 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 13.8885 +[titan] 2025-07-09 23:34:23,560 - root - INFO - lr: 1.8700e-04 gnorm: 0.79 [ 9:59:24<12:02:36] +[titan] 2025-07-09 23:34:27,481 - root - INFO - step: 45345 loss: 16.7893 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6096 global_avg_mtp_loss: 14.1798 +[titan] 2025-07-09 23:34:27,481 - root - INFO - lr: 1.8698e-04 gnorm: 0.79 [ 9:59:27<12:02:32] +[titan] 2025-07-09 23:34:30,634 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:34:31,438 - root - INFO - step: 45350 loss: 16.5091 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 13.9577 +[titan] 2025-07-09 23:34:31,439 - root - INFO - lr: 1.8696e-04 gnorm: 0.78 [ 9:59:31<12:02:28] +[titan] 2025-07-09 23:34:35,347 - root - INFO - step: 45355 loss: 16.7467 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.1576 +[titan] 2025-07-09 23:34:35,347 - root - INFO - lr: 1.8694e-04 gnorm: 0.81 [ 9:59:35<12:02:24] +[titan] 2025-07-09 23:34:39,311 - root - INFO - step: 45360 loss: 16.3348 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8260 +[titan] 2025-07-09 23:34:39,311 - root - INFO - lr: 1.8692e-04 gnorm: 0.81 [ 9:59:39<12:02:20] +[titan] 2025-07-09 23:34:43,221 - root - INFO - step: 45365 loss: 16.6908 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1214 +[titan] 2025-07-09 23:34:43,221 - root - INFO - lr: 1.8690e-04 gnorm: 0.77 [ 9:59:43<12:02:16] +[titan] 2025-07-09 23:34:47,174 - root - INFO - step: 45370 loss: 16.5767 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 14.0138 +[titan] 2025-07-09 23:34:47,175 - root - INFO - lr: 1.8688e-04 gnorm: 0.73 [ 9:59:47<12:02:12] +[titan] 2025-07-09 23:34:51,097 - root - INFO - step: 45375 loss: 16.4243 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.8950 +[titan] 2025-07-09 23:34:51,097 - root - INFO - lr: 1.8686e-04 gnorm: 0.76 [ 9:59:51<12:02:08] +[titan] 2025-07-09 23:34:55,029 - root - INFO - step: 45380 loss: 16.3605 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.8357 +[titan] 2025-07-09 23:34:55,029 - root - INFO - lr: 1.8684e-04 gnorm: 0.87 [ 9:59:55<12:02:04] +[titan] 2025-07-09 23:34:58,953 - root - INFO - step: 45385 loss: 16.3839 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-09 23:34:58,953 - root - INFO - lr: 1.8681e-04 gnorm: 0.84 [ 9:59:59<12:02:00] +[titan] 2025-07-09 23:35:02,863 - root - INFO - step: 45390 loss: 16.6660 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0963 +[titan] 2025-07-09 23:35:02,863 - root - INFO - lr: 1.8679e-04 gnorm: 0.80 [10:00:03<12:01:56] +[titan] 2025-07-09 23:35:06,768 - root - INFO - step: 45395 loss: 16.7033 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.1289 +[titan] 2025-07-09 23:35:06,768 - root - INFO - lr: 1.8677e-04 gnorm: 0.82 [10:00:07<12:01:52] +[titan] 2025-07-09 23:35:09,884 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:35:10,671 - root - INFO - step: 45400 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8868 +[titan] 2025-07-09 23:35:10,671 - root - INFO - lr: 1.8675e-04 gnorm: 0.80 [10:00:11<12:01:48] +[titan] 2025-07-09 23:35:14,569 - root - INFO - step: 45405 loss: 16.3122 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5033 global_avg_mtp_loss: 13.8089 +[titan] 2025-07-09 23:35:14,569 - root - INFO - lr: 1.8673e-04 gnorm: 0.78 [10:00:15<12:01:44] +[titan] 2025-07-09 23:35:18,518 - root - INFO - step: 45410 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 82,976 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9137 +[titan] 2025-07-09 23:35:18,518 - root - INFO - lr: 1.8671e-04 gnorm: 0.80 [10:00:18<12:01:40] +[titan] 2025-07-09 23:35:22,431 - root - INFO - step: 45415 loss: 16.5139 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5509 global_avg_mtp_loss: 13.9630 +[titan] 2025-07-09 23:35:22,431 - root - INFO - lr: 1.8669e-04 gnorm: 0.82 [10:00:22<12:01:36] +[titan] 2025-07-09 23:35:26,365 - root - INFO - step: 45420 loss: 16.4514 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9140 +[titan] 2025-07-09 23:35:26,365 - root - INFO - lr: 1.8667e-04 gnorm: 0.81 [10:00:26<12:01:32] +[titan] 2025-07-09 23:35:30,305 - root - INFO - step: 45425 loss: 16.7719 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.6063 global_avg_mtp_loss: 14.1656 +[titan] 2025-07-09 23:35:30,306 - root - INFO - lr: 1.8665e-04 gnorm: 0.77 [10:00:30<12:01:28] +[titan] 2025-07-09 23:35:34,226 - root - INFO - step: 45430 loss: 16.3989 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.8707 +[titan] 2025-07-09 23:35:34,226 - root - INFO - lr: 1.8662e-04 gnorm: 0.79 [10:00:34<12:01:24] +[titan] 2025-07-09 23:35:38,132 - root - INFO - step: 45435 loss: 16.5641 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 14.0127 +[titan] 2025-07-09 23:35:38,132 - root - INFO - lr: 1.8660e-04 gnorm: 0.81 [10:00:38<12:01:20] +[titan] 2025-07-09 23:35:42,084 - root - INFO - step: 45440 loss: 16.5637 memory: 44.58GiB(31.99%) tps: 82,920 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 13.9899 +[titan] 2025-07-09 23:35:42,084 - root - INFO - lr: 1.8658e-04 gnorm: 0.82 [10:00:42<12:01:16] +[titan] 2025-07-09 23:35:45,998 - root - INFO - step: 45445 loss: 16.9400 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6423 global_avg_mtp_loss: 14.2977 +[titan] 2025-07-09 23:35:45,998 - root - INFO - lr: 1.8656e-04 gnorm: 0.79 [10:00:46<12:01:12] +[titan] 2025-07-09 23:35:49,149 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:35:49,935 - root - INFO - step: 45450 loss: 16.5450 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9995 +[titan] 2025-07-09 23:35:49,935 - root - INFO - lr: 1.8654e-04 gnorm: 0.79 [10:00:50<12:01:08] +[titan] 2025-07-09 23:35:53,853 - root - INFO - step: 45455 loss: 16.6439 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5680 global_avg_mtp_loss: 14.0760 +[titan] 2025-07-09 23:35:53,853 - root - INFO - lr: 1.8652e-04 gnorm: 0.75 [10:00:54<12:01:04] +[titan] 2025-07-09 23:35:57,779 - root - INFO - step: 45460 loss: 16.6815 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.1054 +[titan] 2025-07-09 23:35:57,779 - root - INFO - lr: 1.8650e-04 gnorm: 0.73 [10:00:58<12:01:00] +[titan] 2025-07-09 23:36:01,697 - root - INFO - step: 45465 loss: 16.6041 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0472 +[titan] 2025-07-09 23:36:01,697 - root - INFO - lr: 1.8648e-04 gnorm: 0.79 [10:01:02<12:00:56] +[titan] 2025-07-09 23:36:05,625 - root - INFO - step: 45470 loss: 16.4204 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.8964 +[titan] 2025-07-09 23:36:05,625 - root - INFO - lr: 1.8645e-04 gnorm: 0.85 [10:01:06<12:00:52] +[titan] 2025-07-09 23:36:09,537 - root - INFO - step: 45475 loss: 16.4754 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9397 +[titan] 2025-07-09 23:36:09,538 - root - INFO - lr: 1.8643e-04 gnorm: 0.78 [10:01:10<12:00:48] +[titan] 2025-07-09 23:36:13,463 - root - INFO - step: 45480 loss: 16.3558 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.8571 +[titan] 2025-07-09 23:36:13,463 - root - INFO - lr: 1.8641e-04 gnorm: 0.82 [10:01:13<12:00:44] +[titan] 2025-07-09 23:36:17,380 - root - INFO - step: 45485 loss: 16.5549 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 13.9905 +[titan] 2025-07-09 23:36:17,381 - root - INFO - lr: 1.8639e-04 gnorm: 0.79 [10:01:17<12:00:40] +[titan] 2025-07-09 23:36:21,304 - root - INFO - step: 45490 loss: 16.5186 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9798 +[titan] 2025-07-09 23:36:21,305 - root - INFO - lr: 1.8637e-04 gnorm: 0.73 [10:01:21<12:00:36] +[titan] 2025-07-09 23:36:25,260 - root - INFO - step: 45495 loss: 16.6598 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0889 +[titan] 2025-07-09 23:36:25,260 - root - INFO - lr: 1.8635e-04 gnorm: 0.78 [10:01:25<12:00:32] +[titan] 2025-07-09 23:36:28,403 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:36:29,190 - root - INFO - step: 45500 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 14.0239 +[titan] 2025-07-09 23:36:29,191 - root - INFO - lr: 1.8633e-04 gnorm: 0.79 [10:01:29<12:00:28] +[titan] 2025-07-09 23:36:33,100 - root - INFO - step: 45505 loss: 16.8727 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6073 global_avg_mtp_loss: 14.2654 +[titan] 2025-07-09 23:36:33,100 - root - INFO - lr: 1.8631e-04 gnorm: 1.05 [10:01:33<12:00:24] +[titan] 2025-07-09 23:36:37,024 - root - INFO - step: 45510 loss: 16.4084 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-09 23:36:37,025 - root - INFO - lr: 1.8629e-04 gnorm: 0.80 [10:01:37<12:00:20] +[titan] 2025-07-09 23:36:40,965 - root - INFO - step: 45515 loss: 16.7213 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5732 global_avg_mtp_loss: 14.1481 +[titan] 2025-07-09 23:36:40,965 - root - INFO - lr: 1.8626e-04 gnorm: 0.95 [10:01:41<12:00:16] +[titan] 2025-07-09 23:36:44,872 - root - INFO - step: 45520 loss: 16.4797 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9417 +[titan] 2025-07-09 23:36:44,872 - root - INFO - lr: 1.8624e-04 gnorm: 0.83 [10:01:45<12:00:12] +[titan] 2025-07-09 23:36:48,803 - root - INFO - step: 45525 loss: 16.6254 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-09 23:36:48,803 - root - INFO - lr: 1.8622e-04 gnorm: 0.97 [10:01:49<12:00:08] +[titan] 2025-07-09 23:36:52,726 - root - INFO - step: 45530 loss: 16.4461 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9156 +[titan] 2025-07-09 23:36:52,726 - root - INFO - lr: 1.8620e-04 gnorm: 0.82 [10:01:53<12:00:04] +[titan] 2025-07-09 23:36:56,640 - root - INFO - step: 45535 loss: 16.5340 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9905 +[titan] 2025-07-09 23:36:56,640 - root - INFO - lr: 1.8618e-04 gnorm: 0.75 [10:01:57<12:00:00] +[titan] 2025-07-09 23:37:00,556 - root - INFO - step: 45540 loss: 16.6412 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.0674 +[titan] 2025-07-09 23:37:00,557 - root - INFO - lr: 1.8616e-04 gnorm: 0.74 [10:02:01<11:59:56] +[titan] 2025-07-09 23:37:04,471 - root - INFO - step: 45545 loss: 16.4524 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 13.9178 +[titan] 2025-07-09 23:37:04,471 - root - INFO - lr: 1.8614e-04 gnorm: 0.86 [10:02:04<11:59:52] +[titan] 2025-07-09 23:37:07,625 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:37:08,415 - root - INFO - step: 45550 loss: 16.6773 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.0998 +[titan] 2025-07-09 23:37:08,416 - root - INFO - lr: 1.8612e-04 gnorm: 0.87 [10:02:08<11:59:48] +[titan] 2025-07-09 23:37:12,360 - root - INFO - step: 45555 loss: 16.6715 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5679 global_avg_mtp_loss: 14.1036 +[titan] 2025-07-09 23:37:12,360 - root - INFO - lr: 1.8610e-04 gnorm: 0.81 [10:02:12<11:59:44] +[titan] 2025-07-09 23:37:16,284 - root - INFO - step: 45560 loss: 16.5603 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 13.9928 +[titan] 2025-07-09 23:37:16,284 - root - INFO - lr: 1.8607e-04 gnorm: 0.75 [10:02:16<11:59:40] +[titan] 2025-07-09 23:37:20,308 - root - INFO - step: 45565 loss: 16.7561 memory: 44.58GiB(31.99%) tps: 81,434 tflops: 281.04 mfu: 28.42% global_avg_ntp_loss: 2.6082 global_avg_mtp_loss: 14.1478 +[titan] 2025-07-09 23:37:20,308 - root - INFO - lr: 1.8605e-04 gnorm: 0.82 [10:02:20<11:59:36] +[titan] 2025-07-09 23:37:22,811 - root - INFO - Dumping profiler traces at step 45568 +[titan] 2025-07-09 23:37:22,841 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:37:24,438 - root - INFO - step: 45570 loss: 16.6102 memory: 44.58GiB(31.99%) tps: 79,355 tflops: 273.87 mfu: 27.69% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0443 +[titan] 2025-07-09 23:37:24,438 - root - INFO - lr: 1.8603e-04 gnorm: 0.80 [10:02:24<11:59:32] +[titan] 2025-07-09 23:37:28,378 - root - INFO - step: 45575 loss: 16.3470 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8277 +[titan] 2025-07-09 23:37:28,378 - root - INFO - lr: 1.8601e-04 gnorm: 0.77 [10:02:28<11:59:28] +[titan] 2025-07-09 23:37:32,298 - root - INFO - step: 45580 loss: 16.5551 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0043 +[titan] 2025-07-09 23:37:32,298 - root - INFO - lr: 1.8599e-04 gnorm: 0.83 [10:02:32<11:59:24] +[titan] 2025-07-09 23:37:36,205 - root - INFO - step: 45585 loss: 16.6901 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1101 +[titan] 2025-07-09 23:37:36,205 - root - INFO - lr: 1.8597e-04 gnorm: 0.81 [10:02:36<11:59:20] +[titan] 2025-07-09 23:37:40,112 - root - INFO - step: 45590 loss: 16.7670 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.1838 +[titan] 2025-07-09 23:37:40,112 - root - INFO - lr: 1.8595e-04 gnorm: 0.89 [10:02:40<11:59:16] +[titan] 2025-07-09 23:37:44,019 - root - INFO - step: 45595 loss: 16.6635 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5798 global_avg_mtp_loss: 14.0837 +[titan] 2025-07-09 23:37:44,019 - root - INFO - lr: 1.8593e-04 gnorm: 0.80 [10:02:44<11:59:12] +[titan] 2025-07-09 23:37:47,143 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:37:47,930 - root - INFO - step: 45600 loss: 16.6831 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.1049 +[titan] 2025-07-09 23:37:47,931 - root - INFO - lr: 1.8590e-04 gnorm: 0.72 [10:02:48<11:59:08] +[titan] 2025-07-09 23:37:51,878 - root - INFO - step: 45605 loss: 16.4782 memory: 44.58GiB(31.99%) tps: 83,007 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9458 +[titan] 2025-07-09 23:37:51,879 - root - INFO - lr: 1.8588e-04 gnorm: 0.81 [10:02:52<11:59:04] +[titan] 2025-07-09 23:37:55,791 - root - INFO - step: 45610 loss: 16.5021 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 13.9517 +[titan] 2025-07-09 23:37:55,791 - root - INFO - lr: 1.8586e-04 gnorm: 0.82 [10:02:56<11:59:00] +[titan] 2025-07-09 23:37:59,703 - root - INFO - step: 45615 loss: 16.3188 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8121 +[titan] 2025-07-09 23:37:59,703 - root - INFO - lr: 1.8584e-04 gnorm: 0.81 [10:03:00<11:58:56] +[titan] 2025-07-09 23:38:03,614 - root - INFO - step: 45620 loss: 16.5554 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 14.0133 +[titan] 2025-07-09 23:38:03,614 - root - INFO - lr: 1.8582e-04 gnorm: 0.79 [10:03:04<11:58:52] +[titan] 2025-07-09 23:38:07,509 - root - INFO - step: 45625 loss: 16.6045 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-09 23:38:07,509 - root - INFO - lr: 1.8580e-04 gnorm: 0.82 [10:03:07<11:58:48] +[titan] 2025-07-09 23:38:11,398 - root - INFO - step: 45630 loss: 16.6645 memory: 44.58GiB(31.99%) tps: 84,267 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0961 +[titan] 2025-07-09 23:38:11,398 - root - INFO - lr: 1.8578e-04 gnorm: 0.80 [10:03:11<11:58:44] +[titan] 2025-07-09 23:38:15,331 - root - INFO - step: 45635 loss: 16.4306 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.8905 +[titan] 2025-07-09 23:38:15,331 - root - INFO - lr: 1.8576e-04 gnorm: 0.82 [10:03:15<11:58:40] +[titan] 2025-07-09 23:38:19,266 - root - INFO - step: 45640 loss: 16.7596 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1750 +[titan] 2025-07-09 23:38:19,266 - root - INFO - lr: 1.8574e-04 gnorm: 0.77 [10:03:19<11:58:36] +[titan] 2025-07-09 23:38:23,210 - root - INFO - step: 45645 loss: 16.5652 memory: 44.58GiB(31.99%) tps: 83,093 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 14.0139 +[titan] 2025-07-09 23:38:23,210 - root - INFO - lr: 1.8571e-04 gnorm: 0.76 [10:03:23<11:58:32] +[titan] 2025-07-09 23:38:26,344 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:38:27,132 - root - INFO - step: 45650 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0979 +[titan] 2025-07-09 23:38:27,132 - root - INFO - lr: 1.8569e-04 gnorm: 0.77 [10:03:27<11:58:28] +[titan] 2025-07-09 23:38:31,066 - root - INFO - step: 45655 loss: 16.5849 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0195 +[titan] 2025-07-09 23:38:31,066 - root - INFO - lr: 1.8567e-04 gnorm: 0.76 [10:03:31<11:58:24] +[titan] 2025-07-09 23:38:34,992 - root - INFO - step: 45660 loss: 16.6517 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5799 global_avg_mtp_loss: 14.0719 +[titan] 2025-07-09 23:38:34,992 - root - INFO - lr: 1.8565e-04 gnorm: 0.82 [10:03:35<11:58:20] +[titan] 2025-07-09 23:38:38,934 - root - INFO - step: 45665 loss: 16.5081 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 13.9562 +[titan] 2025-07-09 23:38:38,935 - root - INFO - lr: 1.8563e-04 gnorm: 0.86 [10:03:39<11:58:16] +[titan] 2025-07-09 23:38:42,859 - root - INFO - step: 45670 loss: 16.4765 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-09 23:38:42,859 - root - INFO - lr: 1.8561e-04 gnorm: 0.83 [10:03:43<11:58:12] +[titan] 2025-07-09 23:38:46,763 - root - INFO - step: 45675 loss: 16.4678 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.9316 +[titan] 2025-07-09 23:38:46,764 - root - INFO - lr: 1.8559e-04 gnorm: 0.81 [10:03:47<11:58:07] +[titan] 2025-07-09 23:38:50,669 - root - INFO - step: 45680 loss: 16.3106 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.7992 +[titan] 2025-07-09 23:38:50,669 - root - INFO - lr: 1.8557e-04 gnorm: 0.79 [10:03:51<11:58:03] +[titan] 2025-07-09 23:38:54,623 - root - INFO - step: 45685 loss: 16.8074 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.6066 global_avg_mtp_loss: 14.2008 +[titan] 2025-07-09 23:38:54,623 - root - INFO - lr: 1.8554e-04 gnorm: 0.79 [10:03:55<11:57:59] +[titan] 2025-07-09 23:38:58,513 - root - INFO - step: 45690 loss: 16.7423 memory: 44.58GiB(31.99%) tps: 84,246 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.1647 +[titan] 2025-07-09 23:38:58,513 - root - INFO - lr: 1.8552e-04 gnorm: 0.82 [10:03:58<11:57:55] +[titan] 2025-07-09 23:39:02,420 - root - INFO - step: 45695 loss: 16.6984 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.1218 +[titan] 2025-07-09 23:39:02,421 - root - INFO - lr: 1.8550e-04 gnorm: 0.83 [10:04:02<11:57:51] +[titan] 2025-07-09 23:39:05,544 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:39:06,331 - root - INFO - step: 45700 loss: 16.5786 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0190 +[titan] 2025-07-09 23:39:06,332 - root - INFO - lr: 1.8548e-04 gnorm: 0.77 [10:04:06<11:57:47] +[titan] 2025-07-09 23:39:10,246 - root - INFO - step: 45705 loss: 16.4823 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 13.9311 +[titan] 2025-07-09 23:39:10,246 - root - INFO - lr: 1.8546e-04 gnorm: 0.83 [10:04:10<11:57:43] +[titan] 2025-07-09 23:39:14,178 - root - INFO - step: 45710 loss: 16.5327 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.9886 +[titan] 2025-07-09 23:39:14,178 - root - INFO - lr: 1.8544e-04 gnorm: 0.84 [10:04:14<11:57:39] +[titan] 2025-07-09 23:39:18,092 - root - INFO - step: 45715 loss: 16.5331 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9757 +[titan] 2025-07-09 23:39:18,093 - root - INFO - lr: 1.8542e-04 gnorm: 0.86 [10:04:18<11:57:35] +[titan] 2025-07-09 23:39:22,005 - root - INFO - step: 45720 loss: 16.6480 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.0797 +[titan] 2025-07-09 23:39:22,006 - root - INFO - lr: 1.8540e-04 gnorm: 0.80 [10:04:22<11:57:31] +[titan] 2025-07-09 23:39:25,935 - root - INFO - step: 45725 loss: 16.7140 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.1404 +[titan] 2025-07-09 23:39:25,935 - root - INFO - lr: 1.8538e-04 gnorm: 0.78 [10:04:26<11:57:27] +[titan] 2025-07-09 23:39:29,908 - root - INFO - step: 45730 loss: 16.5800 memory: 44.58GiB(31.99%) tps: 82,465 tflops: 284.60 mfu: 28.78% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-09 23:39:29,909 - root - INFO - lr: 1.8535e-04 gnorm: 0.88 [10:04:30<11:57:23] +[titan] 2025-07-09 23:39:33,844 - root - INFO - step: 45735 loss: 16.5233 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9867 +[titan] 2025-07-09 23:39:33,844 - root - INFO - lr: 1.8533e-04 gnorm: 0.77 [10:04:34<11:57:19] +[titan] 2025-07-09 23:39:37,795 - root - INFO - step: 45740 loss: 16.5563 memory: 44.58GiB(31.99%) tps: 82,937 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0107 +[titan] 2025-07-09 23:39:37,796 - root - INFO - lr: 1.8531e-04 gnorm: 0.91 [10:04:38<11:57:15] +[titan] 2025-07-09 23:39:41,696 - root - INFO - step: 45745 loss: 16.6514 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-09 23:39:41,696 - root - INFO - lr: 1.8529e-04 gnorm: 0.84 [10:04:42<11:57:11] +[titan] 2025-07-09 23:39:44,935 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:39:45,725 - root - INFO - step: 45750 loss: 16.0964 memory: 44.58GiB(31.99%) tps: 81,327 tflops: 280.67 mfu: 28.38% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.6253 +[titan] 2025-07-09 23:39:45,725 - root - INFO - lr: 1.8527e-04 gnorm: 0.85 [10:04:46<11:57:07] +[titan] 2025-07-09 23:39:49,646 - root - INFO - step: 45755 loss: 16.7342 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.1634 +[titan] 2025-07-09 23:39:49,646 - root - INFO - lr: 1.8525e-04 gnorm: 0.81 [10:04:50<11:57:03] +[titan] 2025-07-09 23:39:53,561 - root - INFO - step: 45760 loss: 16.3899 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-09 23:39:53,561 - root - INFO - lr: 1.8523e-04 gnorm: 0.89 [10:04:54<11:56:59] +[titan] 2025-07-09 23:39:57,520 - root - INFO - step: 45765 loss: 16.7868 memory: 44.58GiB(31.99%) tps: 82,789 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5955 global_avg_mtp_loss: 14.1913 +[titan] 2025-07-09 23:39:57,520 - root - INFO - lr: 1.8521e-04 gnorm: 0.82 [10:04:57<11:56:55] +[titan] 2025-07-09 23:40:01,414 - root - INFO - step: 45770 loss: 16.7390 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.1615 +[titan] 2025-07-09 23:40:01,414 - root - INFO - lr: 1.8518e-04 gnorm: 0.84 [10:05:01<11:56:51] +[titan] 2025-07-09 23:40:05,326 - root - INFO - step: 45775 loss: 16.7109 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1213 +[titan] 2025-07-09 23:40:05,326 - root - INFO - lr: 1.8516e-04 gnorm: 0.76 [10:05:05<11:56:47] +[titan] 2025-07-09 23:40:09,247 - root - INFO - step: 45780 loss: 16.7284 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.1511 +[titan] 2025-07-09 23:40:09,247 - root - INFO - lr: 1.8514e-04 gnorm: 0.81 [10:05:09<11:56:43] +[titan] 2025-07-09 23:40:13,189 - root - INFO - step: 45785 loss: 16.5893 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0250 +[titan] 2025-07-09 23:40:13,189 - root - INFO - lr: 1.8512e-04 gnorm: 0.82 [10:05:13<11:56:39] +[titan] 2025-07-09 23:40:17,088 - root - INFO - step: 45790 loss: 16.5418 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 13.9902 +[titan] 2025-07-09 23:40:17,088 - root - INFO - lr: 1.8510e-04 gnorm: 0.84 [10:05:17<11:56:35] +[titan] 2025-07-09 23:40:21,002 - root - INFO - step: 45795 loss: 16.1992 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.7088 +[titan] 2025-07-09 23:40:21,002 - root - INFO - lr: 1.8508e-04 gnorm: 0.87 [10:05:21<11:56:31] +[titan] 2025-07-09 23:40:24,130 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:40:24,914 - root - INFO - step: 45800 loss: 16.7092 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1239 +[titan] 2025-07-09 23:40:24,914 - root - INFO - lr: 1.8506e-04 gnorm: 0.82 [10:05:25<11:56:27] +[titan] 2025-07-09 23:40:28,814 - root - INFO - step: 45805 loss: 16.4629 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 13.9197 +[titan] 2025-07-09 23:40:28,814 - root - INFO - lr: 1.8504e-04 gnorm: 0.80 [10:05:29<11:56:23] +[titan] 2025-07-09 23:40:32,709 - root - INFO - step: 45810 loss: 16.4655 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9237 +[titan] 2025-07-09 23:40:32,710 - root - INFO - lr: 1.8502e-04 gnorm: 0.78 [10:05:33<11:56:19] +[titan] 2025-07-09 23:40:36,626 - root - INFO - step: 45815 loss: 16.5633 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 14.0168 +[titan] 2025-07-09 23:40:36,626 - root - INFO - lr: 1.8499e-04 gnorm: 0.81 [10:05:37<11:56:15] +[titan] 2025-07-09 23:40:40,582 - root - INFO - step: 45820 loss: 16.4570 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9282 +[titan] 2025-07-09 23:40:40,583 - root - INFO - lr: 1.8497e-04 gnorm: 0.81 [10:05:41<11:56:11] +[titan] 2025-07-09 23:40:44,514 - root - INFO - step: 45825 loss: 16.4923 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9618 +[titan] 2025-07-09 23:40:44,514 - root - INFO - lr: 1.8495e-04 gnorm: 0.80 [10:05:44<11:56:07] +[titan] 2025-07-09 23:40:48,419 - root - INFO - step: 45830 loss: 16.5598 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5543 global_avg_mtp_loss: 14.0055 +[titan] 2025-07-09 23:40:48,419 - root - INFO - lr: 1.8493e-04 gnorm: 0.83 [10:05:48<11:56:03] +[titan] 2025-07-09 23:40:52,355 - root - INFO - step: 45835 loss: 16.5255 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9800 +[titan] 2025-07-09 23:40:52,356 - root - INFO - lr: 1.8491e-04 gnorm: 0.81 [10:05:52<11:55:59] +[titan] 2025-07-09 23:40:56,288 - root - INFO - step: 45840 loss: 16.3294 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8154 +[titan] 2025-07-09 23:40:56,288 - root - INFO - lr: 1.8489e-04 gnorm: 0.81 [10:05:56<11:55:55] +[titan] 2025-07-09 23:41:00,247 - root - INFO - step: 45845 loss: 16.6317 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0629 +[titan] 2025-07-09 23:41:00,247 - root - INFO - lr: 1.8487e-04 gnorm: 0.77 [10:06:00<11:55:51] +[titan] 2025-07-09 23:41:03,391 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:41:04,194 - root - INFO - step: 45850 loss: 16.7920 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.6052 global_avg_mtp_loss: 14.1868 +[titan] 2025-07-09 23:41:04,194 - root - INFO - lr: 1.8485e-04 gnorm: 0.81 [10:06:04<11:55:47] +[titan] 2025-07-09 23:41:08,109 - root - INFO - step: 45855 loss: 16.3511 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.8397 +[titan] 2025-07-09 23:41:08,110 - root - INFO - lr: 1.8482e-04 gnorm: 0.84 [10:06:08<11:55:43] +[titan] 2025-07-09 23:41:12,066 - root - INFO - step: 45860 loss: 16.5716 memory: 44.58GiB(31.99%) tps: 82,817 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0171 +[titan] 2025-07-09 23:41:12,067 - root - INFO - lr: 1.8480e-04 gnorm: 0.82 [10:06:12<11:55:39] +[titan] 2025-07-09 23:41:15,998 - root - INFO - step: 45865 loss: 16.6478 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.0783 +[titan] 2025-07-09 23:41:15,998 - root - INFO - lr: 1.8478e-04 gnorm: 0.81 [10:06:16<11:55:35] +[titan] 2025-07-09 23:41:19,929 - root - INFO - step: 45870 loss: 16.3810 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.8480 +[titan] 2025-07-09 23:41:19,929 - root - INFO - lr: 1.8476e-04 gnorm: 0.84 [10:06:20<11:55:31] +[titan] 2025-07-09 23:41:23,858 - root - INFO - step: 45875 loss: 16.6478 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0734 +[titan] 2025-07-09 23:41:23,859 - root - INFO - lr: 1.8474e-04 gnorm: 0.81 [10:06:24<11:55:27] +[titan] 2025-07-09 23:41:27,789 - root - INFO - step: 45880 loss: 16.4985 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 13.9540 +[titan] 2025-07-09 23:41:27,790 - root - INFO - lr: 1.8472e-04 gnorm: 0.84 [10:06:28<11:55:23] +[titan] 2025-07-09 23:41:31,688 - root - INFO - step: 45885 loss: 16.4214 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.8948 +[titan] 2025-07-09 23:41:31,688 - root - INFO - lr: 1.8470e-04 gnorm: 0.80 [10:06:32<11:55:19] +[titan] 2025-07-09 23:41:35,606 - root - INFO - step: 45890 loss: 16.9558 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.6344 global_avg_mtp_loss: 14.3214 +[titan] 2025-07-09 23:41:35,607 - root - INFO - lr: 1.8468e-04 gnorm: 0.82 [10:06:36<11:55:15] +[titan] 2025-07-09 23:41:39,515 - root - INFO - step: 45895 loss: 16.6461 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0809 +[titan] 2025-07-09 23:41:39,516 - root - INFO - lr: 1.8466e-04 gnorm: 0.81 [10:06:39<11:55:11] +[titan] 2025-07-09 23:41:42,685 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:41:43,472 - root - INFO - step: 45900 loss: 16.4433 memory: 44.58GiB(31.99%) tps: 82,825 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9162 +[titan] 2025-07-09 23:41:43,472 - root - INFO - lr: 1.8463e-04 gnorm: 0.79 [10:06:43<11:55:07] +[titan] 2025-07-09 23:41:47,406 - root - INFO - step: 45905 loss: 16.8781 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.6345 global_avg_mtp_loss: 14.2436 +[titan] 2025-07-09 23:41:47,406 - root - INFO - lr: 1.8461e-04 gnorm: 1.00 [10:06:47<11:55:03] +[titan] 2025-07-09 23:41:51,328 - root - INFO - step: 45910 loss: 16.7665 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5928 global_avg_mtp_loss: 14.1737 +[titan] 2025-07-09 23:41:51,328 - root - INFO - lr: 1.8459e-04 gnorm: 0.81 [10:06:51<11:54:59] +[titan] 2025-07-09 23:41:55,233 - root - INFO - step: 45915 loss: 16.4325 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9005 +[titan] 2025-07-09 23:41:55,233 - root - INFO - lr: 1.8457e-04 gnorm: 0.79 [10:06:55<11:54:55] +[titan] 2025-07-09 23:41:59,163 - root - INFO - step: 45920 loss: 17.0553 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.6446 global_avg_mtp_loss: 14.4107 +[titan] 2025-07-09 23:41:59,163 - root - INFO - lr: 1.8455e-04 gnorm: 0.81 [10:06:59<11:54:51] +[titan] 2025-07-09 23:42:03,100 - root - INFO - step: 45925 loss: 16.5360 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9991 +[titan] 2025-07-09 23:42:03,100 - root - INFO - lr: 1.8453e-04 gnorm: 0.78 [10:07:03<11:54:47] +[titan] 2025-07-09 23:42:06,995 - root - INFO - step: 45930 loss: 16.6362 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.0633 +[titan] 2025-07-09 23:42:06,996 - root - INFO - lr: 1.8451e-04 gnorm: 0.83 [10:07:07<11:54:43] +[titan] 2025-07-09 23:42:10,907 - root - INFO - step: 45935 loss: 16.7209 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5874 global_avg_mtp_loss: 14.1335 +[titan] 2025-07-09 23:42:10,907 - root - INFO - lr: 1.8449e-04 gnorm: 0.81 [10:07:11<11:54:39] +[titan] 2025-07-09 23:42:14,844 - root - INFO - step: 45940 loss: 16.7041 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5834 global_avg_mtp_loss: 14.1207 +[titan] 2025-07-09 23:42:14,845 - root - INFO - lr: 1.8446e-04 gnorm: 0.80 [10:07:15<11:54:35] +[titan] 2025-07-09 23:42:18,795 - root - INFO - step: 45945 loss: 16.4565 memory: 44.58GiB(31.99%) tps: 82,947 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9149 +[titan] 2025-07-09 23:42:18,795 - root - INFO - lr: 1.8444e-04 gnorm: 0.80 [10:07:19<11:54:31] +[titan] 2025-07-09 23:42:21,933 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:42:22,720 - root - INFO - step: 45950 loss: 16.7248 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5932 global_avg_mtp_loss: 14.1316 +[titan] 2025-07-09 23:42:22,720 - root - INFO - lr: 1.8442e-04 gnorm: 0.80 [10:07:23<11:54:27] +[titan] 2025-07-09 23:42:26,619 - root - INFO - step: 45955 loss: 16.3322 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8298 +[titan] 2025-07-09 23:42:26,619 - root - INFO - lr: 1.8440e-04 gnorm: 0.79 [10:07:27<11:54:23] +[titan] 2025-07-09 23:42:30,574 - root - INFO - step: 45960 loss: 16.2748 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.7650 +[titan] 2025-07-09 23:42:30,574 - root - INFO - lr: 1.8438e-04 gnorm: 0.79 [10:07:30<11:54:19] +[titan] 2025-07-09 23:42:34,485 - root - INFO - step: 45965 loss: 16.7237 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.1473 +[titan] 2025-07-09 23:42:34,485 - root - INFO - lr: 1.8436e-04 gnorm: 0.80 [10:07:34<11:54:15] +[titan] 2025-07-09 23:42:38,383 - root - INFO - step: 45970 loss: 16.3559 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8447 +[titan] 2025-07-09 23:42:38,383 - root - INFO - lr: 1.8434e-04 gnorm: 0.79 [10:07:38<11:54:11] +[titan] 2025-07-09 23:42:42,331 - root - INFO - step: 45975 loss: 16.6892 memory: 44.58GiB(31.99%) tps: 83,003 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.1190 +[titan] 2025-07-09 23:42:42,331 - root - INFO - lr: 1.8432e-04 gnorm: 0.82 [10:07:42<11:54:07] +[titan] 2025-07-09 23:42:46,253 - root - INFO - step: 45980 loss: 16.2735 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.7818 +[titan] 2025-07-09 23:42:46,253 - root - INFO - lr: 1.8429e-04 gnorm: 0.76 [10:07:46<11:54:03] +[titan] 2025-07-09 23:42:50,226 - root - INFO - step: 45985 loss: 16.3505 memory: 44.58GiB(31.99%) tps: 82,477 tflops: 284.64 mfu: 28.78% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8316 +[titan] 2025-07-09 23:42:50,226 - root - INFO - lr: 1.8427e-04 gnorm: 0.77 [10:07:50<11:53:59] +[titan] 2025-07-09 23:42:54,135 - root - INFO - step: 45990 loss: 16.3486 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.8247 +[titan] 2025-07-09 23:42:54,135 - root - INFO - lr: 1.8425e-04 gnorm: 0.82 [10:07:54<11:53:55] +[titan] 2025-07-09 23:42:58,053 - root - INFO - step: 45995 loss: 16.3579 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.8367 +[titan] 2025-07-09 23:42:58,053 - root - INFO - lr: 1.8423e-04 gnorm: 0.87 [10:07:58<11:53:51] +[titan] 2025-07-09 23:43:01,188 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:43:01,993 - root - INFO - step: 46000 loss: 16.9626 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.6336 global_avg_mtp_loss: 14.3290 +[titan] 2025-07-09 23:43:01,993 - root - INFO - lr: 1.8421e-04 gnorm: 0.88 [10:08:02<11:53:47] +[titan] 2025-07-09 23:43:05,936 - root - INFO - step: 46005 loss: 16.4255 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.9022 +[titan] 2025-07-09 23:43:05,936 - root - INFO - lr: 1.8419e-04 gnorm: 0.78 [10:08:06<11:53:43] +[titan] 2025-07-09 23:43:09,889 - root - INFO - step: 46010 loss: 16.6265 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5612 global_avg_mtp_loss: 14.0653 +[titan] 2025-07-09 23:43:09,889 - root - INFO - lr: 1.8417e-04 gnorm: 0.80 [10:08:10<11:53:39] +[titan] 2025-07-09 23:43:13,819 - root - INFO - step: 46015 loss: 16.5415 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9904 +[titan] 2025-07-09 23:43:13,820 - root - INFO - lr: 1.8415e-04 gnorm: 0.81 [10:08:14<11:53:35] +[titan] 2025-07-09 23:43:17,758 - root - INFO - step: 46020 loss: 16.3644 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.8445 +[titan] 2025-07-09 23:43:17,759 - root - INFO - lr: 1.8413e-04 gnorm: 0.77 [10:08:18<11:53:31] +[titan] 2025-07-09 23:43:21,695 - root - INFO - step: 46025 loss: 16.6502 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0810 +[titan] 2025-07-09 23:43:21,695 - root - INFO - lr: 1.8410e-04 gnorm: 0.79 [10:08:22<11:53:27] +[titan] 2025-07-09 23:43:25,619 - root - INFO - step: 46030 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.8623 +[titan] 2025-07-09 23:43:25,619 - root - INFO - lr: 1.8408e-04 gnorm: 0.82 [10:08:26<11:53:23] +[titan] 2025-07-09 23:43:29,556 - root - INFO - step: 46035 loss: 16.5111 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9650 +[titan] 2025-07-09 23:43:29,556 - root - INFO - lr: 1.8406e-04 gnorm: 0.93 [10:08:29<11:53:19] +[titan] 2025-07-09 23:43:33,476 - root - INFO - step: 46040 loss: 16.5265 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9885 +[titan] 2025-07-09 23:43:33,476 - root - INFO - lr: 1.8404e-04 gnorm: 0.82 [10:08:33<11:53:15] +[titan] 2025-07-09 23:43:37,381 - root - INFO - step: 46045 loss: 16.4252 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.9016 +[titan] 2025-07-09 23:43:37,381 - root - INFO - lr: 1.8402e-04 gnorm: 0.78 [10:08:37<11:53:11] +[titan] 2025-07-09 23:43:40,504 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:43:41,303 - root - INFO - step: 46050 loss: 16.4877 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5287 global_avg_mtp_loss: 13.9590 +[titan] 2025-07-09 23:43:41,303 - root - INFO - lr: 1.8400e-04 gnorm: 0.78 [10:08:41<11:53:07] +[titan] 2025-07-09 23:43:45,222 - root - INFO - step: 46055 loss: 16.5962 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.0377 +[titan] 2025-07-09 23:43:45,222 - root - INFO - lr: 1.8398e-04 gnorm: 0.81 [10:08:45<11:53:03] +[titan] 2025-07-09 23:43:49,174 - root - INFO - step: 46060 loss: 16.5306 memory: 44.58GiB(31.99%) tps: 82,931 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9853 +[titan] 2025-07-09 23:43:49,174 - root - INFO - lr: 1.8396e-04 gnorm: 0.97 [10:08:49<11:52:59] +[titan] 2025-07-09 23:43:53,080 - root - INFO - step: 46065 loss: 16.7468 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.1637 +[titan] 2025-07-09 23:43:53,080 - root - INFO - lr: 1.8393e-04 gnorm: 0.88 [10:08:53<11:52:55] +[titan] 2025-07-09 23:43:56,985 - root - INFO - step: 46070 loss: 16.8339 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.6044 global_avg_mtp_loss: 14.2295 +[titan] 2025-07-09 23:43:56,985 - root - INFO - lr: 1.8391e-04 gnorm: 0.81 [10:08:57<11:52:51] +[titan] 2025-07-09 23:44:00,919 - root - INFO - step: 46075 loss: 16.3199 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-09 23:44:00,919 - root - INFO - lr: 1.8389e-04 gnorm: 0.90 [10:09:01<11:52:47] +[titan] 2025-07-09 23:44:04,970 - root - INFO - step: 46080 loss: 16.7838 memory: 44.58GiB(31.99%) tps: 80,905 tflops: 279.22 mfu: 28.23% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.1921 +[titan] 2025-07-09 23:44:04,970 - root - INFO - lr: 1.8387e-04 gnorm: 0.86 [10:09:05<11:52:43] +[titan] 2025-07-09 23:44:05,123 - root - INFO - Dumping profiler traces at step 46080 +[titan] 2025-07-09 23:44:05,156 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:44:09,068 - root - INFO - step: 46085 loss: 16.7872 memory: 44.58GiB(31.99%) tps: 79,959 tflops: 275.95 mfu: 27.90% global_avg_ntp_loss: 2.5968 global_avg_mtp_loss: 14.1905 +[titan] 2025-07-09 23:44:09,068 - root - INFO - lr: 1.8385e-04 gnorm: 0.85 [10:09:09<11:52:39] +[titan] 2025-07-09 23:44:12,995 - root - INFO - step: 46090 loss: 16.7213 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1334 +[titan] 2025-07-09 23:44:12,996 - root - INFO - lr: 1.8383e-04 gnorm: 0.82 [10:09:13<11:52:35] +[titan] 2025-07-09 23:44:16,894 - root - INFO - step: 46095 loss: 16.4143 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.8827 +[titan] 2025-07-09 23:44:16,894 - root - INFO - lr: 1.8381e-04 gnorm: 0.78 [10:09:17<11:52:31] +[titan] 2025-07-09 23:44:20,009 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:44:20,798 - root - INFO - step: 46100 loss: 16.6877 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.1103 +[titan] 2025-07-09 23:44:20,798 - root - INFO - lr: 1.8379e-04 gnorm: 0.81 [10:09:21<11:52:27] +[titan] 2025-07-09 23:44:24,713 - root - INFO - step: 46105 loss: 16.5816 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0324 +[titan] 2025-07-09 23:44:24,713 - root - INFO - lr: 1.8376e-04 gnorm: 0.81 [10:09:25<11:52:23] +[titan] 2025-07-09 23:44:28,618 - root - INFO - step: 46110 loss: 16.5341 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9862 +[titan] 2025-07-09 23:44:28,618 - root - INFO - lr: 1.8374e-04 gnorm: 0.78 [10:09:29<11:52:19] +[titan] 2025-07-09 23:44:32,522 - root - INFO - step: 46115 loss: 16.6284 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0588 +[titan] 2025-07-09 23:44:32,522 - root - INFO - lr: 1.8372e-04 gnorm: 0.80 [10:09:32<11:52:15] +[titan] 2025-07-09 23:44:36,481 - root - INFO - step: 46120 loss: 16.6438 memory: 44.58GiB(31.99%) tps: 82,768 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.0733 +[titan] 2025-07-09 23:44:36,481 - root - INFO - lr: 1.8370e-04 gnorm: 0.84 [10:09:36<11:52:11] +[titan] 2025-07-09 23:44:40,401 - root - INFO - step: 46125 loss: 16.5792 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 14.0324 +[titan] 2025-07-09 23:44:40,401 - root - INFO - lr: 1.8368e-04 gnorm: 0.91 [10:09:40<11:52:07] +[titan] 2025-07-09 23:44:44,336 - root - INFO - step: 46130 loss: 16.5165 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9701 +[titan] 2025-07-09 23:44:44,337 - root - INFO - lr: 1.8366e-04 gnorm: 0.94 [10:09:44<11:52:03] +[titan] 2025-07-09 23:44:48,232 - root - INFO - step: 46135 loss: 16.7761 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1836 +[titan] 2025-07-09 23:44:48,232 - root - INFO - lr: 1.8364e-04 gnorm: 0.81 [10:09:48<11:51:59] +[titan] 2025-07-09 23:44:52,119 - root - INFO - step: 46140 loss: 16.6378 memory: 44.58GiB(31.99%) tps: 84,301 tflops: 290.94 mfu: 29.42% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0713 +[titan] 2025-07-09 23:44:52,119 - root - INFO - lr: 1.8362e-04 gnorm: 0.90 [10:09:52<11:51:55] +[titan] 2025-07-09 23:44:56,014 - root - INFO - step: 46145 loss: 16.7688 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5953 global_avg_mtp_loss: 14.1736 +[titan] 2025-07-09 23:44:56,014 - root - INFO - lr: 1.8360e-04 gnorm: 0.77 [10:09:56<11:51:51] +[titan] 2025-07-09 23:44:59,166 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:44:59,961 - root - INFO - step: 46150 loss: 16.5000 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 13.9523 +[titan] 2025-07-09 23:44:59,962 - root - INFO - lr: 1.8357e-04 gnorm: 0.77 [10:10:00<11:51:47] +[titan] 2025-07-09 23:45:03,872 - root - INFO - step: 46155 loss: 16.3741 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8544 +[titan] 2025-07-09 23:45:03,872 - root - INFO - lr: 1.8355e-04 gnorm: 0.79 [10:10:04<11:51:43] +[titan] 2025-07-09 23:45:07,793 - root - INFO - step: 46160 loss: 16.5339 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9925 +[titan] 2025-07-09 23:45:07,794 - root - INFO - lr: 1.8353e-04 gnorm: 0.81 [10:10:08<11:51:38] +[titan] 2025-07-09 23:45:11,692 - root - INFO - step: 46165 loss: 16.5396 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 13.9902 +[titan] 2025-07-09 23:45:11,692 - root - INFO - lr: 1.8351e-04 gnorm: 0.79 [10:10:12<11:51:34] +[titan] 2025-07-09 23:45:15,615 - root - INFO - step: 46170 loss: 16.5110 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 13.9626 +[titan] 2025-07-09 23:45:15,615 - root - INFO - lr: 1.8349e-04 gnorm: 0.81 [10:10:16<11:51:30] +[titan] 2025-07-09 23:45:19,528 - root - INFO - step: 46175 loss: 16.5748 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 14.0237 +[titan] 2025-07-09 23:45:19,528 - root - INFO - lr: 1.8347e-04 gnorm: 0.78 [10:10:19<11:51:26] +[titan] 2025-07-09 23:45:23,445 - root - INFO - step: 46180 loss: 16.7465 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5941 global_avg_mtp_loss: 14.1524 +[titan] 2025-07-09 23:45:23,445 - root - INFO - lr: 1.8345e-04 gnorm: 0.77 [10:10:23<11:51:22] +[titan] 2025-07-09 23:45:27,357 - root - INFO - step: 46185 loss: 16.3407 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8174 +[titan] 2025-07-09 23:45:27,357 - root - INFO - lr: 1.8343e-04 gnorm: 0.80 [10:10:27<11:51:18] +[titan] 2025-07-09 23:45:31,287 - root - INFO - step: 46190 loss: 16.2427 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.7338 +[titan] 2025-07-09 23:45:31,287 - root - INFO - lr: 1.8340e-04 gnorm: 0.81 [10:10:31<11:51:14] +[titan] 2025-07-09 23:45:35,240 - root - INFO - step: 46195 loss: 16.5921 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0362 +[titan] 2025-07-09 23:45:35,240 - root - INFO - lr: 1.8338e-04 gnorm: 0.76 [10:10:35<11:51:10] +[titan] 2025-07-09 23:45:38,352 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:45:39,140 - root - INFO - step: 46200 loss: 16.7325 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.1501 +[titan] 2025-07-09 23:45:39,141 - root - INFO - lr: 1.8336e-04 gnorm: 0.79 [10:10:39<11:51:06] +[titan] 2025-07-09 23:45:43,071 - root - INFO - step: 46205 loss: 16.3928 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.8632 +[titan] 2025-07-09 23:45:43,071 - root - INFO - lr: 1.8334e-04 gnorm: 0.79 [10:10:43<11:51:02] +[titan] 2025-07-09 23:45:46,970 - root - INFO - step: 46210 loss: 16.5062 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.9635 +[titan] 2025-07-09 23:45:46,971 - root - INFO - lr: 1.8332e-04 gnorm: 0.80 [10:10:47<11:50:58] +[titan] 2025-07-09 23:45:50,885 - root - INFO - step: 46215 loss: 16.5030 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 13.9558 +[titan] 2025-07-09 23:45:50,885 - root - INFO - lr: 1.8330e-04 gnorm: 0.78 [10:10:51<11:50:54] +[titan] 2025-07-09 23:45:54,785 - root - INFO - step: 46220 loss: 16.7108 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1278 +[titan] 2025-07-09 23:45:54,786 - root - INFO - lr: 1.8328e-04 gnorm: 0.76 [10:10:55<11:50:50] +[titan] 2025-07-09 23:45:58,733 - root - INFO - step: 46225 loss: 16.3931 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.8654 +[titan] 2025-07-09 23:45:58,733 - root - INFO - lr: 1.8326e-04 gnorm: 0.80 [10:10:59<11:50:46] +[titan] 2025-07-09 23:46:02,648 - root - INFO - step: 46230 loss: 16.6382 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5657 global_avg_mtp_loss: 14.0724 +[titan] 2025-07-09 23:46:02,648 - root - INFO - lr: 1.8323e-04 gnorm: 0.77 [10:11:03<11:50:42] +[titan] 2025-07-09 23:46:06,588 - root - INFO - step: 46235 loss: 16.7338 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5908 global_avg_mtp_loss: 14.1431 +[titan] 2025-07-09 23:46:06,588 - root - INFO - lr: 1.8321e-04 gnorm: 0.83 [10:11:06<11:50:38] +[titan] 2025-07-09 23:46:10,495 - root - INFO - step: 46240 loss: 16.3558 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-09 23:46:10,495 - root - INFO - lr: 1.8319e-04 gnorm: 0.79 [10:11:10<11:50:34] +[titan] 2025-07-09 23:46:14,414 - root - INFO - step: 46245 loss: 16.6868 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5717 global_avg_mtp_loss: 14.1151 +[titan] 2025-07-09 23:46:14,414 - root - INFO - lr: 1.8317e-04 gnorm: 0.85 [10:11:14<11:50:30] +[titan] 2025-07-09 23:46:17,549 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:46:18,348 - root - INFO - step: 46250 loss: 16.5827 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0249 +[titan] 2025-07-09 23:46:18,348 - root - INFO - lr: 1.8315e-04 gnorm: 0.82 [10:11:18<11:50:26] +[titan] 2025-07-09 23:46:22,262 - root - INFO - step: 46255 loss: 16.6522 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-09 23:46:22,262 - root - INFO - lr: 1.8313e-04 gnorm: 0.85 [10:11:22<11:50:22] +[titan] 2025-07-09 23:46:26,160 - root - INFO - step: 46260 loss: 16.7365 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5983 global_avg_mtp_loss: 14.1382 +[titan] 2025-07-09 23:46:26,160 - root - INFO - lr: 1.8311e-04 gnorm: 0.81 [10:11:26<11:50:18] +[titan] 2025-07-09 23:46:30,088 - root - INFO - step: 46265 loss: 16.3522 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5149 global_avg_mtp_loss: 13.8373 +[titan] 2025-07-09 23:46:30,088 - root - INFO - lr: 1.8309e-04 gnorm: 0.79 [10:11:30<11:50:14] +[titan] 2025-07-09 23:46:34,036 - root - INFO - step: 46270 loss: 16.4682 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.9455 +[titan] 2025-07-09 23:46:34,036 - root - INFO - lr: 1.8306e-04 gnorm: 0.76 [10:11:34<11:50:10] +[titan] 2025-07-09 23:46:37,940 - root - INFO - step: 46275 loss: 16.7090 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.1296 +[titan] 2025-07-09 23:46:37,940 - root - INFO - lr: 1.8304e-04 gnorm: 0.79 [10:11:38<11:50:06] +[titan] 2025-07-09 23:46:41,839 - root - INFO - step: 46280 loss: 16.5815 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-09 23:46:41,839 - root - INFO - lr: 1.8302e-04 gnorm: 0.78 [10:11:42<11:50:02] +[titan] 2025-07-09 23:46:45,789 - root - INFO - step: 46285 loss: 16.6653 memory: 44.58GiB(31.99%) tps: 82,958 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5732 global_avg_mtp_loss: 14.0921 +[titan] 2025-07-09 23:46:45,790 - root - INFO - lr: 1.8300e-04 gnorm: 0.78 [10:11:46<11:49:58] +[titan] 2025-07-09 23:46:49,725 - root - INFO - step: 46290 loss: 16.6113 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0492 +[titan] 2025-07-09 23:46:49,725 - root - INFO - lr: 1.8298e-04 gnorm: 0.85 [10:11:50<11:49:54] +[titan] 2025-07-09 23:46:53,630 - root - INFO - step: 46295 loss: 16.6813 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.1066 +[titan] 2025-07-09 23:46:53,630 - root - INFO - lr: 1.8296e-04 gnorm: 0.81 [10:11:54<11:49:50] +[titan] 2025-07-09 23:46:56,763 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:46:57,570 - root - INFO - step: 46300 loss: 16.3521 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8416 +[titan] 2025-07-09 23:46:57,570 - root - INFO - lr: 1.8294e-04 gnorm: 0.79 [10:11:57<11:49:46] +[titan] 2025-07-09 23:47:01,492 - root - INFO - step: 46305 loss: 16.7714 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.1924 +[titan] 2025-07-09 23:47:01,492 - root - INFO - lr: 1.8292e-04 gnorm: 0.76 [10:12:01<11:49:42] +[titan] 2025-07-09 23:47:05,407 - root - INFO - step: 46310 loss: 16.5268 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.9886 +[titan] 2025-07-09 23:47:05,408 - root - INFO - lr: 1.8289e-04 gnorm: 0.75 [10:12:05<11:49:38] +[titan] 2025-07-09 23:47:09,324 - root - INFO - step: 46315 loss: 16.6404 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0650 +[titan] 2025-07-09 23:47:09,324 - root - INFO - lr: 1.8287e-04 gnorm: 0.78 [10:12:09<11:49:34] +[titan] 2025-07-09 23:47:13,249 - root - INFO - step: 46320 loss: 16.8124 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.2218 +[titan] 2025-07-09 23:47:13,250 - root - INFO - lr: 1.8285e-04 gnorm: 0.82 [10:12:13<11:49:30] +[titan] 2025-07-09 23:47:17,191 - root - INFO - step: 46325 loss: 16.5532 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0050 +[titan] 2025-07-09 23:47:17,191 - root - INFO - lr: 1.8283e-04 gnorm: 0.79 [10:12:17<11:49:26] +[titan] 2025-07-09 23:47:21,135 - root - INFO - step: 46330 loss: 16.1935 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7067 +[titan] 2025-07-09 23:47:21,135 - root - INFO - lr: 1.8281e-04 gnorm: 0.79 [10:12:21<11:49:22] +[titan] 2025-07-09 23:47:25,083 - root - INFO - step: 46335 loss: 16.6710 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.1002 +[titan] 2025-07-09 23:47:25,084 - root - INFO - lr: 1.8279e-04 gnorm: 0.75 [10:12:25<11:49:18] +[titan] 2025-07-09 23:47:29,017 - root - INFO - step: 46340 loss: 16.5538 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 13.9891 +[titan] 2025-07-09 23:47:29,017 - root - INFO - lr: 1.8277e-04 gnorm: 0.85 [10:12:29<11:49:14] +[titan] 2025-07-09 23:47:32,930 - root - INFO - step: 46345 loss: 16.3381 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8279 +[titan] 2025-07-09 23:47:32,930 - root - INFO - lr: 1.8275e-04 gnorm: 0.83 [10:12:33<11:49:10] +[titan] 2025-07-09 23:47:36,053 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:47:36,842 - root - INFO - step: 46350 loss: 16.6203 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.0501 +[titan] 2025-07-09 23:47:36,842 - root - INFO - lr: 1.8272e-04 gnorm: 0.78 [10:12:37<11:49:06] +[titan] 2025-07-09 23:47:40,756 - root - INFO - step: 46355 loss: 16.6120 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0492 +[titan] 2025-07-09 23:47:40,757 - root - INFO - lr: 1.8270e-04 gnorm: 0.75 [10:12:41<11:49:02] +[titan] 2025-07-09 23:47:44,666 - root - INFO - step: 46360 loss: 16.3352 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8393 +[titan] 2025-07-09 23:47:44,666 - root - INFO - lr: 1.8268e-04 gnorm: 0.79 [10:12:45<11:48:58] +[titan] 2025-07-09 23:47:48,588 - root - INFO - step: 46365 loss: 16.3776 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8610 +[titan] 2025-07-09 23:47:48,588 - root - INFO - lr: 1.8266e-04 gnorm: 0.83 [10:12:48<11:48:54] +[titan] 2025-07-09 23:47:52,633 - root - INFO - step: 46370 loss: 16.6061 memory: 44.58GiB(31.99%) tps: 81,006 tflops: 279.56 mfu: 28.27% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 14.0597 +[titan] 2025-07-09 23:47:52,633 - root - INFO - lr: 1.8264e-04 gnorm: 0.77 [10:12:53<11:48:50] +[titan] 2025-07-09 23:47:56,557 - root - INFO - step: 46375 loss: 16.4711 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9316 +[titan] 2025-07-09 23:47:56,557 - root - INFO - lr: 1.8262e-04 gnorm: 0.79 [10:12:56<11:48:46] +[titan] 2025-07-09 23:48:00,490 - root - INFO - step: 46380 loss: 16.5943 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5542 global_avg_mtp_loss: 14.0401 +[titan] 2025-07-09 23:48:00,490 - root - INFO - lr: 1.8260e-04 gnorm: 0.77 [10:13:00<11:48:42] +[titan] 2025-07-09 23:48:04,399 - root - INFO - step: 46385 loss: 16.7267 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.1432 +[titan] 2025-07-09 23:48:04,399 - root - INFO - lr: 1.8258e-04 gnorm: 0.79 [10:13:04<11:48:38] +[titan] 2025-07-09 23:48:08,431 - root - INFO - step: 46390 loss: 16.7717 memory: 44.58GiB(31.99%) tps: 81,283 tflops: 280.52 mfu: 28.36% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1747 +[titan] 2025-07-09 23:48:08,431 - root - INFO - lr: 1.8256e-04 gnorm: 0.82 [10:13:08<11:48:34] +[titan] 2025-07-09 23:48:12,345 - root - INFO - step: 46395 loss: 16.7127 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.1388 +[titan] 2025-07-09 23:48:12,345 - root - INFO - lr: 1.8253e-04 gnorm: 0.77 [10:13:12<11:48:30] +[titan] 2025-07-09 23:48:15,464 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:48:16,249 - root - INFO - step: 46400 loss: 16.3686 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-09 23:48:16,249 - root - INFO - lr: 1.8251e-04 gnorm: 0.81 [10:13:16<11:48:26] +[titan] 2025-07-09 23:48:20,338 - root - INFO - step: 46405 loss: 16.6448 memory: 44.58GiB(31.99%) tps: 80,146 tflops: 276.60 mfu: 27.97% global_avg_ntp_loss: 2.5627 global_avg_mtp_loss: 14.0821 +[titan] 2025-07-09 23:48:20,338 - root - INFO - lr: 1.8249e-04 gnorm: 0.79 [10:13:20<11:48:22] +[titan] 2025-07-09 23:48:24,253 - root - INFO - step: 46410 loss: 16.7865 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.2016 +[titan] 2025-07-09 23:48:24,254 - root - INFO - lr: 1.8247e-04 gnorm: 0.82 [10:13:24<11:48:18] +[titan] 2025-07-09 23:48:28,208 - root - INFO - step: 46415 loss: 16.5182 memory: 44.58GiB(31.99%) tps: 82,863 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9795 +[titan] 2025-07-09 23:48:28,209 - root - INFO - lr: 1.8245e-04 gnorm: 0.79 [10:13:28<11:48:14] +[titan] 2025-07-09 23:48:32,136 - root - INFO - step: 46420 loss: 16.6783 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5686 global_avg_mtp_loss: 14.1097 +[titan] 2025-07-09 23:48:32,136 - root - INFO - lr: 1.8243e-04 gnorm: 0.85 [10:13:32<11:48:10] +[titan] 2025-07-09 23:48:36,054 - root - INFO - step: 46425 loss: 16.6535 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0885 +[titan] 2025-07-09 23:48:36,054 - root - INFO - lr: 1.8241e-04 gnorm: 0.85 [10:13:36<11:48:06] +[titan] 2025-07-09 23:48:40,009 - root - INFO - step: 46430 loss: 16.6020 memory: 44.58GiB(31.99%) tps: 82,858 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5616 global_avg_mtp_loss: 14.0404 +[titan] 2025-07-09 23:48:40,009 - root - INFO - lr: 1.8239e-04 gnorm: 0.77 [10:13:40<11:48:02] +[titan] 2025-07-09 23:48:43,925 - root - INFO - step: 46435 loss: 16.7322 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1478 +[titan] 2025-07-09 23:48:43,926 - root - INFO - lr: 1.8236e-04 gnorm: 0.79 [10:13:44<11:47:58] +[titan] 2025-07-09 23:48:47,844 - root - INFO - step: 46440 loss: 16.5468 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 13.9832 +[titan] 2025-07-09 23:48:47,845 - root - INFO - lr: 1.8234e-04 gnorm: 0.84 [10:13:48<11:47:54] +[titan] 2025-07-09 23:48:51,797 - root - INFO - step: 46445 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.7936 +[titan] 2025-07-09 23:48:51,798 - root - INFO - lr: 1.8232e-04 gnorm: 0.83 [10:13:52<11:47:50] +[titan] 2025-07-09 23:48:54,936 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:48:55,729 - root - INFO - step: 46450 loss: 16.4730 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9440 +[titan] 2025-07-09 23:48:55,729 - root - INFO - lr: 1.8230e-04 gnorm: 0.76 [10:13:56<11:47:46] +[titan] 2025-07-09 23:48:59,634 - root - INFO - step: 46455 loss: 16.4501 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9199 +[titan] 2025-07-09 23:48:59,635 - root - INFO - lr: 1.8228e-04 gnorm: 0.77 [10:14:00<11:47:42] +[titan] 2025-07-09 23:49:03,553 - root - INFO - step: 46460 loss: 16.7242 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5889 global_avg_mtp_loss: 14.1353 +[titan] 2025-07-09 23:49:03,553 - root - INFO - lr: 1.8226e-04 gnorm: 0.77 [10:14:03<11:47:38] +[titan] 2025-07-09 23:49:07,450 - root - INFO - step: 46465 loss: 16.6917 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.1167 +[titan] 2025-07-09 23:49:07,450 - root - INFO - lr: 1.8224e-04 gnorm: 0.80 [10:14:07<11:47:34] +[titan] 2025-07-09 23:49:11,355 - root - INFO - step: 46470 loss: 16.5516 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5392 global_avg_mtp_loss: 14.0124 +[titan] 2025-07-09 23:49:11,355 - root - INFO - lr: 1.8222e-04 gnorm: 0.75 [10:14:11<11:47:30] +[titan] 2025-07-09 23:49:15,251 - root - INFO - step: 46475 loss: 16.6023 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5598 global_avg_mtp_loss: 14.0426 +[titan] 2025-07-09 23:49:15,252 - root - INFO - lr: 1.8219e-04 gnorm: 0.78 [10:14:15<11:47:26] +[titan] 2025-07-09 23:49:19,150 - root - INFO - step: 46480 loss: 16.6895 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1055 +[titan] 2025-07-09 23:49:19,150 - root - INFO - lr: 1.8217e-04 gnorm: 0.88 [10:14:19<11:47:22] +[titan] 2025-07-09 23:49:23,054 - root - INFO - step: 46485 loss: 16.6338 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0688 +[titan] 2025-07-09 23:49:23,054 - root - INFO - lr: 1.8215e-04 gnorm: 0.75 [10:14:23<11:47:18] +[titan] 2025-07-09 23:49:26,967 - root - INFO - step: 46490 loss: 16.6826 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5784 global_avg_mtp_loss: 14.1043 +[titan] 2025-07-09 23:49:26,967 - root - INFO - lr: 1.8213e-04 gnorm: 0.81 [10:14:27<11:47:14] +[titan] 2025-07-09 23:49:30,863 - root - INFO - step: 46495 loss: 16.3457 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-09 23:49:30,864 - root - INFO - lr: 1.8211e-04 gnorm: 0.84 [10:14:31<11:47:10] +[titan] 2025-07-09 23:49:33,996 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:49:34,786 - root - INFO - step: 46500 loss: 16.7710 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.6016 global_avg_mtp_loss: 14.1693 +[titan] 2025-07-09 23:49:34,786 - root - INFO - lr: 1.8209e-04 gnorm: 0.80 [10:14:35<11:47:06] +[titan] 2025-07-09 23:49:38,716 - root - INFO - step: 46505 loss: 16.4469 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9050 +[titan] 2025-07-09 23:49:38,716 - root - INFO - lr: 1.8207e-04 gnorm: 0.77 [10:14:39<11:47:02] +[titan] 2025-07-09 23:49:42,652 - root - INFO - step: 46510 loss: 16.6209 memory: 44.58GiB(31.99%) tps: 83,256 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0544 +[titan] 2025-07-09 23:49:42,652 - root - INFO - lr: 1.8205e-04 gnorm: 0.83 [10:14:43<11:46:58] +[titan] 2025-07-09 23:49:46,593 - root - INFO - step: 46515 loss: 16.5604 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0082 +[titan] 2025-07-09 23:49:46,593 - root - INFO - lr: 1.8202e-04 gnorm: 0.80 [10:14:46<11:46:54] +[titan] 2025-07-09 23:49:50,500 - root - INFO - step: 46520 loss: 16.3081 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.7961 +[titan] 2025-07-09 23:49:50,500 - root - INFO - lr: 1.8200e-04 gnorm: 0.85 [10:14:50<11:46:50] +[titan] 2025-07-09 23:49:54,423 - root - INFO - step: 46525 loss: 16.5429 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 13.9800 +[titan] 2025-07-09 23:49:54,423 - root - INFO - lr: 1.8198e-04 gnorm: 0.79 [10:14:54<11:46:46] +[titan] 2025-07-09 23:49:58,351 - root - INFO - step: 46530 loss: 16.3594 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.8351 +[titan] 2025-07-09 23:49:58,351 - root - INFO - lr: 1.8196e-04 gnorm: 0.83 [10:14:58<11:46:42] +[titan] 2025-07-09 23:50:02,267 - root - INFO - step: 46535 loss: 16.3408 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.8206 +[titan] 2025-07-09 23:50:02,267 - root - INFO - lr: 1.8194e-04 gnorm: 0.77 [10:15:02<11:46:38] +[titan] 2025-07-09 23:50:06,165 - root - INFO - step: 46540 loss: 17.0122 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.6483 global_avg_mtp_loss: 14.3639 +[titan] 2025-07-09 23:50:06,165 - root - INFO - lr: 1.8192e-04 gnorm: 0.84 [10:15:06<11:46:34] +[titan] 2025-07-09 23:50:10,071 - root - INFO - step: 46545 loss: 16.5353 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9868 +[titan] 2025-07-09 23:50:10,071 - root - INFO - lr: 1.8190e-04 gnorm: 0.83 [10:15:10<11:46:30] +[titan] 2025-07-09 23:50:13,185 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:50:13,975 - root - INFO - step: 46550 loss: 16.4691 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9391 +[titan] 2025-07-09 23:50:13,975 - root - INFO - lr: 1.8188e-04 gnorm: 0.79 [10:15:14<11:46:26] +[titan] 2025-07-09 23:50:17,891 - root - INFO - step: 46555 loss: 16.6718 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.1066 +[titan] 2025-07-09 23:50:17,891 - root - INFO - lr: 1.8185e-04 gnorm: 0.77 [10:15:18<11:46:22] +[titan] 2025-07-09 23:50:21,814 - root - INFO - step: 46560 loss: 16.5437 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 13.9782 +[titan] 2025-07-09 23:50:21,815 - root - INFO - lr: 1.8183e-04 gnorm: 0.85 [10:15:22<11:46:18] +[titan] 2025-07-09 23:50:25,768 - root - INFO - step: 46565 loss: 16.8080 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.6023 global_avg_mtp_loss: 14.2057 +[titan] 2025-07-09 23:50:25,768 - root - INFO - lr: 1.8181e-04 gnorm: 0.75 [10:15:26<11:46:14] +[titan] 2025-07-09 23:50:29,681 - root - INFO - step: 46570 loss: 16.6732 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0992 +[titan] 2025-07-09 23:50:29,681 - root - INFO - lr: 1.8179e-04 gnorm: 0.81 [10:15:30<11:46:10] +[titan] 2025-07-09 23:50:33,621 - root - INFO - step: 46575 loss: 16.5948 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0241 +[titan] 2025-07-09 23:50:33,621 - root - INFO - lr: 1.8177e-04 gnorm: 0.85 [10:15:33<11:46:06] +[titan] 2025-07-09 23:50:37,585 - root - INFO - step: 46580 loss: 16.5947 memory: 44.58GiB(31.99%) tps: 82,672 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0342 +[titan] 2025-07-09 23:50:37,586 - root - INFO - lr: 1.8175e-04 gnorm: 0.76 [10:15:37<11:46:02] +[titan] 2025-07-09 23:50:41,484 - root - INFO - step: 46585 loss: 16.5687 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 13.9976 +[titan] 2025-07-09 23:50:41,484 - root - INFO - lr: 1.8173e-04 gnorm: 0.78 [10:15:41<11:45:58] +[titan] 2025-07-09 23:50:45,479 - root - INFO - step: 46590 loss: 16.7040 memory: 44.58GiB(31.99%) tps: 82,038 tflops: 283.13 mfu: 28.63% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.1247 +[titan] 2025-07-09 23:50:45,479 - root - INFO - lr: 1.8171e-04 gnorm: 0.77 [10:15:45<11:45:54] +[titan] 2025-07-09 23:50:47,221 - root - INFO - Dumping profiler traces at step 46592 +[titan] 2025-07-09 23:50:47,252 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:50:49,600 - root - INFO - step: 46595 loss: 16.4626 memory: 44.58GiB(31.99%) tps: 79,519 tflops: 274.43 mfu: 27.75% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9288 +[titan] 2025-07-09 23:50:49,600 - root - INFO - lr: 1.8168e-04 gnorm: 0.79 [10:15:49<11:45:50] +[titan] 2025-07-09 23:50:52,719 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:50:53,508 - root - INFO - step: 46600 loss: 16.6174 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0587 +[titan] 2025-07-09 23:50:53,508 - root - INFO - lr: 1.8166e-04 gnorm: 0.76 [10:15:53<11:45:46] +[titan] 2025-07-09 23:50:57,474 - root - INFO - step: 46605 loss: 16.5907 memory: 44.58GiB(31.99%) tps: 82,617 tflops: 285.12 mfu: 28.83% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 14.0459 +[titan] 2025-07-09 23:50:57,475 - root - INFO - lr: 1.8164e-04 gnorm: 0.78 [10:15:57<11:45:42] +[titan] 2025-07-09 23:51:01,406 - root - INFO - step: 46610 loss: 16.7179 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.1347 +[titan] 2025-07-09 23:51:01,406 - root - INFO - lr: 1.8162e-04 gnorm: 0.81 [10:16:01<11:45:38] +[titan] 2025-07-09 23:51:05,311 - root - INFO - step: 46615 loss: 16.3865 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8656 +[titan] 2025-07-09 23:51:05,312 - root - INFO - lr: 1.8160e-04 gnorm: 0.87 [10:16:05<11:45:34] +[titan] 2025-07-09 23:51:09,213 - root - INFO - step: 46620 loss: 16.3860 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8787 +[titan] 2025-07-09 23:51:09,214 - root - INFO - lr: 1.8158e-04 gnorm: 0.78 [10:16:09<11:45:30] +[titan] 2025-07-09 23:51:13,162 - root - INFO - step: 46625 loss: 16.5256 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9860 +[titan] 2025-07-09 23:51:13,162 - root - INFO - lr: 1.8156e-04 gnorm: 0.81 [10:16:13<11:45:26] +[titan] 2025-07-09 23:51:17,080 - root - INFO - step: 46630 loss: 16.4682 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9359 +[titan] 2025-07-09 23:51:17,081 - root - INFO - lr: 1.8154e-04 gnorm: 0.75 [10:16:17<11:45:22] +[titan] 2025-07-09 23:51:20,980 - root - INFO - step: 46635 loss: 16.4600 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9341 +[titan] 2025-07-09 23:51:20,980 - root - INFO - lr: 1.8151e-04 gnorm: 0.79 [10:16:21<11:45:18] +[titan] 2025-07-09 23:51:24,914 - root - INFO - step: 46640 loss: 16.6590 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0832 +[titan] 2025-07-09 23:51:24,915 - root - INFO - lr: 1.8149e-04 gnorm: 0.78 [10:16:25<11:45:14] +[titan] 2025-07-09 23:51:28,837 - root - INFO - step: 46645 loss: 16.6627 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5685 global_avg_mtp_loss: 14.0942 +[titan] 2025-07-09 23:51:28,837 - root - INFO - lr: 1.8147e-04 gnorm: 0.84 [10:16:29<11:45:10] +[titan] 2025-07-09 23:51:31,979 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:51:32,766 - root - INFO - step: 46650 loss: 16.5620 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0042 +[titan] 2025-07-09 23:51:32,766 - root - INFO - lr: 1.8145e-04 gnorm: 0.89 [10:16:33<11:45:06] +[titan] 2025-07-09 23:51:36,673 - root - INFO - step: 46655 loss: 16.6588 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0896 +[titan] 2025-07-09 23:51:36,673 - root - INFO - lr: 1.8143e-04 gnorm: 0.90 [10:16:37<11:45:02] +[titan] 2025-07-09 23:51:40,583 - root - INFO - step: 46660 loss: 16.5064 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9696 +[titan] 2025-07-09 23:51:40,584 - root - INFO - lr: 1.8141e-04 gnorm: 0.86 [10:16:40<11:44:58] +[titan] 2025-07-09 23:51:44,476 - root - INFO - step: 46665 loss: 16.4381 memory: 44.58GiB(31.99%) tps: 84,182 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9053 +[titan] 2025-07-09 23:51:44,477 - root - INFO - lr: 1.8139e-04 gnorm: 1.17 [10:16:44<11:44:54] +[titan] 2025-07-09 23:51:48,432 - root - INFO - step: 46670 loss: 16.5323 memory: 44.58GiB(31.99%) tps: 82,853 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 13.9788 +[titan] 2025-07-09 23:51:48,432 - root - INFO - lr: 1.8137e-04 gnorm: 0.85 [10:16:48<11:44:50] +[titan] 2025-07-09 23:51:52,343 - root - INFO - step: 46675 loss: 16.3204 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.7941 +[titan] 2025-07-09 23:51:52,343 - root - INFO - lr: 1.8134e-04 gnorm: 0.83 [10:16:52<11:44:46] +[titan] 2025-07-09 23:51:56,262 - root - INFO - step: 46680 loss: 16.6086 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0516 +[titan] 2025-07-09 23:51:56,262 - root - INFO - lr: 1.8132e-04 gnorm: 0.79 [10:16:56<11:44:42] +[titan] 2025-07-09 23:52:00,181 - root - INFO - step: 46685 loss: 16.9099 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.6196 global_avg_mtp_loss: 14.2902 +[titan] 2025-07-09 23:52:00,181 - root - INFO - lr: 1.8130e-04 gnorm: 0.82 [10:17:00<11:44:38] +[titan] 2025-07-09 23:52:04,093 - root - INFO - step: 46690 loss: 16.6543 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0859 +[titan] 2025-07-09 23:52:04,093 - root - INFO - lr: 1.8128e-04 gnorm: 0.76 [10:17:04<11:44:34] +[titan] 2025-07-09 23:52:07,992 - root - INFO - step: 46695 loss: 16.3274 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8284 +[titan] 2025-07-09 23:52:07,992 - root - INFO - lr: 1.8126e-04 gnorm: 0.79 [10:17:08<11:44:29] +[titan] 2025-07-09 23:52:11,136 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:52:11,931 - root - INFO - step: 46700 loss: 16.4157 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.8864 +[titan] 2025-07-09 23:52:11,931 - root - INFO - lr: 1.8124e-04 gnorm: 0.92 [10:17:12<11:44:25] +[titan] 2025-07-09 23:52:15,867 - root - INFO - step: 46705 loss: 16.7006 memory: 44.58GiB(31.99%) tps: 83,247 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5929 global_avg_mtp_loss: 14.1077 +[titan] 2025-07-09 23:52:15,868 - root - INFO - lr: 1.8122e-04 gnorm: 1.07 [10:17:16<11:44:21] +[titan] 2025-07-09 23:52:19,784 - root - INFO - step: 46710 loss: 16.6768 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.0959 +[titan] 2025-07-09 23:52:19,784 - root - INFO - lr: 1.8119e-04 gnorm: 0.83 [10:17:20<11:44:17] +[titan] 2025-07-09 23:52:23,689 - root - INFO - step: 46715 loss: 16.7594 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5866 global_avg_mtp_loss: 14.1729 +[titan] 2025-07-09 23:52:23,689 - root - INFO - lr: 1.8117e-04 gnorm: 0.80 [10:17:24<11:44:13] +[titan] 2025-07-09 23:52:27,600 - root - INFO - step: 46720 loss: 16.5805 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0271 +[titan] 2025-07-09 23:52:27,601 - root - INFO - lr: 1.8115e-04 gnorm: 0.82 [10:17:27<11:44:09] +[titan] 2025-07-09 23:52:31,541 - root - INFO - step: 46725 loss: 16.5951 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0415 +[titan] 2025-07-09 23:52:31,541 - root - INFO - lr: 1.8113e-04 gnorm: 0.82 [10:17:31<11:44:05] +[titan] 2025-07-09 23:52:35,462 - root - INFO - step: 46730 loss: 16.3969 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8827 +[titan] 2025-07-09 23:52:35,463 - root - INFO - lr: 1.8111e-04 gnorm: 0.78 [10:17:35<11:44:01] +[titan] 2025-07-09 23:52:39,351 - root - INFO - step: 46735 loss: 16.2202 memory: 44.58GiB(31.99%) tps: 84,266 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7278 +[titan] 2025-07-09 23:52:39,352 - root - INFO - lr: 1.8109e-04 gnorm: 0.85 [10:17:39<11:43:57] +[titan] 2025-07-09 23:52:43,265 - root - INFO - step: 46740 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 13.8970 +[titan] 2025-07-09 23:52:43,265 - root - INFO - lr: 1.8107e-04 gnorm: 0.78 [10:17:43<11:43:53] +[titan] 2025-07-09 23:52:47,174 - root - INFO - step: 46745 loss: 16.5830 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0219 +[titan] 2025-07-09 23:52:47,174 - root - INFO - lr: 1.8105e-04 gnorm: 0.81 [10:17:47<11:43:49] +[titan] 2025-07-09 23:52:50,307 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:52:51,280 - root - INFO - step: 46750 loss: 16.7251 memory: 44.58GiB(31.99%) tps: 79,808 tflops: 275.43 mfu: 27.85% global_avg_ntp_loss: 2.5933 global_avg_mtp_loss: 14.1318 +[titan] 2025-07-09 23:52:51,281 - root - INFO - lr: 1.8102e-04 gnorm: 0.79 [10:17:51<11:43:45] +[titan] 2025-07-09 23:52:55,201 - root - INFO - step: 46755 loss: 16.5229 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5598 global_avg_mtp_loss: 13.9631 +[titan] 2025-07-09 23:52:55,202 - root - INFO - lr: 1.8100e-04 gnorm: 0.89 [10:17:55<11:43:41] +[titan] 2025-07-09 23:52:59,154 - root - INFO - step: 46760 loss: 16.4946 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9620 +[titan] 2025-07-09 23:52:59,155 - root - INFO - lr: 1.8098e-04 gnorm: 0.80 [10:17:59<11:43:37] +[titan] 2025-07-09 23:53:03,067 - root - INFO - step: 46765 loss: 16.5434 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 14.0009 +[titan] 2025-07-09 23:53:03,068 - root - INFO - lr: 1.8096e-04 gnorm: 0.81 [10:18:03<11:43:33] +[titan] 2025-07-09 23:53:07,041 - root - INFO - step: 46770 loss: 16.4548 memory: 44.58GiB(31.99%) tps: 82,471 tflops: 284.62 mfu: 28.78% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.9404 +[titan] 2025-07-09 23:53:07,041 - root - INFO - lr: 1.8094e-04 gnorm: 0.77 [10:18:07<11:43:30] +[titan] 2025-07-09 23:53:11,032 - root - INFO - step: 46775 loss: 16.7575 memory: 44.58GiB(31.99%) tps: 82,106 tflops: 283.36 mfu: 28.65% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.1706 +[titan] 2025-07-09 23:53:11,033 - root - INFO - lr: 1.8092e-04 gnorm: 0.75 [10:18:11<11:43:26] +[titan] 2025-07-09 23:53:14,945 - root - INFO - step: 46780 loss: 16.5565 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 14.0085 +[titan] 2025-07-09 23:53:14,945 - root - INFO - lr: 1.8090e-04 gnorm: 1.03 [10:18:15<11:43:22] +[titan] 2025-07-09 23:53:18,906 - root - INFO - step: 46785 loss: 16.3978 memory: 44.58GiB(31.99%) tps: 82,740 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8774 +[titan] 2025-07-09 23:53:18,906 - root - INFO - lr: 1.8088e-04 gnorm: 0.84 [10:18:19<11:43:18] +[titan] 2025-07-09 23:53:22,821 - root - INFO - step: 46790 loss: 16.6370 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0765 +[titan] 2025-07-09 23:53:22,821 - root - INFO - lr: 1.8085e-04 gnorm: 0.81 [10:18:23<11:43:14] +[titan] 2025-07-09 23:53:26,748 - root - INFO - step: 46795 loss: 16.5197 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9767 +[titan] 2025-07-09 23:53:26,749 - root - INFO - lr: 1.8083e-04 gnorm: 0.82 [10:18:27<11:43:10] +[titan] 2025-07-09 23:53:29,872 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:53:30,688 - root - INFO - step: 46800 loss: 16.4440 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.02% global_avg_ntp_loss: 2.5375 global_avg_mtp_loss: 13.9065 +[titan] 2025-07-09 23:53:30,689 - root - INFO - lr: 1.8081e-04 gnorm: 0.77 [10:18:31<11:43:06] +[titan] 2025-07-09 23:53:34,586 - root - INFO - step: 46805 loss: 16.5110 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9681 +[titan] 2025-07-09 23:53:34,586 - root - INFO - lr: 1.8079e-04 gnorm: 0.83 [10:18:34<11:43:02] +[titan] 2025-07-09 23:53:38,485 - root - INFO - step: 46810 loss: 16.6218 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5624 global_avg_mtp_loss: 14.0594 +[titan] 2025-07-09 23:53:38,485 - root - INFO - lr: 1.8077e-04 gnorm: 0.85 [10:18:38<11:42:57] +[titan] 2025-07-09 23:53:42,398 - root - INFO - step: 46815 loss: 16.7094 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.1235 +[titan] 2025-07-09 23:53:42,398 - root - INFO - lr: 1.8075e-04 gnorm: 0.79 [10:18:42<11:42:53] +[titan] 2025-07-09 23:53:46,348 - root - INFO - step: 46820 loss: 16.6019 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 14.0550 +[titan] 2025-07-09 23:53:46,349 - root - INFO - lr: 1.8073e-04 gnorm: 0.82 [10:18:46<11:42:49] +[titan] 2025-07-09 23:53:50,258 - root - INFO - step: 46825 loss: 16.7633 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5999 global_avg_mtp_loss: 14.1634 +[titan] 2025-07-09 23:53:50,258 - root - INFO - lr: 1.8071e-04 gnorm: 0.82 [10:18:50<11:42:45] +[titan] 2025-07-09 23:53:54,178 - root - INFO - step: 46830 loss: 16.6554 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5852 global_avg_mtp_loss: 14.0703 +[titan] 2025-07-09 23:53:54,178 - root - INFO - lr: 1.8068e-04 gnorm: 0.82 [10:18:54<11:42:41] +[titan] 2025-07-09 23:53:58,090 - root - INFO - step: 46835 loss: 16.2054 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7166 +[titan] 2025-07-09 23:53:58,090 - root - INFO - lr: 1.8066e-04 gnorm: 0.82 [10:18:58<11:42:37] +[titan] 2025-07-09 23:54:02,017 - root - INFO - step: 46840 loss: 16.6191 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0563 +[titan] 2025-07-09 23:54:02,018 - root - INFO - lr: 1.8064e-04 gnorm: 1.00 [10:19:02<11:42:33] +[titan] 2025-07-09 23:54:05,933 - root - INFO - step: 46845 loss: 16.5701 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0132 +[titan] 2025-07-09 23:54:05,934 - root - INFO - lr: 1.8062e-04 gnorm: 0.77 [10:19:06<11:42:29] +[titan] 2025-07-09 23:54:09,062 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:54:09,851 - root - INFO - step: 46850 loss: 16.5197 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9883 +[titan] 2025-07-09 23:54:09,851 - root - INFO - lr: 1.8060e-04 gnorm: 0.74 [10:19:10<11:42:25] +[titan] 2025-07-09 23:54:13,770 - root - INFO - step: 46855 loss: 16.5481 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 13.9834 +[titan] 2025-07-09 23:54:13,770 - root - INFO - lr: 1.8058e-04 gnorm: 0.80 [10:19:14<11:42:21] +[titan] 2025-07-09 23:54:17,681 - root - INFO - step: 46860 loss: 16.4330 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9010 +[titan] 2025-07-09 23:54:17,681 - root - INFO - lr: 1.8056e-04 gnorm: 0.79 [10:19:18<11:42:17] +[titan] 2025-07-09 23:54:21,587 - root - INFO - step: 46865 loss: 16.7298 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5776 global_avg_mtp_loss: 14.1522 +[titan] 2025-07-09 23:54:21,587 - root - INFO - lr: 1.8054e-04 gnorm: 0.78 [10:19:21<11:42:13] +[titan] 2025-07-09 23:54:25,542 - root - INFO - step: 46870 loss: 16.5121 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9656 +[titan] 2025-07-09 23:54:25,542 - root - INFO - lr: 1.8051e-04 gnorm: 0.79 [10:19:25<11:42:09] +[titan] 2025-07-09 23:54:29,495 - root - INFO - step: 46875 loss: 16.3401 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-09 23:54:29,495 - root - INFO - lr: 1.8049e-04 gnorm: 0.85 [10:19:29<11:42:05] +[titan] 2025-07-09 23:54:33,402 - root - INFO - step: 46880 loss: 16.6752 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-09 23:54:33,402 - root - INFO - lr: 1.8047e-04 gnorm: 0.78 [10:19:33<11:42:01] +[titan] 2025-07-09 23:54:37,310 - root - INFO - step: 46885 loss: 16.5378 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5671 global_avg_mtp_loss: 13.9707 +[titan] 2025-07-09 23:54:37,310 - root - INFO - lr: 1.8045e-04 gnorm: 0.99 [10:19:37<11:41:57] +[titan] 2025-07-09 23:54:41,256 - root - INFO - step: 46890 loss: 16.6402 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0832 +[titan] 2025-07-09 23:54:41,256 - root - INFO - lr: 1.8043e-04 gnorm: 0.80 [10:19:41<11:41:53] +[titan] 2025-07-09 23:54:45,167 - root - INFO - step: 46895 loss: 16.6838 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.1107 +[titan] 2025-07-09 23:54:45,167 - root - INFO - lr: 1.8041e-04 gnorm: 0.83 [10:19:45<11:41:49] +[titan] 2025-07-09 23:54:48,309 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:54:49,100 - root - INFO - step: 46900 loss: 16.5682 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0162 +[titan] 2025-07-09 23:54:49,100 - root - INFO - lr: 1.8039e-04 gnorm: 0.84 [10:19:49<11:41:45] +[titan] 2025-07-09 23:54:53,016 - root - INFO - step: 46905 loss: 16.7207 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.1326 +[titan] 2025-07-09 23:54:53,017 - root - INFO - lr: 1.8037e-04 gnorm: 0.86 [10:19:53<11:41:41] +[titan] 2025-07-09 23:54:56,914 - root - INFO - step: 46910 loss: 16.7057 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5741 global_avg_mtp_loss: 14.1316 +[titan] 2025-07-09 23:54:56,914 - root - INFO - lr: 1.8034e-04 gnorm: 0.82 [10:19:57<11:41:37] +[titan] 2025-07-09 23:55:00,833 - root - INFO - step: 46915 loss: 16.7652 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5954 global_avg_mtp_loss: 14.1698 +[titan] 2025-07-09 23:55:00,833 - root - INFO - lr: 1.8032e-04 gnorm: 0.81 [10:20:01<11:41:33] +[titan] 2025-07-09 23:55:04,743 - root - INFO - step: 46920 loss: 16.6115 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0481 +[titan] 2025-07-09 23:55:04,743 - root - INFO - lr: 1.8030e-04 gnorm: 0.83 [10:20:05<11:41:29] +[titan] 2025-07-09 23:55:08,648 - root - INFO - step: 46925 loss: 16.5415 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 13.9824 +[titan] 2025-07-09 23:55:08,648 - root - INFO - lr: 1.8028e-04 gnorm: 0.78 [10:20:08<11:41:25] +[titan] 2025-07-09 23:55:12,548 - root - INFO - step: 46930 loss: 16.4499 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9206 +[titan] 2025-07-09 23:55:12,548 - root - INFO - lr: 1.8026e-04 gnorm: 0.78 [10:20:12<11:41:21] +[titan] 2025-07-09 23:55:16,483 - root - INFO - step: 46935 loss: 16.5169 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9717 +[titan] 2025-07-09 23:55:16,483 - root - INFO - lr: 1.8024e-04 gnorm: 0.79 [10:20:16<11:41:17] +[titan] 2025-07-09 23:55:20,380 - root - INFO - step: 46940 loss: 16.6307 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5724 global_avg_mtp_loss: 14.0583 +[titan] 2025-07-09 23:55:20,380 - root - INFO - lr: 1.8022e-04 gnorm: 0.81 [10:20:20<11:41:13] +[titan] 2025-07-09 23:55:24,277 - root - INFO - step: 46945 loss: 16.5288 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 13.9758 +[titan] 2025-07-09 23:55:24,277 - root - INFO - lr: 1.8020e-04 gnorm: 0.77 [10:20:24<11:41:09] +[titan] 2025-07-09 23:55:27,437 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:55:28,225 - root - INFO - step: 46950 loss: 16.4777 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-09 23:55:28,225 - root - INFO - lr: 1.8017e-04 gnorm: 0.77 [10:20:28<11:41:05] +[titan] 2025-07-09 23:55:32,135 - root - INFO - step: 46955 loss: 16.5259 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5631 global_avg_mtp_loss: 13.9629 +[titan] 2025-07-09 23:55:32,135 - root - INFO - lr: 1.8015e-04 gnorm: 0.77 [10:20:32<11:41:01] +[titan] 2025-07-09 23:55:36,043 - root - INFO - step: 46960 loss: 16.3800 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.8645 +[titan] 2025-07-09 23:55:36,044 - root - INFO - lr: 1.8013e-04 gnorm: 0.77 [10:20:36<11:40:57] +[titan] 2025-07-09 23:55:39,941 - root - INFO - step: 46965 loss: 16.4476 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.9091 +[titan] 2025-07-09 23:55:39,941 - root - INFO - lr: 1.8011e-04 gnorm: 0.77 [10:20:40<11:40:53] +[titan] 2025-07-09 23:55:43,866 - root - INFO - step: 46970 loss: 16.6583 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0853 +[titan] 2025-07-09 23:55:43,867 - root - INFO - lr: 1.8009e-04 gnorm: 0.80 [10:20:44<11:40:49] +[titan] 2025-07-09 23:55:47,790 - root - INFO - step: 46975 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-09 23:55:47,790 - root - INFO - lr: 1.8007e-04 gnorm: 0.80 [10:20:48<11:40:45] +[titan] 2025-07-09 23:55:51,770 - root - INFO - step: 46980 loss: 16.4706 memory: 44.58GiB(31.99%) tps: 82,352 tflops: 284.21 mfu: 28.74% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-09 23:55:51,770 - root - INFO - lr: 1.8005e-04 gnorm: 0.77 [10:20:52<11:40:41] +[titan] 2025-07-09 23:55:55,666 - root - INFO - step: 46985 loss: 16.6507 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0954 +[titan] 2025-07-09 23:55:55,666 - root - INFO - lr: 1.8002e-04 gnorm: 0.79 [10:20:55<11:40:37] +[titan] 2025-07-09 23:55:59,606 - root - INFO - step: 46990 loss: 16.4906 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9406 +[titan] 2025-07-09 23:55:59,606 - root - INFO - lr: 1.8000e-04 gnorm: 0.86 [10:20:59<11:40:33] +[titan] 2025-07-09 23:56:03,515 - root - INFO - step: 46995 loss: 16.6222 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0563 +[titan] 2025-07-09 23:56:03,516 - root - INFO - lr: 1.7998e-04 gnorm: 0.81 [10:21:03<11:40:29] +[titan] 2025-07-09 23:56:06,628 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:56:07,420 - root - INFO - step: 47000 loss: 16.4932 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-09 23:56:07,421 - root - INFO - lr: 1.7996e-04 gnorm: 0.78 [10:21:07<11:40:25] +[titan] 2025-07-09 23:56:11,342 - root - INFO - step: 47005 loss: 16.6171 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0634 +[titan] 2025-07-09 23:56:11,342 - root - INFO - lr: 1.7994e-04 gnorm: 0.79 [10:21:11<11:40:21] +[titan] 2025-07-09 23:56:15,307 - root - INFO - step: 47010 loss: 16.7435 memory: 44.58GiB(31.99%) tps: 82,631 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1571 +[titan] 2025-07-09 23:56:15,308 - root - INFO - lr: 1.7992e-04 gnorm: 0.80 [10:21:15<11:40:17] +[titan] 2025-07-09 23:56:19,228 - root - INFO - step: 47015 loss: 16.4253 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-09 23:56:19,229 - root - INFO - lr: 1.7990e-04 gnorm: 0.83 [10:21:19<11:40:13] +[titan] 2025-07-09 23:56:23,149 - root - INFO - step: 47020 loss: 16.5772 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 14.0270 +[titan] 2025-07-09 23:56:23,149 - root - INFO - lr: 1.7988e-04 gnorm: 0.79 [10:21:23<11:40:09] +[titan] 2025-07-09 23:56:27,063 - root - INFO - step: 47025 loss: 16.3323 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8157 +[titan] 2025-07-09 23:56:27,064 - root - INFO - lr: 1.7985e-04 gnorm: 0.79 [10:21:27<11:40:05] +[titan] 2025-07-09 23:56:30,972 - root - INFO - step: 47030 loss: 16.9101 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.6271 global_avg_mtp_loss: 14.2830 +[titan] 2025-07-09 23:56:30,972 - root - INFO - lr: 1.7983e-04 gnorm: 0.76 [10:21:31<11:40:01] +[titan] 2025-07-09 23:56:34,870 - root - INFO - step: 47035 loss: 16.4926 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9530 +[titan] 2025-07-09 23:56:34,871 - root - INFO - lr: 1.7981e-04 gnorm: 1.14 [10:21:35<11:39:57] +[titan] 2025-07-09 23:56:38,779 - root - INFO - step: 47040 loss: 16.3828 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8686 +[titan] 2025-07-09 23:56:38,779 - root - INFO - lr: 1.7979e-04 gnorm: 0.80 [10:21:39<11:39:53] +[titan] 2025-07-09 23:56:42,691 - root - INFO - step: 47045 loss: 16.7700 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5735 global_avg_mtp_loss: 14.1965 +[titan] 2025-07-09 23:56:42,691 - root - INFO - lr: 1.7977e-04 gnorm: 0.79 [10:21:43<11:39:49] +[titan] 2025-07-09 23:56:45,801 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:56:46,592 - root - INFO - step: 47050 loss: 16.5716 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.0077 +[titan] 2025-07-09 23:56:46,592 - root - INFO - lr: 1.7975e-04 gnorm: 0.93 [10:21:46<11:39:45] +[titan] 2025-07-09 23:56:50,499 - root - INFO - step: 47055 loss: 16.6431 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0837 +[titan] 2025-07-09 23:56:50,499 - root - INFO - lr: 1.7973e-04 gnorm: 0.88 [10:21:50<11:39:41] +[titan] 2025-07-09 23:56:54,396 - root - INFO - step: 47060 loss: 16.4304 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9011 +[titan] 2025-07-09 23:56:54,396 - root - INFO - lr: 1.7971e-04 gnorm: 0.83 [10:21:54<11:39:37] +[titan] 2025-07-09 23:56:58,301 - root - INFO - step: 47065 loss: 16.6980 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5813 global_avg_mtp_loss: 14.1167 +[titan] 2025-07-09 23:56:58,302 - root - INFO - lr: 1.7968e-04 gnorm: 0.82 [10:21:58<11:39:33] +[titan] 2025-07-09 23:57:02,214 - root - INFO - step: 47070 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9402 +[titan] 2025-07-09 23:57:02,214 - root - INFO - lr: 1.7966e-04 gnorm: 0.82 [10:22:02<11:39:29] +[titan] 2025-07-09 23:57:06,134 - root - INFO - step: 47075 loss: 16.6019 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0440 +[titan] 2025-07-09 23:57:06,134 - root - INFO - lr: 1.7964e-04 gnorm: 0.78 [10:22:06<11:39:25] +[titan] 2025-07-09 23:57:10,062 - root - INFO - step: 47080 loss: 16.1541 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.6754 +[titan] 2025-07-09 23:57:10,062 - root - INFO - lr: 1.7962e-04 gnorm: 0.83 [10:22:10<11:39:20] +[titan] 2025-07-09 23:57:13,985 - root - INFO - step: 47085 loss: 16.5128 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9688 +[titan] 2025-07-09 23:57:13,985 - root - INFO - lr: 1.7960e-04 gnorm: 0.81 [10:22:14<11:39:16] +[titan] 2025-07-09 23:57:17,994 - root - INFO - step: 47090 loss: 16.5562 memory: 44.58GiB(31.99%) tps: 81,744 tflops: 282.11 mfu: 28.53% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 14.0098 +[titan] 2025-07-09 23:57:17,994 - root - INFO - lr: 1.7958e-04 gnorm: 0.82 [10:22:18<11:39:13] +[titan] 2025-07-09 23:57:21,922 - root - INFO - step: 47095 loss: 16.7939 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.1686 +[titan] 2025-07-09 23:57:21,923 - root - INFO - lr: 1.7956e-04 gnorm: 1.54 [10:22:22<11:39:09] +[titan] 2025-07-09 23:57:25,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:57:25,842 - root - INFO - step: 47100 loss: 16.5880 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0240 +[titan] 2025-07-09 23:57:25,842 - root - INFO - lr: 1.7954e-04 gnorm: 1.06 [10:22:26<11:39:05] +[titan] 2025-07-09 23:57:29,186 - root - INFO - Dumping profiler traces at step 47104 +[titan] 2025-07-09 23:57:29,218 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-09 23:57:30,019 - root - INFO - step: 47105 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 78,449 tflops: 270.74 mfu: 27.38% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.0402 +[titan] 2025-07-09 23:57:30,019 - root - INFO - lr: 1.7951e-04 gnorm: 1.10 [10:22:30<11:39:01] +[titan] 2025-07-09 23:57:33,939 - root - INFO - step: 47110 loss: 16.6848 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5865 global_avg_mtp_loss: 14.0983 +[titan] 2025-07-09 23:57:33,939 - root - INFO - lr: 1.7949e-04 gnorm: 0.83 [10:22:34<11:38:57] +[titan] 2025-07-09 23:57:37,897 - root - INFO - step: 47115 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.5355 global_avg_mtp_loss: 13.9709 +[titan] 2025-07-09 23:57:37,898 - root - INFO - lr: 1.7947e-04 gnorm: 0.84 [10:22:38<11:38:53] +[titan] 2025-07-09 23:57:41,821 - root - INFO - step: 47120 loss: 16.3946 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.8717 +[titan] 2025-07-09 23:57:41,821 - root - INFO - lr: 1.7945e-04 gnorm: 0.83 [10:22:42<11:38:49] +[titan] 2025-07-09 23:57:45,727 - root - INFO - step: 47125 loss: 16.5494 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9983 +[titan] 2025-07-09 23:57:45,727 - root - INFO - lr: 1.7943e-04 gnorm: 0.79 [10:22:46<11:38:45] +[titan] 2025-07-09 23:57:49,645 - root - INFO - step: 47130 loss: 16.6589 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5649 global_avg_mtp_loss: 14.0940 +[titan] 2025-07-09 23:57:49,645 - root - INFO - lr: 1.7941e-04 gnorm: 0.73 [10:22:49<11:38:41] +[titan] 2025-07-09 23:57:53,545 - root - INFO - step: 47135 loss: 16.5047 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 13.9463 +[titan] 2025-07-09 23:57:53,546 - root - INFO - lr: 1.7939e-04 gnorm: 0.80 [10:22:53<11:38:37] +[titan] 2025-07-09 23:57:57,445 - root - INFO - step: 47140 loss: 16.4521 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9071 +[titan] 2025-07-09 23:57:57,445 - root - INFO - lr: 1.7936e-04 gnorm: 0.75 [10:22:57<11:38:33] +[titan] 2025-07-09 23:58:01,559 - root - INFO - step: 47145 loss: 16.6829 memory: 44.58GiB(31.99%) tps: 79,648 tflops: 274.88 mfu: 27.79% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.1144 +[titan] 2025-07-09 23:58:01,560 - root - INFO - lr: 1.7934e-04 gnorm: 0.80 [10:23:01<11:38:29] +[titan] 2025-07-09 23:58:04,710 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:58:05,522 - root - INFO - step: 47150 loss: 16.8653 memory: 44.58GiB(31.99%) tps: 82,691 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.6084 global_avg_mtp_loss: 14.2568 +[titan] 2025-07-09 23:58:05,523 - root - INFO - lr: 1.7932e-04 gnorm: 0.76 [10:23:05<11:38:25] +[titan] 2025-07-09 23:58:09,438 - root - INFO - step: 47155 loss: 16.8303 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.2409 +[titan] 2025-07-09 23:58:09,438 - root - INFO - lr: 1.7930e-04 gnorm: 0.76 [10:23:09<11:38:21] +[titan] 2025-07-09 23:58:13,384 - root - INFO - step: 47160 loss: 16.5216 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9735 +[titan] 2025-07-09 23:58:13,384 - root - INFO - lr: 1.7928e-04 gnorm: 0.90 [10:23:13<11:38:17] +[titan] 2025-07-09 23:58:17,318 - root - INFO - step: 47165 loss: 16.2670 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7685 +[titan] 2025-07-09 23:58:17,318 - root - INFO - lr: 1.7926e-04 gnorm: 0.81 [10:23:17<11:38:13] +[titan] 2025-07-09 23:58:21,229 - root - INFO - step: 47170 loss: 16.3591 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5111 global_avg_mtp_loss: 13.8480 +[titan] 2025-07-09 23:58:21,229 - root - INFO - lr: 1.7924e-04 gnorm: 0.78 [10:23:21<11:38:09] +[titan] 2025-07-09 23:58:25,130 - root - INFO - step: 47175 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 13.9601 +[titan] 2025-07-09 23:58:25,131 - root - INFO - lr: 1.7922e-04 gnorm: 0.83 [10:23:25<11:38:05] +[titan] 2025-07-09 23:58:29,214 - root - INFO - step: 47180 loss: 16.6810 memory: 44.58GiB(31.99%) tps: 80,242 tflops: 276.93 mfu: 28.00% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0992 +[titan] 2025-07-09 23:58:29,215 - root - INFO - lr: 1.7919e-04 gnorm: 0.89 [10:23:29<11:38:01] +[titan] 2025-07-09 23:58:33,132 - root - INFO - step: 47185 loss: 16.5951 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0308 +[titan] 2025-07-09 23:58:33,133 - root - INFO - lr: 1.7917e-04 gnorm: 0.83 [10:23:33<11:37:57] +[titan] 2025-07-09 23:58:37,039 - root - INFO - step: 47190 loss: 16.1173 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.6461 +[titan] 2025-07-09 23:58:37,039 - root - INFO - lr: 1.7915e-04 gnorm: 0.84 [10:23:37<11:37:53] +[titan] 2025-07-09 23:58:40,971 - root - INFO - step: 47195 loss: 16.4046 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8839 +[titan] 2025-07-09 23:58:40,972 - root - INFO - lr: 1.7913e-04 gnorm: 0.82 [10:23:41<11:37:49] +[titan] 2025-07-09 23:58:44,093 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:58:44,890 - root - INFO - step: 47200 loss: 16.8204 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5916 global_avg_mtp_loss: 14.2288 +[titan] 2025-07-09 23:58:44,891 - root - INFO - lr: 1.7911e-04 gnorm: 0.81 [10:23:45<11:37:45] +[titan] 2025-07-09 23:58:48,815 - root - INFO - step: 47205 loss: 16.5087 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5390 global_avg_mtp_loss: 13.9697 +[titan] 2025-07-09 23:58:48,815 - root - INFO - lr: 1.7909e-04 gnorm: 0.79 [10:23:49<11:37:41] +[titan] 2025-07-09 23:58:52,733 - root - INFO - step: 47210 loss: 16.4971 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9541 +[titan] 2025-07-09 23:58:52,734 - root - INFO - lr: 1.7907e-04 gnorm: 0.79 [10:23:53<11:37:37] +[titan] 2025-07-09 23:58:56,659 - root - INFO - step: 47215 loss: 16.0768 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6120 +[titan] 2025-07-09 23:58:56,659 - root - INFO - lr: 1.7905e-04 gnorm: 0.83 [10:23:56<11:37:33] +[titan] 2025-07-09 23:59:00,616 - root - INFO - step: 47220 loss: 16.6428 memory: 44.58GiB(31.99%) tps: 82,807 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0813 +[titan] 2025-07-09 23:59:00,616 - root - INFO - lr: 1.7902e-04 gnorm: 0.76 [10:24:00<11:37:29] +[titan] 2025-07-09 23:59:04,560 - root - INFO - step: 47225 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 14.0091 +[titan] 2025-07-09 23:59:04,560 - root - INFO - lr: 1.7900e-04 gnorm: 0.77 [10:24:04<11:37:25] +[titan] 2025-07-09 23:59:08,476 - root - INFO - step: 47230 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5541 global_avg_mtp_loss: 14.0667 +[titan] 2025-07-09 23:59:08,477 - root - INFO - lr: 1.7898e-04 gnorm: 0.83 [10:24:08<11:37:21] +[titan] 2025-07-09 23:59:12,387 - root - INFO - step: 47235 loss: 16.6594 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0858 +[titan] 2025-07-09 23:59:12,388 - root - INFO - lr: 1.7896e-04 gnorm: 0.86 [10:24:12<11:37:17] +[titan] 2025-07-09 23:59:16,302 - root - INFO - step: 47240 loss: 16.4510 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5390 global_avg_mtp_loss: 13.9120 +[titan] 2025-07-09 23:59:16,303 - root - INFO - lr: 1.7894e-04 gnorm: 0.76 [10:24:16<11:37:13] +[titan] 2025-07-09 23:59:20,213 - root - INFO - step: 47245 loss: 16.4325 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9006 +[titan] 2025-07-09 23:59:20,213 - root - INFO - lr: 1.7892e-04 gnorm: 0.78 [10:24:20<11:37:09] +[titan] 2025-07-09 23:59:23,337 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-09 23:59:24,134 - root - INFO - step: 47250 loss: 16.5828 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0353 +[titan] 2025-07-09 23:59:24,134 - root - INFO - lr: 1.7890e-04 gnorm: 0.84 [10:24:24<11:37:05] +[titan] 2025-07-09 23:59:28,063 - root - INFO - step: 47255 loss: 16.4900 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5287 global_avg_mtp_loss: 13.9613 +[titan] 2025-07-09 23:59:28,063 - root - INFO - lr: 1.7887e-04 gnorm: 0.83 [10:24:28<11:37:01] +[titan] 2025-07-09 23:59:31,974 - root - INFO - step: 47260 loss: 16.4395 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.9201 +[titan] 2025-07-09 23:59:31,974 - root - INFO - lr: 1.7885e-04 gnorm: 0.79 [10:24:32<11:36:57] +[titan] 2025-07-09 23:59:35,912 - root - INFO - step: 47265 loss: 16.6225 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 14.0641 +[titan] 2025-07-09 23:59:35,912 - root - INFO - lr: 1.7883e-04 gnorm: 0.81 [10:24:36<11:36:53] +[titan] 2025-07-09 23:59:39,862 - root - INFO - step: 47270 loss: 16.6486 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5691 global_avg_mtp_loss: 14.0795 +[titan] 2025-07-09 23:59:39,862 - root - INFO - lr: 1.7881e-04 gnorm: 0.81 [10:24:40<11:36:49] +[titan] 2025-07-09 23:59:43,795 - root - INFO - step: 47275 loss: 16.6250 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0561 +[titan] 2025-07-09 23:59:43,795 - root - INFO - lr: 1.7879e-04 gnorm: 0.80 [10:24:44<11:36:45] +[titan] 2025-07-09 23:59:47,712 - root - INFO - step: 47280 loss: 16.4488 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-09 23:59:47,713 - root - INFO - lr: 1.7877e-04 gnorm: 0.81 [10:24:48<11:36:41] +[titan] 2025-07-09 23:59:51,615 - root - INFO - step: 47285 loss: 16.4823 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-09 23:59:51,616 - root - INFO - lr: 1.7875e-04 gnorm: 0.83 [10:24:51<11:36:37] +[titan] 2025-07-09 23:59:55,539 - root - INFO - step: 47290 loss: 16.5359 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 13.9883 +[titan] 2025-07-09 23:59:55,539 - root - INFO - lr: 1.7873e-04 gnorm: 0.84 [10:24:55<11:36:33] +[titan] 2025-07-09 23:59:59,442 - root - INFO - step: 47295 loss: 16.6432 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0822 +[titan] 2025-07-09 23:59:59,442 - root - INFO - lr: 1.7870e-04 gnorm: 0.80 [10:24:59<11:36:29] +[titan] 2025-07-10 00:00:02,553 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:00:03,348 - root - INFO - step: 47300 loss: 17.0611 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.6568 global_avg_mtp_loss: 14.4043 +[titan] 2025-07-10 00:00:03,348 - root - INFO - lr: 1.7868e-04 gnorm: 0.84 [10:25:03<11:36:25] +[titan] 2025-07-10 00:00:07,265 - root - INFO - step: 47305 loss: 16.2799 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 00:00:07,265 - root - INFO - lr: 1.7866e-04 gnorm: 0.77 [10:25:07<11:36:21] +[titan] 2025-07-10 00:00:11,219 - root - INFO - step: 47310 loss: 16.5461 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-10 00:00:11,219 - root - INFO - lr: 1.7864e-04 gnorm: 0.82 [10:25:11<11:36:17] +[titan] 2025-07-10 00:00:15,116 - root - INFO - step: 47315 loss: 16.5282 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9832 +[titan] 2025-07-10 00:00:15,117 - root - INFO - lr: 1.7862e-04 gnorm: 0.81 [10:25:15<11:36:13] +[titan] 2025-07-10 00:00:19,020 - root - INFO - step: 47320 loss: 16.5969 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5543 global_avg_mtp_loss: 14.0426 +[titan] 2025-07-10 00:00:19,020 - root - INFO - lr: 1.7860e-04 gnorm: 0.77 [10:25:19<11:36:09] +[titan] 2025-07-10 00:00:22,918 - root - INFO - step: 47325 loss: 16.7997 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.2031 +[titan] 2025-07-10 00:00:22,918 - root - INFO - lr: 1.7858e-04 gnorm: 0.84 [10:25:23<11:36:05] +[titan] 2025-07-10 00:00:26,844 - root - INFO - step: 47330 loss: 16.3986 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8905 +[titan] 2025-07-10 00:00:26,845 - root - INFO - lr: 1.7856e-04 gnorm: 0.83 [10:25:27<11:36:01] +[titan] 2025-07-10 00:00:30,776 - root - INFO - step: 47335 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-10 00:00:30,776 - root - INFO - lr: 1.7853e-04 gnorm: 0.78 [10:25:31<11:35:57] +[titan] 2025-07-10 00:00:34,689 - root - INFO - step: 47340 loss: 16.5349 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.9987 +[titan] 2025-07-10 00:00:34,689 - root - INFO - lr: 1.7851e-04 gnorm: 0.81 [10:25:34<11:35:53] +[titan] 2025-07-10 00:00:38,617 - root - INFO - step: 47345 loss: 16.6465 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0777 +[titan] 2025-07-10 00:00:38,617 - root - INFO - lr: 1.7849e-04 gnorm: 0.84 [10:25:38<11:35:49] +[titan] 2025-07-10 00:00:41,720 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:00:42,512 - root - INFO - step: 47350 loss: 16.6746 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5794 global_avg_mtp_loss: 14.0952 +[titan] 2025-07-10 00:00:42,512 - root - INFO - lr: 1.7847e-04 gnorm: 0.85 [10:25:42<11:35:45] +[titan] 2025-07-10 00:00:46,415 - root - INFO - step: 47355 loss: 16.5932 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 14.0415 +[titan] 2025-07-10 00:00:46,415 - root - INFO - lr: 1.7845e-04 gnorm: 0.84 [10:25:46<11:35:41] +[titan] 2025-07-10 00:00:50,335 - root - INFO - step: 47360 loss: 16.4915 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-10 00:00:50,335 - root - INFO - lr: 1.7843e-04 gnorm: 0.84 [10:25:50<11:35:37] +[titan] 2025-07-10 00:00:54,251 - root - INFO - step: 47365 loss: 16.4022 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.8771 +[titan] 2025-07-10 00:00:54,252 - root - INFO - lr: 1.7841e-04 gnorm: 0.76 [10:25:54<11:35:32] +[titan] 2025-07-10 00:00:58,157 - root - INFO - step: 47370 loss: 16.6749 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5793 global_avg_mtp_loss: 14.0956 +[titan] 2025-07-10 00:00:58,157 - root - INFO - lr: 1.7838e-04 gnorm: 0.84 [10:25:58<11:35:28] +[titan] 2025-07-10 00:01:02,065 - root - INFO - step: 47375 loss: 16.4515 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9210 +[titan] 2025-07-10 00:01:02,065 - root - INFO - lr: 1.7836e-04 gnorm: 0.81 [10:26:02<11:35:24] +[titan] 2025-07-10 00:01:05,960 - root - INFO - step: 47380 loss: 16.5553 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0096 +[titan] 2025-07-10 00:01:05,960 - root - INFO - lr: 1.7834e-04 gnorm: 0.80 [10:26:06<11:35:20] +[titan] 2025-07-10 00:01:09,868 - root - INFO - step: 47385 loss: 16.5286 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9910 +[titan] 2025-07-10 00:01:09,869 - root - INFO - lr: 1.7832e-04 gnorm: 0.86 [10:26:10<11:35:16] +[titan] 2025-07-10 00:01:13,779 - root - INFO - step: 47390 loss: 16.7708 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5917 global_avg_mtp_loss: 14.1791 +[titan] 2025-07-10 00:01:13,780 - root - INFO - lr: 1.7830e-04 gnorm: 0.77 [10:26:14<11:35:12] +[titan] 2025-07-10 00:01:17,673 - root - INFO - step: 47395 loss: 16.7901 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.2062 +[titan] 2025-07-10 00:01:17,674 - root - INFO - lr: 1.7828e-04 gnorm: 0.79 [10:26:17<11:35:08] +[titan] 2025-07-10 00:01:20,785 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:01:21,572 - root - INFO - step: 47400 loss: 16.5569 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0072 +[titan] 2025-07-10 00:01:21,573 - root - INFO - lr: 1.7826e-04 gnorm: 0.86 [10:26:21<11:35:04] +[titan] 2025-07-10 00:01:25,467 - root - INFO - step: 47405 loss: 16.2128 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.7044 +[titan] 2025-07-10 00:01:25,467 - root - INFO - lr: 1.7824e-04 gnorm: 0.79 [10:26:25<11:35:00] +[titan] 2025-07-10 00:01:29,417 - root - INFO - step: 47410 loss: 16.4248 memory: 44.58GiB(31.99%) tps: 82,963 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 00:01:29,417 - root - INFO - lr: 1.7821e-04 gnorm: 0.84 [10:26:29<11:34:56] +[titan] 2025-07-10 00:01:33,325 - root - INFO - step: 47415 loss: 16.8295 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6083 global_avg_mtp_loss: 14.2213 +[titan] 2025-07-10 00:01:33,326 - root - INFO - lr: 1.7819e-04 gnorm: 0.77 [10:26:33<11:34:52] +[titan] 2025-07-10 00:01:37,273 - root - INFO - step: 47420 loss: 16.7492 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.1758 +[titan] 2025-07-10 00:01:37,273 - root - INFO - lr: 1.7817e-04 gnorm: 0.79 [10:26:37<11:34:48] +[titan] 2025-07-10 00:01:41,176 - root - INFO - step: 47425 loss: 16.6570 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0894 +[titan] 2025-07-10 00:01:41,176 - root - INFO - lr: 1.7815e-04 gnorm: 0.78 [10:26:41<11:34:44] +[titan] 2025-07-10 00:01:45,098 - root - INFO - step: 47430 loss: 16.3530 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8353 +[titan] 2025-07-10 00:01:45,099 - root - INFO - lr: 1.7813e-04 gnorm: 0.80 [10:26:45<11:34:40] +[titan] 2025-07-10 00:01:49,001 - root - INFO - step: 47435 loss: 16.6255 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0568 +[titan] 2025-07-10 00:01:49,001 - root - INFO - lr: 1.7811e-04 gnorm: 0.88 [10:26:49<11:34:36] +[titan] 2025-07-10 00:01:52,919 - root - INFO - step: 47440 loss: 16.6619 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.0916 +[titan] 2025-07-10 00:01:52,919 - root - INFO - lr: 1.7809e-04 gnorm: 0.80 [10:26:53<11:34:32] +[titan] 2025-07-10 00:01:56,884 - root - INFO - step: 47445 loss: 16.4202 memory: 44.58GiB(31.99%) tps: 82,655 tflops: 285.25 mfu: 28.84% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-10 00:01:56,884 - root - INFO - lr: 1.7806e-04 gnorm: 0.83 [10:26:57<11:34:28] +[titan] 2025-07-10 00:02:00,008 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:02:00,808 - root - INFO - step: 47450 loss: 16.4033 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.8807 +[titan] 2025-07-10 00:02:00,808 - root - INFO - lr: 1.7804e-04 gnorm: 0.87 [10:27:01<11:34:24] +[titan] 2025-07-10 00:02:04,743 - root - INFO - step: 47455 loss: 16.2071 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7323 +[titan] 2025-07-10 00:02:04,743 - root - INFO - lr: 1.7802e-04 gnorm: 0.82 [10:27:05<11:34:20] +[titan] 2025-07-10 00:02:08,671 - root - INFO - step: 47460 loss: 16.4335 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9056 +[titan] 2025-07-10 00:02:08,671 - root - INFO - lr: 1.7800e-04 gnorm: 0.81 [10:27:08<11:34:16] +[titan] 2025-07-10 00:02:12,631 - root - INFO - step: 47465 loss: 16.3561 memory: 44.58GiB(31.99%) tps: 82,743 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8482 +[titan] 2025-07-10 00:02:12,632 - root - INFO - lr: 1.7798e-04 gnorm: 0.76 [10:27:12<11:34:12] +[titan] 2025-07-10 00:02:16,549 - root - INFO - step: 47470 loss: 16.6705 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.1029 +[titan] 2025-07-10 00:02:16,550 - root - INFO - lr: 1.7796e-04 gnorm: 0.80 [10:27:16<11:34:08] +[titan] 2025-07-10 00:02:20,506 - root - INFO - step: 47475 loss: 16.6602 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0848 +[titan] 2025-07-10 00:02:20,507 - root - INFO - lr: 1.7794e-04 gnorm: 0.81 [10:27:20<11:34:04] +[titan] 2025-07-10 00:02:24,428 - root - INFO - step: 47480 loss: 16.4161 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.9051 +[titan] 2025-07-10 00:02:24,428 - root - INFO - lr: 1.7792e-04 gnorm: 0.80 [10:27:24<11:34:00] +[titan] 2025-07-10 00:02:28,361 - root - INFO - step: 47485 loss: 16.5597 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 14.0191 +[titan] 2025-07-10 00:02:28,361 - root - INFO - lr: 1.7789e-04 gnorm: 0.82 [10:27:28<11:33:56] +[titan] 2025-07-10 00:02:32,269 - root - INFO - step: 47490 loss: 16.7377 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1520 +[titan] 2025-07-10 00:02:32,270 - root - INFO - lr: 1.7787e-04 gnorm: 0.85 [10:27:32<11:33:52] +[titan] 2025-07-10 00:02:36,190 - root - INFO - step: 47495 loss: 16.2107 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7181 +[titan] 2025-07-10 00:02:36,191 - root - INFO - lr: 1.7785e-04 gnorm: 0.80 [10:27:36<11:33:48] +[titan] 2025-07-10 00:02:39,327 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:02:40,121 - root - INFO - step: 47500 loss: 16.5624 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-10 00:02:40,121 - root - INFO - lr: 1.7783e-04 gnorm: 0.79 [10:27:40<11:33:44] +[titan] 2025-07-10 00:02:44,047 - root - INFO - step: 47505 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8027 +[titan] 2025-07-10 00:02:44,047 - root - INFO - lr: 1.7781e-04 gnorm: 0.78 [10:27:44<11:33:40] +[titan] 2025-07-10 00:02:47,949 - root - INFO - step: 47510 loss: 16.7136 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1296 +[titan] 2025-07-10 00:02:47,950 - root - INFO - lr: 1.7779e-04 gnorm: 0.80 [10:27:48<11:33:36] +[titan] 2025-07-10 00:02:51,866 - root - INFO - step: 47515 loss: 16.3176 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8124 +[titan] 2025-07-10 00:02:51,866 - root - INFO - lr: 1.7777e-04 gnorm: 0.82 [10:27:52<11:33:32] +[titan] 2025-07-10 00:02:55,789 - root - INFO - step: 47520 loss: 16.5847 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 14.0358 +[titan] 2025-07-10 00:02:55,790 - root - INFO - lr: 1.7775e-04 gnorm: 0.77 [10:27:56<11:33:28] +[titan] 2025-07-10 00:02:59,741 - root - INFO - step: 47525 loss: 16.2950 memory: 44.58GiB(31.99%) tps: 82,933 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.7874 +[titan] 2025-07-10 00:02:59,741 - root - INFO - lr: 1.7772e-04 gnorm: 0.83 [10:28:00<11:33:24] +[titan] 2025-07-10 00:03:03,655 - root - INFO - step: 47530 loss: 16.6713 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5682 global_avg_mtp_loss: 14.1031 +[titan] 2025-07-10 00:03:03,655 - root - INFO - lr: 1.7770e-04 gnorm: 0.82 [10:28:03<11:33:20] +[titan] 2025-07-10 00:03:07,567 - root - INFO - step: 47535 loss: 16.7192 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5871 global_avg_mtp_loss: 14.1321 +[titan] 2025-07-10 00:03:07,568 - root - INFO - lr: 1.7768e-04 gnorm: 0.82 [10:28:07<11:33:16] +[titan] 2025-07-10 00:03:11,524 - root - INFO - step: 47540 loss: 16.4119 memory: 44.58GiB(31.99%) tps: 82,821 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.8885 +[titan] 2025-07-10 00:03:11,524 - root - INFO - lr: 1.7766e-04 gnorm: 0.79 [10:28:11<11:33:12] +[titan] 2025-07-10 00:03:15,442 - root - INFO - step: 47545 loss: 16.5573 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 14.0082 +[titan] 2025-07-10 00:03:15,442 - root - INFO - lr: 1.7764e-04 gnorm: 0.87 [10:28:15<11:33:08] +[titan] 2025-07-10 00:03:18,568 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:03:19,363 - root - INFO - step: 47550 loss: 16.3955 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8722 +[titan] 2025-07-10 00:03:19,363 - root - INFO - lr: 1.7762e-04 gnorm: 0.86 [10:28:19<11:33:04] +[titan] 2025-07-10 00:03:23,287 - root - INFO - step: 47555 loss: 16.6075 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-10 00:03:23,288 - root - INFO - lr: 1.7760e-04 gnorm: 0.85 [10:28:23<11:33:00] +[titan] 2025-07-10 00:03:27,208 - root - INFO - step: 47560 loss: 16.5389 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5499 global_avg_mtp_loss: 13.9889 +[titan] 2025-07-10 00:03:27,208 - root - INFO - lr: 1.7757e-04 gnorm: 0.78 [10:28:27<11:32:56] +[titan] 2025-07-10 00:03:31,111 - root - INFO - step: 47565 loss: 16.4961 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.9708 +[titan] 2025-07-10 00:03:31,111 - root - INFO - lr: 1.7755e-04 gnorm: 0.79 [10:28:31<11:32:52] +[titan] 2025-07-10 00:03:35,047 - root - INFO - step: 47570 loss: 16.5623 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-10 00:03:35,047 - root - INFO - lr: 1.7753e-04 gnorm: 0.79 [10:28:35<11:32:48] +[titan] 2025-07-10 00:03:39,005 - root - INFO - step: 47575 loss: 16.4781 memory: 44.58GiB(31.99%) tps: 82,792 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 00:03:39,005 - root - INFO - lr: 1.7751e-04 gnorm: 0.76 [10:28:39<11:32:44] +[titan] 2025-07-10 00:03:42,931 - root - INFO - step: 47580 loss: 16.7316 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5828 global_avg_mtp_loss: 14.1488 +[titan] 2025-07-10 00:03:42,932 - root - INFO - lr: 1.7749e-04 gnorm: 0.82 [10:28:43<11:32:40] +[titan] 2025-07-10 00:03:46,854 - root - INFO - step: 47585 loss: 16.4684 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9368 +[titan] 2025-07-10 00:03:46,854 - root - INFO - lr: 1.7747e-04 gnorm: 0.80 [10:28:47<11:32:36] +[titan] 2025-07-10 00:03:50,802 - root - INFO - step: 47590 loss: 16.6056 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5548 global_avg_mtp_loss: 14.0507 +[titan] 2025-07-10 00:03:50,802 - root - INFO - lr: 1.7745e-04 gnorm: 0.81 [10:28:51<11:32:32] +[titan] 2025-07-10 00:03:54,699 - root - INFO - step: 47595 loss: 16.2519 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7592 +[titan] 2025-07-10 00:03:54,699 - root - INFO - lr: 1.7743e-04 gnorm: 0.82 [10:28:54<11:32:28] +[titan] 2025-07-10 00:03:57,835 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:03:58,629 - root - INFO - step: 47600 loss: 16.8034 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5960 global_avg_mtp_loss: 14.2074 +[titan] 2025-07-10 00:03:58,629 - root - INFO - lr: 1.7740e-04 gnorm: 0.81 [10:28:58<11:32:24] +[titan] 2025-07-10 00:04:02,564 - root - INFO - step: 47605 loss: 16.7932 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5933 global_avg_mtp_loss: 14.1998 +[titan] 2025-07-10 00:04:02,564 - root - INFO - lr: 1.7738e-04 gnorm: 0.94 [10:29:02<11:32:20] +[titan] 2025-07-10 00:04:06,477 - root - INFO - step: 47610 loss: 16.6346 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0706 +[titan] 2025-07-10 00:04:06,477 - root - INFO - lr: 1.7736e-04 gnorm: 0.79 [10:29:06<11:32:16] +[titan] 2025-07-10 00:04:10,468 - root - INFO - step: 47615 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 82,108 tflops: 283.37 mfu: 28.65% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.8801 +[titan] 2025-07-10 00:04:10,468 - root - INFO - lr: 1.7734e-04 gnorm: 0.79 [10:29:10<11:32:12] +[titan] 2025-07-10 00:04:11,422 - root - INFO - Dumping profiler traces at step 47616 +[titan] 2025-07-10 00:04:11,454 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:04:14,606 - root - INFO - step: 47620 loss: 16.5635 memory: 44.58GiB(31.99%) tps: 79,189 tflops: 273.29 mfu: 27.63% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 14.0140 +[titan] 2025-07-10 00:04:14,606 - root - INFO - lr: 1.7732e-04 gnorm: 0.78 [10:29:14<11:32:08] +[titan] 2025-07-10 00:04:18,510 - root - INFO - step: 47625 loss: 16.6061 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0554 +[titan] 2025-07-10 00:04:18,511 - root - INFO - lr: 1.7730e-04 gnorm: 0.80 [10:29:18<11:32:04] +[titan] 2025-07-10 00:04:22,435 - root - INFO - step: 47630 loss: 16.6865 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.1198 +[titan] 2025-07-10 00:04:22,435 - root - INFO - lr: 1.7728e-04 gnorm: 0.86 [10:29:22<11:32:00] +[titan] 2025-07-10 00:04:26,355 - root - INFO - step: 47635 loss: 16.5402 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9947 +[titan] 2025-07-10 00:04:26,355 - root - INFO - lr: 1.7725e-04 gnorm: 0.84 [10:29:26<11:31:56] +[titan] 2025-07-10 00:04:30,262 - root - INFO - step: 47640 loss: 16.4256 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.8936 +[titan] 2025-07-10 00:04:30,262 - root - INFO - lr: 1.7723e-04 gnorm: 0.80 [10:29:30<11:31:52] +[titan] 2025-07-10 00:04:34,237 - root - INFO - step: 47645 loss: 16.1154 memory: 44.58GiB(31.99%) tps: 82,428 tflops: 284.47 mfu: 28.76% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.6330 +[titan] 2025-07-10 00:04:34,237 - root - INFO - lr: 1.7721e-04 gnorm: 0.76 [10:29:34<11:31:48] +[titan] 2025-07-10 00:04:37,368 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:04:38,173 - root - INFO - step: 47650 loss: 16.5945 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0423 +[titan] 2025-07-10 00:04:38,174 - root - INFO - lr: 1.7719e-04 gnorm: 0.80 [10:29:38<11:31:44] +[titan] 2025-07-10 00:04:42,109 - root - INFO - step: 47655 loss: 16.9023 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.6351 global_avg_mtp_loss: 14.2672 +[titan] 2025-07-10 00:04:42,109 - root - INFO - lr: 1.7717e-04 gnorm: 0.88 [10:29:42<11:31:40] +[titan] 2025-07-10 00:04:46,016 - root - INFO - step: 47660 loss: 16.6651 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0921 +[titan] 2025-07-10 00:04:46,017 - root - INFO - lr: 1.7715e-04 gnorm: 0.79 [10:29:46<11:31:36] +[titan] 2025-07-10 00:04:49,945 - root - INFO - step: 47665 loss: 16.6248 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.0549 +[titan] 2025-07-10 00:04:49,945 - root - INFO - lr: 1.7713e-04 gnorm: 0.81 [10:29:50<11:31:32] +[titan] 2025-07-10 00:04:53,844 - root - INFO - step: 47670 loss: 16.6720 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.1067 +[titan] 2025-07-10 00:04:53,844 - root - INFO - lr: 1.7711e-04 gnorm: 0.81 [10:29:54<11:31:28] +[titan] 2025-07-10 00:04:57,752 - root - INFO - step: 47675 loss: 16.3228 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.8149 +[titan] 2025-07-10 00:04:57,753 - root - INFO - lr: 1.7708e-04 gnorm: 0.79 [10:29:58<11:31:24] +[titan] 2025-07-10 00:05:01,652 - root - INFO - step: 47680 loss: 16.4908 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9624 +[titan] 2025-07-10 00:05:01,652 - root - INFO - lr: 1.7706e-04 gnorm: 0.78 [10:30:01<11:31:20] +[titan] 2025-07-10 00:05:05,555 - root - INFO - step: 47685 loss: 16.5252 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9794 +[titan] 2025-07-10 00:05:05,555 - root - INFO - lr: 1.7704e-04 gnorm: 0.80 [10:30:05<11:31:16] +[titan] 2025-07-10 00:05:09,446 - root - INFO - step: 47690 loss: 16.3171 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8180 +[titan] 2025-07-10 00:05:09,446 - root - INFO - lr: 1.7702e-04 gnorm: 0.78 [10:30:09<11:31:12] +[titan] 2025-07-10 00:05:13,392 - root - INFO - step: 47695 loss: 16.6362 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-10 00:05:13,392 - root - INFO - lr: 1.7700e-04 gnorm: 0.77 [10:30:13<11:31:08] +[titan] 2025-07-10 00:05:16,526 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:05:17,318 - root - INFO - step: 47700 loss: 16.5749 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0171 +[titan] 2025-07-10 00:05:17,318 - root - INFO - lr: 1.7698e-04 gnorm: 0.80 [10:30:17<11:31:04] +[titan] 2025-07-10 00:05:21,257 - root - INFO - step: 47705 loss: 16.7004 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.1294 +[titan] 2025-07-10 00:05:21,257 - root - INFO - lr: 1.7696e-04 gnorm: 0.75 [10:30:21<11:31:00] +[titan] 2025-07-10 00:05:25,201 - root - INFO - step: 47710 loss: 16.5498 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 14.0025 +[titan] 2025-07-10 00:05:25,202 - root - INFO - lr: 1.7693e-04 gnorm: 0.84 [10:30:25<11:30:56] +[titan] 2025-07-10 00:05:29,181 - root - INFO - step: 47715 loss: 16.5058 memory: 44.58GiB(31.99%) tps: 82,352 tflops: 284.21 mfu: 28.74% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 13.9586 +[titan] 2025-07-10 00:05:29,181 - root - INFO - lr: 1.7691e-04 gnorm: 0.80 [10:30:29<11:30:52] +[titan] 2025-07-10 00:05:33,111 - root - INFO - step: 47720 loss: 16.4484 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9213 +[titan] 2025-07-10 00:05:33,112 - root - INFO - lr: 1.7689e-04 gnorm: 0.79 [10:30:33<11:30:48] +[titan] 2025-07-10 00:05:37,035 - root - INFO - step: 47725 loss: 16.3029 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.7982 +[titan] 2025-07-10 00:05:37,035 - root - INFO - lr: 1.7687e-04 gnorm: 0.79 [10:30:37<11:30:44] +[titan] 2025-07-10 00:05:40,952 - root - INFO - step: 47730 loss: 16.4769 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.9328 +[titan] 2025-07-10 00:05:40,952 - root - INFO - lr: 1.7685e-04 gnorm: 0.80 [10:30:41<11:30:40] +[titan] 2025-07-10 00:05:44,884 - root - INFO - step: 47735 loss: 16.7065 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.1479 +[titan] 2025-07-10 00:05:44,885 - root - INFO - lr: 1.7683e-04 gnorm: 0.77 [10:30:45<11:30:36] +[titan] 2025-07-10 00:05:48,815 - root - INFO - step: 47740 loss: 16.6586 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-10 00:05:48,816 - root - INFO - lr: 1.7681e-04 gnorm: 0.78 [10:30:49<11:30:32] +[titan] 2025-07-10 00:05:52,728 - root - INFO - step: 47745 loss: 16.7393 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5854 global_avg_mtp_loss: 14.1539 +[titan] 2025-07-10 00:05:52,728 - root - INFO - lr: 1.7679e-04 gnorm: 0.80 [10:30:52<11:30:28] +[titan] 2025-07-10 00:05:55,841 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:05:56,631 - root - INFO - step: 47750 loss: 16.2497 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7521 +[titan] 2025-07-10 00:05:56,631 - root - INFO - lr: 1.7676e-04 gnorm: 0.80 [10:30:56<11:30:24] +[titan] 2025-07-10 00:06:00,570 - root - INFO - step: 47755 loss: 16.6059 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 14.0545 +[titan] 2025-07-10 00:06:00,571 - root - INFO - lr: 1.7674e-04 gnorm: 0.82 [10:31:00<11:30:20] +[titan] 2025-07-10 00:06:04,503 - root - INFO - step: 47760 loss: 16.6327 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0756 +[titan] 2025-07-10 00:06:04,503 - root - INFO - lr: 1.7672e-04 gnorm: 0.77 [10:31:04<11:30:16] +[titan] 2025-07-10 00:06:08,417 - root - INFO - step: 47765 loss: 16.7951 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5870 global_avg_mtp_loss: 14.2080 +[titan] 2025-07-10 00:06:08,417 - root - INFO - lr: 1.7670e-04 gnorm: 0.82 [10:31:08<11:30:12] +[titan] 2025-07-10 00:06:12,370 - root - INFO - step: 47770 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 82,906 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9279 +[titan] 2025-07-10 00:06:12,370 - root - INFO - lr: 1.7668e-04 gnorm: 0.80 [10:31:12<11:30:08] +[titan] 2025-07-10 00:06:16,309 - root - INFO - step: 47775 loss: 16.6317 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0666 +[titan] 2025-07-10 00:06:16,310 - root - INFO - lr: 1.7666e-04 gnorm: 0.82 [10:31:16<11:30:04] +[titan] 2025-07-10 00:06:20,242 - root - INFO - step: 47780 loss: 16.2925 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 00:06:20,242 - root - INFO - lr: 1.7664e-04 gnorm: 0.86 [10:31:20<11:30:00] +[titan] 2025-07-10 00:06:24,188 - root - INFO - step: 47785 loss: 16.2194 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7293 +[titan] 2025-07-10 00:06:24,189 - root - INFO - lr: 1.7661e-04 gnorm: 0.86 [10:31:24<11:29:56] +[titan] 2025-07-10 00:06:28,135 - root - INFO - step: 47790 loss: 16.5318 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9909 +[titan] 2025-07-10 00:06:28,135 - root - INFO - lr: 1.7659e-04 gnorm: 0.80 [10:31:28<11:29:52] +[titan] 2025-07-10 00:06:32,038 - root - INFO - step: 47795 loss: 16.9412 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.6279 global_avg_mtp_loss: 14.3133 +[titan] 2025-07-10 00:06:32,038 - root - INFO - lr: 1.7657e-04 gnorm: 0.80 [10:31:32<11:29:48] +[titan] 2025-07-10 00:06:35,183 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:06:35,990 - root - INFO - step: 47800 loss: 16.3821 memory: 44.58GiB(31.99%) tps: 82,934 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8647 +[titan] 2025-07-10 00:06:35,990 - root - INFO - lr: 1.7655e-04 gnorm: 0.80 [10:31:36<11:29:44] +[titan] 2025-07-10 00:06:39,911 - root - INFO - step: 47805 loss: 16.6423 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0605 +[titan] 2025-07-10 00:06:39,911 - root - INFO - lr: 1.7653e-04 gnorm: 0.84 [10:31:40<11:29:40] +[titan] 2025-07-10 00:06:43,822 - root - INFO - step: 47810 loss: 16.5428 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 13.9852 +[titan] 2025-07-10 00:06:43,822 - root - INFO - lr: 1.7651e-04 gnorm: 0.79 [10:31:44<11:29:36] +[titan] 2025-07-10 00:06:47,751 - root - INFO - step: 47815 loss: 16.1880 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7088 +[titan] 2025-07-10 00:06:47,751 - root - INFO - lr: 1.7649e-04 gnorm: 0.80 [10:31:48<11:29:32] +[titan] 2025-07-10 00:06:51,701 - root - INFO - step: 47820 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9279 +[titan] 2025-07-10 00:06:51,702 - root - INFO - lr: 1.7647e-04 gnorm: 0.84 [10:31:51<11:29:28] +[titan] 2025-07-10 00:06:55,643 - root - INFO - step: 47825 loss: 16.6745 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.1085 +[titan] 2025-07-10 00:06:55,643 - root - INFO - lr: 1.7644e-04 gnorm: 0.86 [10:31:55<11:29:24] +[titan] 2025-07-10 00:06:59,571 - root - INFO - step: 47830 loss: 16.4674 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9329 +[titan] 2025-07-10 00:06:59,572 - root - INFO - lr: 1.7642e-04 gnorm: 0.83 [10:31:59<11:29:20] +[titan] 2025-07-10 00:07:03,527 - root - INFO - step: 47835 loss: 16.6572 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0918 +[titan] 2025-07-10 00:07:03,527 - root - INFO - lr: 1.7640e-04 gnorm: 0.83 [10:32:03<11:29:16] +[titan] 2025-07-10 00:07:07,441 - root - INFO - step: 47840 loss: 16.5045 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9706 +[titan] 2025-07-10 00:07:07,441 - root - INFO - lr: 1.7638e-04 gnorm: 0.80 [10:32:07<11:29:12] +[titan] 2025-07-10 00:07:11,348 - root - INFO - step: 47845 loss: 16.6512 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0926 +[titan] 2025-07-10 00:07:11,348 - root - INFO - lr: 1.7636e-04 gnorm: 0.79 [10:32:11<11:29:08] +[titan] 2025-07-10 00:07:14,459 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:07:15,253 - root - INFO - step: 47850 loss: 16.7777 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.1859 +[titan] 2025-07-10 00:07:15,253 - root - INFO - lr: 1.7634e-04 gnorm: 0.82 [10:32:15<11:29:04] +[titan] 2025-07-10 00:07:19,162 - root - INFO - step: 47855 loss: 16.4113 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.8934 +[titan] 2025-07-10 00:07:19,163 - root - INFO - lr: 1.7632e-04 gnorm: 0.84 [10:32:19<11:29:00] +[titan] 2025-07-10 00:07:23,075 - root - INFO - step: 47860 loss: 16.6334 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0813 +[titan] 2025-07-10 00:07:23,075 - root - INFO - lr: 1.7629e-04 gnorm: 0.78 [10:32:23<11:28:56] +[titan] 2025-07-10 00:07:26,990 - root - INFO - step: 47865 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8684 +[titan] 2025-07-10 00:07:26,990 - root - INFO - lr: 1.7627e-04 gnorm: 0.84 [10:32:27<11:28:52] +[titan] 2025-07-10 00:07:30,906 - root - INFO - step: 47870 loss: 16.4557 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 00:07:30,906 - root - INFO - lr: 1.7625e-04 gnorm: 0.82 [10:32:31<11:28:48] +[titan] 2025-07-10 00:07:34,800 - root - INFO - step: 47875 loss: 16.4220 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-10 00:07:34,800 - root - INFO - lr: 1.7623e-04 gnorm: 0.83 [10:32:35<11:28:44] +[titan] 2025-07-10 00:07:38,711 - root - INFO - step: 47880 loss: 16.2199 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7298 +[titan] 2025-07-10 00:07:38,711 - root - INFO - lr: 1.7621e-04 gnorm: 0.84 [10:32:38<11:28:40] +[titan] 2025-07-10 00:07:42,639 - root - INFO - step: 47885 loss: 16.7362 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1553 +[titan] 2025-07-10 00:07:42,639 - root - INFO - lr: 1.7619e-04 gnorm: 0.78 [10:32:42<11:28:36] +[titan] 2025-07-10 00:07:46,536 - root - INFO - step: 47890 loss: 16.7384 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1555 +[titan] 2025-07-10 00:07:46,536 - root - INFO - lr: 1.7617e-04 gnorm: 0.81 [10:32:46<11:28:32] +[titan] 2025-07-10 00:07:50,432 - root - INFO - step: 47895 loss: 16.2390 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7583 +[titan] 2025-07-10 00:07:50,432 - root - INFO - lr: 1.7614e-04 gnorm: 0.75 [10:32:50<11:28:28] +[titan] 2025-07-10 00:07:53,536 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:07:54,324 - root - INFO - step: 47900 loss: 16.7078 memory: 44.58GiB(31.99%) tps: 84,192 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1188 +[titan] 2025-07-10 00:07:54,324 - root - INFO - lr: 1.7612e-04 gnorm: 0.77 [10:32:54<11:28:24] +[titan] 2025-07-10 00:07:58,239 - root - INFO - step: 47905 loss: 16.4435 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9189 +[titan] 2025-07-10 00:07:58,239 - root - INFO - lr: 1.7610e-04 gnorm: 0.79 [10:32:58<11:28:20] +[titan] 2025-07-10 00:08:02,144 - root - INFO - step: 47910 loss: 16.7071 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5713 global_avg_mtp_loss: 14.1358 +[titan] 2025-07-10 00:08:02,144 - root - INFO - lr: 1.7608e-04 gnorm: 0.83 [10:33:02<11:28:16] +[titan] 2025-07-10 00:08:06,059 - root - INFO - step: 47915 loss: 16.5628 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 14.0211 +[titan] 2025-07-10 00:08:06,059 - root - INFO - lr: 1.7606e-04 gnorm: 0.85 [10:33:06<11:28:12] +[titan] 2025-07-10 00:08:09,974 - root - INFO - step: 47920 loss: 16.5149 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 13.9746 +[titan] 2025-07-10 00:08:09,975 - root - INFO - lr: 1.7604e-04 gnorm: 0.85 [10:33:10<11:28:08] +[titan] 2025-07-10 00:08:13,928 - root - INFO - step: 47925 loss: 16.5503 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 00:08:13,928 - root - INFO - lr: 1.7602e-04 gnorm: 0.80 [10:33:14<11:28:04] +[titan] 2025-07-10 00:08:17,879 - root - INFO - step: 47930 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 82,939 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5568 global_avg_mtp_loss: 14.0497 +[titan] 2025-07-10 00:08:17,880 - root - INFO - lr: 1.7600e-04 gnorm: 0.92 [10:33:18<11:28:00] +[titan] 2025-07-10 00:08:21,808 - root - INFO - step: 47935 loss: 16.5888 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0220 +[titan] 2025-07-10 00:08:21,808 - root - INFO - lr: 1.7597e-04 gnorm: 0.84 [10:33:22<11:27:56] +[titan] 2025-07-10 00:08:25,709 - root - INFO - step: 47940 loss: 16.4696 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 13.9293 +[titan] 2025-07-10 00:08:25,710 - root - INFO - lr: 1.7595e-04 gnorm: 0.82 [10:33:25<11:27:52] +[titan] 2025-07-10 00:08:29,637 - root - INFO - step: 47945 loss: 17.0008 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.6494 global_avg_mtp_loss: 14.3514 +[titan] 2025-07-10 00:08:29,638 - root - INFO - lr: 1.7593e-04 gnorm: 0.80 [10:33:29<11:27:48] +[titan] 2025-07-10 00:08:32,769 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:08:33,585 - root - INFO - step: 47950 loss: 16.5133 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9633 +[titan] 2025-07-10 00:08:33,585 - root - INFO - lr: 1.7591e-04 gnorm: 0.82 [10:33:33<11:27:44] +[titan] 2025-07-10 00:08:37,503 - root - INFO - step: 47955 loss: 16.4976 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.9533 +[titan] 2025-07-10 00:08:37,503 - root - INFO - lr: 1.7589e-04 gnorm: 0.80 [10:33:37<11:27:40] +[titan] 2025-07-10 00:08:41,432 - root - INFO - step: 47960 loss: 16.5799 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0263 +[titan] 2025-07-10 00:08:41,432 - root - INFO - lr: 1.7587e-04 gnorm: 0.83 [10:33:41<11:27:36] +[titan] 2025-07-10 00:08:45,349 - root - INFO - step: 47965 loss: 16.4888 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9562 +[titan] 2025-07-10 00:08:45,349 - root - INFO - lr: 1.7585e-04 gnorm: 0.76 [10:33:45<11:27:32] +[titan] 2025-07-10 00:08:49,249 - root - INFO - step: 47970 loss: 16.8433 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6035 global_avg_mtp_loss: 14.2397 +[titan] 2025-07-10 00:08:49,249 - root - INFO - lr: 1.7582e-04 gnorm: 0.84 [10:33:49<11:27:28] +[titan] 2025-07-10 00:08:53,245 - root - INFO - step: 47975 loss: 16.2920 memory: 44.58GiB(31.99%) tps: 82,014 tflops: 283.04 mfu: 28.62% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.7886 +[titan] 2025-07-10 00:08:53,245 - root - INFO - lr: 1.7580e-04 gnorm: 0.81 [10:33:53<11:27:24] +[titan] 2025-07-10 00:08:57,158 - root - INFO - step: 47980 loss: 16.5491 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 13.9940 +[titan] 2025-07-10 00:08:57,158 - root - INFO - lr: 1.7578e-04 gnorm: 0.77 [10:33:57<11:27:20] +[titan] 2025-07-10 00:09:01,120 - root - INFO - step: 47985 loss: 16.5677 memory: 44.58GiB(31.99%) tps: 82,704 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-10 00:09:01,121 - root - INFO - lr: 1.7576e-04 gnorm: 0.80 [10:34:01<11:27:16] +[titan] 2025-07-10 00:09:05,038 - root - INFO - step: 47990 loss: 16.3707 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.8468 +[titan] 2025-07-10 00:09:05,038 - root - INFO - lr: 1.7574e-04 gnorm: 0.90 [10:34:05<11:27:12] +[titan] 2025-07-10 00:09:08,944 - root - INFO - step: 47995 loss: 16.6848 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.1118 +[titan] 2025-07-10 00:09:08,944 - root - INFO - lr: 1.7572e-04 gnorm: 0.80 [10:34:09<11:27:08] +[titan] 2025-07-10 00:09:12,056 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:09:12,848 - root - INFO - step: 48000 loss: 16.5490 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 14.0077 +[titan] 2025-07-10 00:09:12,848 - root - INFO - lr: 1.7570e-04 gnorm: 0.81 [10:34:13<11:27:04] +[titan] 2025-07-10 00:09:16,768 - root - INFO - step: 48005 loss: 16.8624 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6033 global_avg_mtp_loss: 14.2590 +[titan] 2025-07-10 00:09:16,768 - root - INFO - lr: 1.7568e-04 gnorm: 0.83 [10:34:17<11:27:00] +[titan] 2025-07-10 00:09:20,686 - root - INFO - step: 48010 loss: 16.5388 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 13.9874 +[titan] 2025-07-10 00:09:20,686 - root - INFO - lr: 1.7565e-04 gnorm: 0.84 [10:34:20<11:26:56] +[titan] 2025-07-10 00:09:24,599 - root - INFO - step: 48015 loss: 16.7638 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5937 global_avg_mtp_loss: 14.1700 +[titan] 2025-07-10 00:09:24,599 - root - INFO - lr: 1.7563e-04 gnorm: 0.79 [10:34:24<11:26:52] +[titan] 2025-07-10 00:09:28,496 - root - INFO - step: 48020 loss: 16.3825 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8616 +[titan] 2025-07-10 00:09:28,497 - root - INFO - lr: 1.7561e-04 gnorm: 0.82 [10:34:28<11:26:48] +[titan] 2025-07-10 00:09:32,451 - root - INFO - step: 48025 loss: 16.8032 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.6081 global_avg_mtp_loss: 14.1951 +[titan] 2025-07-10 00:09:32,451 - root - INFO - lr: 1.7559e-04 gnorm: 0.76 [10:34:32<11:26:44] +[titan] 2025-07-10 00:09:36,361 - root - INFO - step: 48030 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 00:09:36,361 - root - INFO - lr: 1.7557e-04 gnorm: 0.77 [10:34:36<11:26:40] +[titan] 2025-07-10 00:09:40,291 - root - INFO - step: 48035 loss: 16.6064 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0487 +[titan] 2025-07-10 00:09:40,292 - root - INFO - lr: 1.7555e-04 gnorm: 0.81 [10:34:40<11:26:36] +[titan] 2025-07-10 00:09:44,230 - root - INFO - step: 48040 loss: 16.5826 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0293 +[titan] 2025-07-10 00:09:44,231 - root - INFO - lr: 1.7553e-04 gnorm: 0.83 [10:34:44<11:26:32] +[titan] 2025-07-10 00:09:48,141 - root - INFO - step: 48045 loss: 16.5201 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 13.9671 +[titan] 2025-07-10 00:09:48,141 - root - INFO - lr: 1.7550e-04 gnorm: 0.83 [10:34:48<11:26:28] +[titan] 2025-07-10 00:09:51,246 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:09:52,031 - root - INFO - step: 48050 loss: 16.4837 memory: 44.58GiB(31.99%) tps: 84,234 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.9604 +[titan] 2025-07-10 00:09:52,031 - root - INFO - lr: 1.7548e-04 gnorm: 0.78 [10:34:52<11:26:24] +[titan] 2025-07-10 00:09:55,957 - root - INFO - step: 48055 loss: 16.7808 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1883 +[titan] 2025-07-10 00:09:55,957 - root - INFO - lr: 1.7546e-04 gnorm: 0.83 [10:34:56<11:26:20] +[titan] 2025-07-10 00:09:59,852 - root - INFO - step: 48060 loss: 16.5074 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5446 global_avg_mtp_loss: 13.9627 +[titan] 2025-07-10 00:09:59,853 - root - INFO - lr: 1.7544e-04 gnorm: 0.87 [10:35:00<11:26:15] +[titan] 2025-07-10 00:10:03,746 - root - INFO - step: 48065 loss: 16.5722 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 14.0263 +[titan] 2025-07-10 00:10:03,746 - root - INFO - lr: 1.7542e-04 gnorm: 0.83 [10:35:03<11:26:11] +[titan] 2025-07-10 00:10:07,659 - root - INFO - step: 48070 loss: 16.6566 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5710 global_avg_mtp_loss: 14.0857 +[titan] 2025-07-10 00:10:07,659 - root - INFO - lr: 1.7540e-04 gnorm: 0.79 [10:35:07<11:26:07] +[titan] 2025-07-10 00:10:11,582 - root - INFO - step: 48075 loss: 16.4160 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.8969 +[titan] 2025-07-10 00:10:11,582 - root - INFO - lr: 1.7538e-04 gnorm: 0.80 [10:35:11<11:26:03] +[titan] 2025-07-10 00:10:15,535 - root - INFO - step: 48080 loss: 16.5283 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9887 +[titan] 2025-07-10 00:10:15,536 - root - INFO - lr: 1.7535e-04 gnorm: 0.86 [10:35:15<11:25:59] +[titan] 2025-07-10 00:10:19,455 - root - INFO - step: 48085 loss: 16.4076 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.8853 +[titan] 2025-07-10 00:10:19,456 - root - INFO - lr: 1.7533e-04 gnorm: 0.83 [10:35:19<11:25:55] +[titan] 2025-07-10 00:10:23,396 - root - INFO - step: 48090 loss: 16.6351 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5669 global_avg_mtp_loss: 14.0681 +[titan] 2025-07-10 00:10:23,396 - root - INFO - lr: 1.7531e-04 gnorm: 0.89 [10:35:23<11:25:51] +[titan] 2025-07-10 00:10:27,317 - root - INFO - step: 48095 loss: 16.4963 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9649 +[titan] 2025-07-10 00:10:27,318 - root - INFO - lr: 1.7529e-04 gnorm: 0.89 [10:35:27<11:25:47] +[titan] 2025-07-10 00:10:30,473 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:10:31,273 - root - INFO - step: 48100 loss: 16.5825 memory: 44.58GiB(31.99%) tps: 82,854 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 14.0206 +[titan] 2025-07-10 00:10:31,273 - root - INFO - lr: 1.7527e-04 gnorm: 0.88 [10:35:31<11:25:43] +[titan] 2025-07-10 00:10:35,194 - root - INFO - step: 48105 loss: 16.3553 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8360 +[titan] 2025-07-10 00:10:35,194 - root - INFO - lr: 1.7525e-04 gnorm: 0.79 [10:35:35<11:25:39] +[titan] 2025-07-10 00:10:39,162 - root - INFO - step: 48110 loss: 16.5329 memory: 44.58GiB(31.99%) tps: 82,596 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9910 +[titan] 2025-07-10 00:10:39,162 - root - INFO - lr: 1.7523e-04 gnorm: 0.84 [10:35:39<11:25:35] +[titan] 2025-07-10 00:10:43,095 - root - INFO - step: 48115 loss: 16.6340 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0656 +[titan] 2025-07-10 00:10:43,095 - root - INFO - lr: 1.7521e-04 gnorm: 0.86 [10:35:43<11:25:31] +[titan] 2025-07-10 00:10:47,001 - root - INFO - step: 48120 loss: 16.3111 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8035 +[titan] 2025-07-10 00:10:47,001 - root - INFO - lr: 1.7518e-04 gnorm: 0.82 [10:35:47<11:25:27] +[titan] 2025-07-10 00:10:51,020 - root - INFO - step: 48125 loss: 16.4171 memory: 44.58GiB(31.99%) tps: 81,544 tflops: 281.42 mfu: 28.46% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.8866 +[titan] 2025-07-10 00:10:51,020 - root - INFO - lr: 1.7516e-04 gnorm: 0.84 [10:35:51<11:25:24] +[titan] 2025-07-10 00:10:53,524 - root - INFO - Dumping profiler traces at step 48128 +[titan] 2025-07-10 00:10:53,557 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:10:55,134 - root - INFO - step: 48130 loss: 16.6192 memory: 44.58GiB(31.99%) tps: 79,653 tflops: 274.89 mfu: 27.80% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0659 +[titan] 2025-07-10 00:10:55,135 - root - INFO - lr: 1.7514e-04 gnorm: 0.81 [10:35:55<11:25:20] +[titan] 2025-07-10 00:10:59,121 - root - INFO - step: 48135 loss: 16.6042 memory: 44.58GiB(31.99%) tps: 82,195 tflops: 283.67 mfu: 28.68% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0400 +[titan] 2025-07-10 00:10:59,122 - root - INFO - lr: 1.7512e-04 gnorm: 0.79 [10:35:59<11:25:16] +[titan] 2025-07-10 00:11:03,064 - root - INFO - step: 48140 loss: 16.7099 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5766 global_avg_mtp_loss: 14.1333 +[titan] 2025-07-10 00:11:03,064 - root - INFO - lr: 1.7510e-04 gnorm: 0.81 [10:36:03<11:25:12] +[titan] 2025-07-10 00:11:07,036 - root - INFO - step: 48145 loss: 16.2170 memory: 44.58GiB(31.99%) tps: 82,496 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.7193 +[titan] 2025-07-10 00:11:07,037 - root - INFO - lr: 1.7508e-04 gnorm: 0.85 [10:36:07<11:25:08] +[titan] 2025-07-10 00:11:10,146 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:11:10,934 - root - INFO - step: 48150 loss: 16.6331 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 14.0747 +[titan] 2025-07-10 00:11:10,934 - root - INFO - lr: 1.7506e-04 gnorm: 0.82 [10:36:11<11:25:04] +[titan] 2025-07-10 00:11:14,879 - root - INFO - step: 48155 loss: 16.6556 memory: 44.58GiB(31.99%) tps: 83,076 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0863 +[titan] 2025-07-10 00:11:14,879 - root - INFO - lr: 1.7503e-04 gnorm: 0.88 [10:36:15<11:25:00] +[titan] 2025-07-10 00:11:18,796 - root - INFO - step: 48160 loss: 16.7278 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5786 global_avg_mtp_loss: 14.1493 +[titan] 2025-07-10 00:11:18,797 - root - INFO - lr: 1.7501e-04 gnorm: 0.84 [10:36:19<11:24:56] +[titan] 2025-07-10 00:11:22,737 - root - INFO - step: 48165 loss: 16.7938 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5983 global_avg_mtp_loss: 14.1955 +[titan] 2025-07-10 00:11:22,737 - root - INFO - lr: 1.7499e-04 gnorm: 0.86 [10:36:22<11:24:52] +[titan] 2025-07-10 00:11:26,669 - root - INFO - step: 48170 loss: 16.6290 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.0470 +[titan] 2025-07-10 00:11:26,670 - root - INFO - lr: 1.7497e-04 gnorm: 0.83 [10:36:26<11:24:48] +[titan] 2025-07-10 00:11:30,612 - root - INFO - step: 48175 loss: 16.3911 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.8724 +[titan] 2025-07-10 00:11:30,613 - root - INFO - lr: 1.7495e-04 gnorm: 0.79 [10:36:30<11:24:44] +[titan] 2025-07-10 00:11:34,547 - root - INFO - step: 48180 loss: 16.7367 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5969 global_avg_mtp_loss: 14.1398 +[titan] 2025-07-10 00:11:34,548 - root - INFO - lr: 1.7493e-04 gnorm: 0.73 [10:36:34<11:24:40] +[titan] 2025-07-10 00:11:38,494 - root - INFO - step: 48185 loss: 16.6221 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0581 +[titan] 2025-07-10 00:11:38,494 - root - INFO - lr: 1.7491e-04 gnorm: 0.78 [10:36:38<11:24:36] +[titan] 2025-07-10 00:11:42,436 - root - INFO - step: 48190 loss: 16.2970 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 00:11:42,437 - root - INFO - lr: 1.7488e-04 gnorm: 0.86 [10:36:42<11:24:32] +[titan] 2025-07-10 00:11:46,359 - root - INFO - step: 48195 loss: 16.5657 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0104 +[titan] 2025-07-10 00:11:46,359 - root - INFO - lr: 1.7486e-04 gnorm: 0.80 [10:36:46<11:24:28] +[titan] 2025-07-10 00:11:49,474 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:11:50,265 - root - INFO - step: 48200 loss: 16.5755 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0267 +[titan] 2025-07-10 00:11:50,265 - root - INFO - lr: 1.7484e-04 gnorm: 0.75 [10:36:50<11:24:24] +[titan] 2025-07-10 00:11:54,177 - root - INFO - step: 48205 loss: 16.6224 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.0493 +[titan] 2025-07-10 00:11:54,178 - root - INFO - lr: 1.7482e-04 gnorm: 0.86 [10:36:54<11:24:20] +[titan] 2025-07-10 00:11:58,077 - root - INFO - step: 48210 loss: 16.5245 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 13.9825 +[titan] 2025-07-10 00:11:58,077 - root - INFO - lr: 1.7480e-04 gnorm: 0.85 [10:36:58<11:24:16] +[titan] 2025-07-10 00:12:01,984 - root - INFO - step: 48215 loss: 16.7371 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1576 +[titan] 2025-07-10 00:12:01,984 - root - INFO - lr: 1.7478e-04 gnorm: 0.80 [10:37:02<11:24:12] +[titan] 2025-07-10 00:12:05,895 - root - INFO - step: 48220 loss: 16.3730 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-10 00:12:05,896 - root - INFO - lr: 1.7476e-04 gnorm: 0.79 [10:37:06<11:24:08] +[titan] 2025-07-10 00:12:09,811 - root - INFO - step: 48225 loss: 16.8668 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.6048 global_avg_mtp_loss: 14.2620 +[titan] 2025-07-10 00:12:09,811 - root - INFO - lr: 1.7474e-04 gnorm: 0.78 [10:37:10<11:24:04] +[titan] 2025-07-10 00:12:13,753 - root - INFO - step: 48230 loss: 16.5970 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0423 +[titan] 2025-07-10 00:12:13,753 - root - INFO - lr: 1.7471e-04 gnorm: 0.83 [10:37:13<11:24:00] +[titan] 2025-07-10 00:12:17,683 - root - INFO - step: 48235 loss: 16.6715 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.1101 +[titan] 2025-07-10 00:12:17,683 - root - INFO - lr: 1.7469e-04 gnorm: 0.77 [10:37:17<11:23:56] +[titan] 2025-07-10 00:12:21,593 - root - INFO - step: 48240 loss: 16.5785 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 14.0227 +[titan] 2025-07-10 00:12:21,593 - root - INFO - lr: 1.7467e-04 gnorm: 0.79 [10:37:21<11:23:52] +[titan] 2025-07-10 00:12:25,529 - root - INFO - step: 48245 loss: 16.6577 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-10 00:12:25,529 - root - INFO - lr: 1.7465e-04 gnorm: 0.80 [10:37:25<11:23:48] +[titan] 2025-07-10 00:12:28,645 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:12:29,433 - root - INFO - step: 48250 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.8935 +[titan] 2025-07-10 00:12:29,433 - root - INFO - lr: 1.7463e-04 gnorm: 0.78 [10:37:29<11:23:44] +[titan] 2025-07-10 00:12:33,339 - root - INFO - step: 48255 loss: 16.2485 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 00:12:33,339 - root - INFO - lr: 1.7461e-04 gnorm: 0.83 [10:37:33<11:23:40] +[titan] 2025-07-10 00:12:37,250 - root - INFO - step: 48260 loss: 16.6581 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0859 +[titan] 2025-07-10 00:12:37,250 - root - INFO - lr: 1.7459e-04 gnorm: 0.77 [10:37:37<11:23:36] +[titan] 2025-07-10 00:12:41,204 - root - INFO - step: 48265 loss: 16.5749 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5607 global_avg_mtp_loss: 14.0142 +[titan] 2025-07-10 00:12:41,204 - root - INFO - lr: 1.7456e-04 gnorm: 0.82 [10:37:41<11:23:32] +[titan] 2025-07-10 00:12:45,131 - root - INFO - step: 48270 loss: 16.4942 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 13.9429 +[titan] 2025-07-10 00:12:45,132 - root - INFO - lr: 1.7454e-04 gnorm: 0.77 [10:37:45<11:23:28] +[titan] 2025-07-10 00:12:49,085 - root - INFO - step: 48275 loss: 16.7062 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1339 +[titan] 2025-07-10 00:12:49,085 - root - INFO - lr: 1.7452e-04 gnorm: 0.76 [10:37:49<11:23:24] +[titan] 2025-07-10 00:12:53,022 - root - INFO - step: 48280 loss: 16.6018 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0410 +[titan] 2025-07-10 00:12:53,023 - root - INFO - lr: 1.7450e-04 gnorm: 0.81 [10:37:53<11:23:20] +[titan] 2025-07-10 00:12:56,936 - root - INFO - step: 48285 loss: 16.7941 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5985 global_avg_mtp_loss: 14.1955 +[titan] 2025-07-10 00:12:56,936 - root - INFO - lr: 1.7448e-04 gnorm: 0.84 [10:37:57<11:23:16] +[titan] 2025-07-10 00:13:00,860 - root - INFO - step: 48290 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-10 00:13:00,860 - root - INFO - lr: 1.7446e-04 gnorm: 0.86 [10:38:01<11:23:12] +[titan] 2025-07-10 00:13:04,771 - root - INFO - step: 48295 loss: 16.6220 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0523 +[titan] 2025-07-10 00:13:04,771 - root - INFO - lr: 1.7444e-04 gnorm: 0.81 [10:38:04<11:23:08] +[titan] 2025-07-10 00:13:07,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:13:08,703 - root - INFO - step: 48300 loss: 16.8299 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5932 global_avg_mtp_loss: 14.2367 +[titan] 2025-07-10 00:13:08,703 - root - INFO - lr: 1.7441e-04 gnorm: 0.83 [10:38:08<11:23:04] +[titan] 2025-07-10 00:13:12,626 - root - INFO - step: 48305 loss: 16.6932 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1083 +[titan] 2025-07-10 00:13:12,627 - root - INFO - lr: 1.7439e-04 gnorm: 0.80 [10:38:12<11:23:00] +[titan] 2025-07-10 00:13:16,590 - root - INFO - step: 48310 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 00:13:16,590 - root - INFO - lr: 1.7437e-04 gnorm: 0.82 [10:38:16<11:22:56] +[titan] 2025-07-10 00:13:20,493 - root - INFO - step: 48315 loss: 16.7907 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.1917 +[titan] 2025-07-10 00:13:20,493 - root - INFO - lr: 1.7435e-04 gnorm: 0.84 [10:38:20<11:22:52] +[titan] 2025-07-10 00:13:24,416 - root - INFO - step: 48320 loss: 16.3355 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8335 +[titan] 2025-07-10 00:13:24,416 - root - INFO - lr: 1.7433e-04 gnorm: 0.79 [10:38:24<11:22:48] +[titan] 2025-07-10 00:13:28,341 - root - INFO - step: 48325 loss: 16.3235 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8103 +[titan] 2025-07-10 00:13:28,342 - root - INFO - lr: 1.7431e-04 gnorm: 0.79 [10:38:28<11:22:44] +[titan] 2025-07-10 00:13:32,238 - root - INFO - step: 48330 loss: 16.7939 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5961 global_avg_mtp_loss: 14.1978 +[titan] 2025-07-10 00:13:32,238 - root - INFO - lr: 1.7429e-04 gnorm: 0.78 [10:38:32<11:22:40] +[titan] 2025-07-10 00:13:36,153 - root - INFO - step: 48335 loss: 16.2261 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7300 +[titan] 2025-07-10 00:13:36,153 - root - INFO - lr: 1.7427e-04 gnorm: 0.83 [10:38:36<11:22:36] +[titan] 2025-07-10 00:13:40,063 - root - INFO - step: 48340 loss: 16.5867 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0331 +[titan] 2025-07-10 00:13:40,064 - root - INFO - lr: 1.7424e-04 gnorm: 0.81 [10:38:40<11:22:32] +[titan] 2025-07-10 00:13:44,034 - root - INFO - step: 48345 loss: 16.5690 memory: 44.58GiB(31.99%) tps: 82,527 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 14.0191 +[titan] 2025-07-10 00:13:44,035 - root - INFO - lr: 1.7422e-04 gnorm: 0.80 [10:38:44<11:22:28] +[titan] 2025-07-10 00:13:47,152 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:13:47,936 - root - INFO - step: 48350 loss: 16.4910 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9457 +[titan] 2025-07-10 00:13:47,936 - root - INFO - lr: 1.7420e-04 gnorm: 0.80 [10:38:48<11:22:24] +[titan] 2025-07-10 00:13:51,859 - root - INFO - step: 48355 loss: 16.4673 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 00:13:51,859 - root - INFO - lr: 1.7418e-04 gnorm: 0.78 [10:38:52<11:22:20] +[titan] 2025-07-10 00:13:55,785 - root - INFO - step: 48360 loss: 16.4098 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8826 +[titan] 2025-07-10 00:13:55,786 - root - INFO - lr: 1.7416e-04 gnorm: 0.84 [10:38:55<11:22:16] +[titan] 2025-07-10 00:13:59,697 - root - INFO - step: 48365 loss: 16.3177 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.7974 +[titan] 2025-07-10 00:13:59,697 - root - INFO - lr: 1.7414e-04 gnorm: 0.79 [10:38:59<11:22:12] +[titan] 2025-07-10 00:14:03,646 - root - INFO - step: 48370 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.8988 +[titan] 2025-07-10 00:14:03,647 - root - INFO - lr: 1.7412e-04 gnorm: 0.79 [10:39:03<11:22:08] +[titan] 2025-07-10 00:14:07,553 - root - INFO - step: 48375 loss: 16.5641 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 13.9958 +[titan] 2025-07-10 00:14:07,553 - root - INFO - lr: 1.7409e-04 gnorm: 0.82 [10:39:07<11:22:04] +[titan] 2025-07-10 00:14:11,460 - root - INFO - step: 48380 loss: 16.6131 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 14.0476 +[titan] 2025-07-10 00:14:11,461 - root - INFO - lr: 1.7407e-04 gnorm: 0.82 [10:39:11<11:22:00] +[titan] 2025-07-10 00:14:15,402 - root - INFO - step: 48385 loss: 16.3843 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.8628 +[titan] 2025-07-10 00:14:15,402 - root - INFO - lr: 1.7405e-04 gnorm: 0.78 [10:39:15<11:21:56] +[titan] 2025-07-10 00:14:19,305 - root - INFO - step: 48390 loss: 16.1672 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.6953 +[titan] 2025-07-10 00:14:19,305 - root - INFO - lr: 1.7403e-04 gnorm: 0.80 [10:39:19<11:21:52] +[titan] 2025-07-10 00:14:23,217 - root - INFO - step: 48395 loss: 16.3868 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8676 +[titan] 2025-07-10 00:14:23,217 - root - INFO - lr: 1.7401e-04 gnorm: 0.75 [10:39:23<11:21:48] +[titan] 2025-07-10 00:14:26,332 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:14:27,122 - root - INFO - step: 48400 loss: 16.5964 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0394 +[titan] 2025-07-10 00:14:27,122 - root - INFO - lr: 1.7399e-04 gnorm: 0.79 [10:39:27<11:21:44] +[titan] 2025-07-10 00:14:31,107 - root - INFO - step: 48405 loss: 16.7215 memory: 44.58GiB(31.99%) tps: 82,250 tflops: 283.86 mfu: 28.70% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.1377 +[titan] 2025-07-10 00:14:31,107 - root - INFO - lr: 1.7397e-04 gnorm: 0.82 [10:39:31<11:21:40] +[titan] 2025-07-10 00:14:35,035 - root - INFO - step: 48410 loss: 16.7356 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5781 global_avg_mtp_loss: 14.1574 +[titan] 2025-07-10 00:14:35,036 - root - INFO - lr: 1.7394e-04 gnorm: 0.81 [10:39:35<11:21:36] +[titan] 2025-07-10 00:14:38,950 - root - INFO - step: 48415 loss: 16.7327 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1532 +[titan] 2025-07-10 00:14:38,950 - root - INFO - lr: 1.7392e-04 gnorm: 0.82 [10:39:39<11:21:32] +[titan] 2025-07-10 00:14:42,916 - root - INFO - step: 48420 loss: 16.4623 memory: 44.58GiB(31.99%) tps: 82,625 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9320 +[titan] 2025-07-10 00:14:42,916 - root - INFO - lr: 1.7390e-04 gnorm: 0.83 [10:39:43<11:21:28] +[titan] 2025-07-10 00:14:46,826 - root - INFO - step: 48425 loss: 16.3880 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.8645 +[titan] 2025-07-10 00:14:46,826 - root - INFO - lr: 1.7388e-04 gnorm: 0.87 [10:39:47<11:21:24] +[titan] 2025-07-10 00:14:50,736 - root - INFO - step: 48430 loss: 16.4541 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9271 +[titan] 2025-07-10 00:14:50,736 - root - INFO - lr: 1.7386e-04 gnorm: 0.81 [10:39:50<11:21:20] +[titan] 2025-07-10 00:14:54,657 - root - INFO - step: 48435 loss: 16.6068 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-10 00:14:54,657 - root - INFO - lr: 1.7384e-04 gnorm: 0.78 [10:39:54<11:21:16] +[titan] 2025-07-10 00:14:58,599 - root - INFO - step: 48440 loss: 16.1574 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.6642 +[titan] 2025-07-10 00:14:58,599 - root - INFO - lr: 1.7382e-04 gnorm: 0.93 [10:39:58<11:21:12] +[titan] 2025-07-10 00:15:02,504 - root - INFO - step: 48445 loss: 16.4790 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.9480 +[titan] 2025-07-10 00:15:02,504 - root - INFO - lr: 1.7380e-04 gnorm: 0.81 [10:40:02<11:21:08] +[titan] 2025-07-10 00:15:05,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:15:06,424 - root - INFO - step: 48450 loss: 16.6905 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1038 +[titan] 2025-07-10 00:15:06,424 - root - INFO - lr: 1.7377e-04 gnorm: 0.84 [10:40:06<11:21:04] +[titan] 2025-07-10 00:15:10,394 - root - INFO - step: 48455 loss: 16.5901 memory: 44.58GiB(31.99%) tps: 82,528 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0307 +[titan] 2025-07-10 00:15:10,395 - root - INFO - lr: 1.7375e-04 gnorm: 0.79 [10:40:10<11:21:00] +[titan] 2025-07-10 00:15:14,296 - root - INFO - step: 48460 loss: 16.7041 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.1225 +[titan] 2025-07-10 00:15:14,296 - root - INFO - lr: 1.7373e-04 gnorm: 0.77 [10:40:14<11:20:56] +[titan] 2025-07-10 00:15:18,220 - root - INFO - step: 48465 loss: 16.2139 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7328 +[titan] 2025-07-10 00:15:18,220 - root - INFO - lr: 1.7371e-04 gnorm: 0.83 [10:40:18<11:20:52] +[titan] 2025-07-10 00:15:22,130 - root - INFO - step: 48470 loss: 16.6845 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5862 global_avg_mtp_loss: 14.0982 +[titan] 2025-07-10 00:15:22,130 - root - INFO - lr: 1.7369e-04 gnorm: 0.79 [10:40:22<11:20:47] +[titan] 2025-07-10 00:15:26,127 - root - INFO - step: 48475 loss: 16.3260 memory: 44.58GiB(31.99%) tps: 81,989 tflops: 282.96 mfu: 28.61% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8235 +[titan] 2025-07-10 00:15:26,127 - root - INFO - lr: 1.7367e-04 gnorm: 0.82 [10:40:26<11:20:44] +[titan] 2025-07-10 00:15:30,069 - root - INFO - step: 48480 loss: 16.5294 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 13.9798 +[titan] 2025-07-10 00:15:30,069 - root - INFO - lr: 1.7365e-04 gnorm: 0.79 [10:40:30<11:20:40] +[titan] 2025-07-10 00:15:34,025 - root - INFO - step: 48485 loss: 16.3070 memory: 44.58GiB(31.99%) tps: 82,849 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8080 +[titan] 2025-07-10 00:15:34,025 - root - INFO - lr: 1.7362e-04 gnorm: 0.79 [10:40:34<11:20:36] +[titan] 2025-07-10 00:15:37,919 - root - INFO - step: 48490 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9184 +[titan] 2025-07-10 00:15:37,920 - root - INFO - lr: 1.7360e-04 gnorm: 0.76 [10:40:38<11:20:32] +[titan] 2025-07-10 00:15:41,822 - root - INFO - step: 48495 loss: 16.6542 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0858 +[titan] 2025-07-10 00:15:41,823 - root - INFO - lr: 1.7358e-04 gnorm: 0.84 [10:40:42<11:20:28] +[titan] 2025-07-10 00:15:44,944 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:15:45,743 - root - INFO - step: 48500 loss: 16.3750 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8588 +[titan] 2025-07-10 00:15:45,744 - root - INFO - lr: 1.7356e-04 gnorm: 0.78 [10:40:45<11:20:24] +[titan] 2025-07-10 00:15:49,656 - root - INFO - step: 48505 loss: 16.3430 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8282 +[titan] 2025-07-10 00:15:49,656 - root - INFO - lr: 1.7354e-04 gnorm: 0.79 [10:40:49<11:20:20] +[titan] 2025-07-10 00:15:53,605 - root - INFO - step: 48510 loss: 16.6790 memory: 44.58GiB(31.99%) tps: 82,987 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.0969 +[titan] 2025-07-10 00:15:53,605 - root - INFO - lr: 1.7352e-04 gnorm: 0.83 [10:40:53<11:20:16] +[titan] 2025-07-10 00:15:57,541 - root - INFO - step: 48515 loss: 16.2786 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7909 +[titan] 2025-07-10 00:15:57,542 - root - INFO - lr: 1.7350e-04 gnorm: 0.75 [10:40:57<11:20:12] +[titan] 2025-07-10 00:16:01,473 - root - INFO - step: 48520 loss: 16.6904 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5741 global_avg_mtp_loss: 14.1163 +[titan] 2025-07-10 00:16:01,474 - root - INFO - lr: 1.7347e-04 gnorm: 0.77 [10:41:01<11:20:08] +[titan] 2025-07-10 00:16:05,394 - root - INFO - step: 48525 loss: 16.7197 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1493 +[titan] 2025-07-10 00:16:05,394 - root - INFO - lr: 1.7345e-04 gnorm: 0.85 [10:41:05<11:20:04] +[titan] 2025-07-10 00:16:09,308 - root - INFO - step: 48530 loss: 16.5625 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 14.0184 +[titan] 2025-07-10 00:16:09,308 - root - INFO - lr: 1.7343e-04 gnorm: 0.81 [10:41:09<11:20:00] +[titan] 2025-07-10 00:16:13,298 - root - INFO - step: 48535 loss: 16.5781 memory: 44.58GiB(31.99%) tps: 82,140 tflops: 283.48 mfu: 28.66% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0276 +[titan] 2025-07-10 00:16:13,298 - root - INFO - lr: 1.7341e-04 gnorm: 0.75 [10:41:13<11:19:56] +[titan] 2025-07-10 00:16:17,247 - root - INFO - step: 48540 loss: 16.4510 memory: 44.58GiB(31.99%) tps: 82,972 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9151 +[titan] 2025-07-10 00:16:17,248 - root - INFO - lr: 1.7339e-04 gnorm: 0.78 [10:41:17<11:19:52] +[titan] 2025-07-10 00:16:21,147 - root - INFO - step: 48545 loss: 16.7957 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.1925 +[titan] 2025-07-10 00:16:21,147 - root - INFO - lr: 1.7337e-04 gnorm: 0.82 [10:41:21<11:19:48] +[titan] 2025-07-10 00:16:24,305 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:16:25,103 - root - INFO - step: 48550 loss: 16.5988 memory: 44.58GiB(31.99%) tps: 82,830 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0515 +[titan] 2025-07-10 00:16:25,104 - root - INFO - lr: 1.7335e-04 gnorm: 0.79 [10:41:25<11:19:44] +[titan] 2025-07-10 00:16:29,031 - root - INFO - step: 48555 loss: 16.8154 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.6165 global_avg_mtp_loss: 14.1989 +[titan] 2025-07-10 00:16:29,031 - root - INFO - lr: 1.7332e-04 gnorm: 0.80 [10:41:29<11:19:40] +[titan] 2025-07-10 00:16:32,958 - root - INFO - step: 48560 loss: 16.7243 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.1476 +[titan] 2025-07-10 00:16:32,958 - root - INFO - lr: 1.7330e-04 gnorm: 0.80 [10:41:33<11:19:36] +[titan] 2025-07-10 00:16:36,899 - root - INFO - step: 48565 loss: 16.2916 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.7848 +[titan] 2025-07-10 00:16:36,899 - root - INFO - lr: 1.7328e-04 gnorm: 0.78 [10:41:37<11:19:32] +[titan] 2025-07-10 00:16:40,815 - root - INFO - step: 48570 loss: 16.3833 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.8620 +[titan] 2025-07-10 00:16:40,815 - root - INFO - lr: 1.7326e-04 gnorm: 0.85 [10:41:40<11:19:28] +[titan] 2025-07-10 00:16:44,775 - root - INFO - step: 48575 loss: 16.6394 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0671 +[titan] 2025-07-10 00:16:44,775 - root - INFO - lr: 1.7324e-04 gnorm: 0.81 [10:41:44<11:19:24] +[titan] 2025-07-10 00:16:48,696 - root - INFO - step: 48580 loss: 16.5357 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5470 global_avg_mtp_loss: 13.9887 +[titan] 2025-07-10 00:16:48,696 - root - INFO - lr: 1.7322e-04 gnorm: 0.84 [10:41:48<11:19:20] +[titan] 2025-07-10 00:16:52,609 - root - INFO - step: 48585 loss: 16.2622 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7636 +[titan] 2025-07-10 00:16:52,609 - root - INFO - lr: 1.7320e-04 gnorm: 0.80 [10:41:52<11:19:16] +[titan] 2025-07-10 00:16:56,500 - root - INFO - step: 48590 loss: 16.5779 memory: 44.58GiB(31.99%) tps: 84,233 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0200 +[titan] 2025-07-10 00:16:56,500 - root - INFO - lr: 1.7318e-04 gnorm: 0.82 [10:41:56<11:19:12] +[titan] 2025-07-10 00:17:00,399 - root - INFO - step: 48595 loss: 16.2509 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.7544 +[titan] 2025-07-10 00:17:00,399 - root - INFO - lr: 1.7315e-04 gnorm: 0.79 [10:42:00<11:19:08] +[titan] 2025-07-10 00:17:03,507 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:17:04,308 - root - INFO - step: 48600 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8764 +[titan] 2025-07-10 00:17:04,308 - root - INFO - lr: 1.7313e-04 gnorm: 0.83 [10:42:04<11:19:04] +[titan] 2025-07-10 00:17:08,213 - root - INFO - step: 48605 loss: 16.2992 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.7910 +[titan] 2025-07-10 00:17:08,214 - root - INFO - lr: 1.7311e-04 gnorm: 0.79 [10:42:08<11:18:59] +[titan] 2025-07-10 00:17:12,132 - root - INFO - step: 48610 loss: 16.2336 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.7299 +[titan] 2025-07-10 00:17:12,132 - root - INFO - lr: 1.7309e-04 gnorm: 0.81 [10:42:12<11:18:55] +[titan] 2025-07-10 00:17:16,051 - root - INFO - step: 48615 loss: 16.3878 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5267 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 00:17:16,051 - root - INFO - lr: 1.7307e-04 gnorm: 0.80 [10:42:16<11:18:51] +[titan] 2025-07-10 00:17:19,951 - root - INFO - step: 48620 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9133 +[titan] 2025-07-10 00:17:19,951 - root - INFO - lr: 1.7305e-04 gnorm: 0.80 [10:42:20<11:18:47] +[titan] 2025-07-10 00:17:23,860 - root - INFO - step: 48625 loss: 16.4455 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9218 +[titan] 2025-07-10 00:17:23,860 - root - INFO - lr: 1.7303e-04 gnorm: 0.77 [10:42:24<11:18:43] +[titan] 2025-07-10 00:17:27,762 - root - INFO - step: 48630 loss: 16.3977 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8718 +[titan] 2025-07-10 00:17:27,763 - root - INFO - lr: 1.7300e-04 gnorm: 0.81 [10:42:27<11:18:39] +[titan] 2025-07-10 00:17:31,665 - root - INFO - step: 48635 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.7846 +[titan] 2025-07-10 00:17:31,665 - root - INFO - lr: 1.7298e-04 gnorm: 0.79 [10:42:31<11:18:35] +[titan] 2025-07-10 00:17:35,653 - root - INFO - step: 48640 loss: 16.8013 memory: 44.58GiB(31.99%) tps: 82,173 tflops: 283.59 mfu: 28.67% global_avg_ntp_loss: 2.6014 global_avg_mtp_loss: 14.1999 +[titan] 2025-07-10 00:17:35,653 - root - INFO - lr: 1.7296e-04 gnorm: 0.80 [10:42:35<11:18:31] +[titan] 2025-07-10 00:17:35,808 - root - INFO - Dumping profiler traces at step 48640 +[titan] 2025-07-10 00:17:35,841 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:17:39,749 - root - INFO - step: 48645 loss: 16.2050 memory: 44.58GiB(31.99%) tps: 80,013 tflops: 276.14 mfu: 27.92% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7191 +[titan] 2025-07-10 00:17:39,749 - root - INFO - lr: 1.7294e-04 gnorm: 0.83 [10:42:39<11:18:28] +[titan] 2025-07-10 00:17:42,864 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:17:43,650 - root - INFO - step: 48650 loss: 16.5861 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0211 +[titan] 2025-07-10 00:17:43,650 - root - INFO - lr: 1.7292e-04 gnorm: 0.80 [10:42:43<11:18:24] +[titan] 2025-07-10 00:17:47,569 - root - INFO - step: 48655 loss: 16.7751 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.6042 global_avg_mtp_loss: 14.1709 +[titan] 2025-07-10 00:17:47,570 - root - INFO - lr: 1.7290e-04 gnorm: 0.83 [10:42:47<11:18:20] +[titan] 2025-07-10 00:17:51,495 - root - INFO - step: 48660 loss: 16.5247 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.9803 +[titan] 2025-07-10 00:17:51,495 - root - INFO - lr: 1.7288e-04 gnorm: 0.81 [10:42:51<11:18:16] +[titan] 2025-07-10 00:17:55,434 - root - INFO - step: 48665 loss: 16.5908 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5610 global_avg_mtp_loss: 14.0298 +[titan] 2025-07-10 00:17:55,434 - root - INFO - lr: 1.7285e-04 gnorm: 0.78 [10:42:55<11:18:12] +[titan] 2025-07-10 00:17:59,359 - root - INFO - step: 48670 loss: 16.4882 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9491 +[titan] 2025-07-10 00:17:59,359 - root - INFO - lr: 1.7283e-04 gnorm: 0.82 [10:42:59<11:18:08] +[titan] 2025-07-10 00:18:03,321 - root - INFO - step: 48675 loss: 16.8165 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.6154 global_avg_mtp_loss: 14.2011 +[titan] 2025-07-10 00:18:03,321 - root - INFO - lr: 1.7281e-04 gnorm: 0.83 [10:43:03<11:18:04] +[titan] 2025-07-10 00:18:07,226 - root - INFO - step: 48680 loss: 16.6978 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5847 global_avg_mtp_loss: 14.1131 +[titan] 2025-07-10 00:18:07,226 - root - INFO - lr: 1.7279e-04 gnorm: 0.78 [10:43:07<11:18:00] +[titan] 2025-07-10 00:18:11,139 - root - INFO - step: 48685 loss: 16.4387 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9021 +[titan] 2025-07-10 00:18:11,139 - root - INFO - lr: 1.7277e-04 gnorm: 0.79 [10:43:11<11:17:56] +[titan] 2025-07-10 00:18:15,101 - root - INFO - step: 48690 loss: 16.4756 memory: 44.58GiB(31.99%) tps: 82,699 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9459 +[titan] 2025-07-10 00:18:15,101 - root - INFO - lr: 1.7275e-04 gnorm: 0.80 [10:43:15<11:17:52] +[titan] 2025-07-10 00:18:19,010 - root - INFO - step: 48695 loss: 16.5722 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5829 global_avg_mtp_loss: 13.9892 +[titan] 2025-07-10 00:18:19,010 - root - INFO - lr: 1.7273e-04 gnorm: 0.85 [10:43:19<11:17:48] +[titan] 2025-07-10 00:18:22,135 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:18:22,931 - root - INFO - step: 48700 loss: 16.5012 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.5449 global_avg_mtp_loss: 13.9563 +[titan] 2025-07-10 00:18:22,931 - root - INFO - lr: 1.7270e-04 gnorm: 0.80 [10:43:23<11:17:44] +[titan] 2025-07-10 00:18:26,839 - root - INFO - step: 48705 loss: 16.4800 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9437 +[titan] 2025-07-10 00:18:26,840 - root - INFO - lr: 1.7268e-04 gnorm: 0.87 [10:43:27<11:17:40] +[titan] 2025-07-10 00:18:30,778 - root - INFO - step: 48710 loss: 16.5869 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5631 global_avg_mtp_loss: 14.0238 +[titan] 2025-07-10 00:18:30,779 - root - INFO - lr: 1.7266e-04 gnorm: 0.79 [10:43:30<11:17:36] +[titan] 2025-07-10 00:18:34,690 - root - INFO - step: 48715 loss: 16.5309 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9882 +[titan] 2025-07-10 00:18:34,691 - root - INFO - lr: 1.7264e-04 gnorm: 0.81 [10:43:34<11:17:32] +[titan] 2025-07-10 00:18:38,597 - root - INFO - step: 48720 loss: 16.6401 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5696 global_avg_mtp_loss: 14.0705 +[titan] 2025-07-10 00:18:38,597 - root - INFO - lr: 1.7262e-04 gnorm: 0.82 [10:43:38<11:17:27] +[titan] 2025-07-10 00:18:42,562 - root - INFO - step: 48725 loss: 16.5292 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9909 +[titan] 2025-07-10 00:18:42,562 - root - INFO - lr: 1.7260e-04 gnorm: 0.85 [10:43:42<11:17:24] +[titan] 2025-07-10 00:18:46,497 - root - INFO - step: 48730 loss: 16.4673 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9359 +[titan] 2025-07-10 00:18:46,498 - root - INFO - lr: 1.7258e-04 gnorm: 0.81 [10:43:46<11:17:20] +[titan] 2025-07-10 00:18:50,411 - root - INFO - step: 48735 loss: 16.4464 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.9183 +[titan] 2025-07-10 00:18:50,412 - root - INFO - lr: 1.7256e-04 gnorm: 0.79 [10:43:50<11:17:16] +[titan] 2025-07-10 00:18:54,351 - root - INFO - step: 48740 loss: 16.8515 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.6061 global_avg_mtp_loss: 14.2453 +[titan] 2025-07-10 00:18:54,351 - root - INFO - lr: 1.7253e-04 gnorm: 0.76 [10:43:54<11:17:12] +[titan] 2025-07-10 00:18:58,283 - root - INFO - step: 48745 loss: 16.5550 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 14.0158 +[titan] 2025-07-10 00:18:58,283 - root - INFO - lr: 1.7251e-04 gnorm: 0.81 [10:43:58<11:17:08] +[titan] 2025-07-10 00:19:01,447 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:19:02,247 - root - INFO - step: 48750 loss: 16.3026 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8015 +[titan] 2025-07-10 00:19:02,247 - root - INFO - lr: 1.7249e-04 gnorm: 0.82 [10:44:02<11:17:04] +[titan] 2025-07-10 00:19:06,163 - root - INFO - step: 48755 loss: 16.5499 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 14.0049 +[titan] 2025-07-10 00:19:06,163 - root - INFO - lr: 1.7247e-04 gnorm: 0.82 [10:44:06<11:17:00] +[titan] 2025-07-10 00:19:10,067 - root - INFO - step: 48760 loss: 16.6450 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0774 +[titan] 2025-07-10 00:19:10,068 - root - INFO - lr: 1.7245e-04 gnorm: 0.75 [10:44:10<11:16:56] +[titan] 2025-07-10 00:19:14,007 - root - INFO - step: 48765 loss: 16.5448 memory: 44.58GiB(31.99%) tps: 83,190 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 13.9914 +[titan] 2025-07-10 00:19:14,007 - root - INFO - lr: 1.7243e-04 gnorm: 0.76 [10:44:14<11:16:52] +[titan] 2025-07-10 00:19:17,909 - root - INFO - step: 48770 loss: 16.7201 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5920 global_avg_mtp_loss: 14.1281 +[titan] 2025-07-10 00:19:17,909 - root - INFO - lr: 1.7241e-04 gnorm: 0.87 [10:44:18<11:16:48] +[titan] 2025-07-10 00:19:21,810 - root - INFO - step: 48775 loss: 16.6517 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5618 global_avg_mtp_loss: 14.0900 +[titan] 2025-07-10 00:19:21,811 - root - INFO - lr: 1.7238e-04 gnorm: 0.84 [10:44:21<11:16:43] +[titan] 2025-07-10 00:19:25,708 - root - INFO - step: 48780 loss: 16.7474 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5855 global_avg_mtp_loss: 14.1619 +[titan] 2025-07-10 00:19:25,708 - root - INFO - lr: 1.7236e-04 gnorm: 0.81 [10:44:25<11:16:39] +[titan] 2025-07-10 00:19:29,616 - root - INFO - step: 48785 loss: 16.5722 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0126 +[titan] 2025-07-10 00:19:29,616 - root - INFO - lr: 1.7234e-04 gnorm: 0.81 [10:44:29<11:16:35] +[titan] 2025-07-10 00:19:33,520 - root - INFO - step: 48790 loss: 16.3556 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8446 +[titan] 2025-07-10 00:19:33,521 - root - INFO - lr: 1.7232e-04 gnorm: 0.85 [10:44:33<11:16:31] +[titan] 2025-07-10 00:19:37,447 - root - INFO - step: 48795 loss: 16.6711 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5573 global_avg_mtp_loss: 14.1138 +[titan] 2025-07-10 00:19:37,447 - root - INFO - lr: 1.7230e-04 gnorm: 0.80 [10:44:37<11:16:27] +[titan] 2025-07-10 00:19:40,618 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:19:41,418 - root - INFO - step: 48800 loss: 16.5874 memory: 44.58GiB(31.99%) tps: 82,525 tflops: 284.81 mfu: 28.80% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0321 +[titan] 2025-07-10 00:19:41,419 - root - INFO - lr: 1.7228e-04 gnorm: 0.82 [10:44:41<11:16:23] +[titan] 2025-07-10 00:19:45,328 - root - INFO - step: 48805 loss: 16.5018 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 13.9553 +[titan] 2025-07-10 00:19:45,328 - root - INFO - lr: 1.7226e-04 gnorm: 0.78 [10:44:45<11:16:19] +[titan] 2025-07-10 00:19:49,262 - root - INFO - step: 48810 loss: 16.6590 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 14.0906 +[titan] 2025-07-10 00:19:49,262 - root - INFO - lr: 1.7223e-04 gnorm: 0.77 [10:44:49<11:16:15] +[titan] 2025-07-10 00:19:53,183 - root - INFO - step: 48815 loss: 16.6714 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5706 global_avg_mtp_loss: 14.1008 +[titan] 2025-07-10 00:19:53,183 - root - INFO - lr: 1.7221e-04 gnorm: 0.81 [10:44:53<11:16:11] +[titan] 2025-07-10 00:19:57,129 - root - INFO - step: 48820 loss: 16.5677 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 14.0281 +[titan] 2025-07-10 00:19:57,129 - root - INFO - lr: 1.7219e-04 gnorm: 0.83 [10:44:57<11:16:07] +[titan] 2025-07-10 00:20:01,048 - root - INFO - step: 48825 loss: 16.7242 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5695 global_avg_mtp_loss: 14.1547 +[titan] 2025-07-10 00:20:01,048 - root - INFO - lr: 1.7217e-04 gnorm: 0.79 [10:45:01<11:16:03] +[titan] 2025-07-10 00:20:04,968 - root - INFO - step: 48830 loss: 16.4665 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9324 +[titan] 2025-07-10 00:20:04,968 - root - INFO - lr: 1.7215e-04 gnorm: 0.78 [10:45:05<11:15:59] +[titan] 2025-07-10 00:20:08,880 - root - INFO - step: 48835 loss: 16.3888 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.8635 +[titan] 2025-07-10 00:20:08,880 - root - INFO - lr: 1.7213e-04 gnorm: 0.81 [10:45:09<11:15:55] +[titan] 2025-07-10 00:20:12,799 - root - INFO - step: 48840 loss: 16.3547 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.8387 +[titan] 2025-07-10 00:20:12,799 - root - INFO - lr: 1.7211e-04 gnorm: 0.78 [10:45:12<11:15:51] +[titan] 2025-07-10 00:20:16,713 - root - INFO - step: 48845 loss: 16.6252 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0551 +[titan] 2025-07-10 00:20:16,713 - root - INFO - lr: 1.7208e-04 gnorm: 0.80 [10:45:16<11:15:47] +[titan] 2025-07-10 00:20:19,859 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:20:20,656 - root - INFO - step: 48850 loss: 16.2549 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.7600 +[titan] 2025-07-10 00:20:20,657 - root - INFO - lr: 1.7206e-04 gnorm: 0.82 [10:45:20<11:15:43] +[titan] 2025-07-10 00:20:24,564 - root - INFO - step: 48855 loss: 16.9439 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.6350 global_avg_mtp_loss: 14.3089 +[titan] 2025-07-10 00:20:24,564 - root - INFO - lr: 1.7204e-04 gnorm: 0.83 [10:45:24<11:15:39] +[titan] 2025-07-10 00:20:28,492 - root - INFO - step: 48860 loss: 16.6977 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5690 global_avg_mtp_loss: 14.1288 +[titan] 2025-07-10 00:20:28,493 - root - INFO - lr: 1.7202e-04 gnorm: 0.84 [10:45:28<11:15:35] +[titan] 2025-07-10 00:20:32,426 - root - INFO - step: 48865 loss: 16.3716 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 00:20:32,426 - root - INFO - lr: 1.7200e-04 gnorm: 0.86 [10:45:32<11:15:31] +[titan] 2025-07-10 00:20:36,338 - root - INFO - step: 48870 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 00:20:36,338 - root - INFO - lr: 1.7198e-04 gnorm: 0.84 [10:45:36<11:15:27] +[titan] 2025-07-10 00:20:40,262 - root - INFO - step: 48875 loss: 16.7042 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.1355 +[titan] 2025-07-10 00:20:40,262 - root - INFO - lr: 1.7196e-04 gnorm: 0.84 [10:45:40<11:15:23] +[titan] 2025-07-10 00:20:44,206 - root - INFO - step: 48880 loss: 16.6519 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0874 +[titan] 2025-07-10 00:20:44,206 - root - INFO - lr: 1.7193e-04 gnorm: 0.78 [10:45:44<11:15:19] +[titan] 2025-07-10 00:20:48,164 - root - INFO - step: 48885 loss: 16.6982 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5904 global_avg_mtp_loss: 14.1078 +[titan] 2025-07-10 00:20:48,164 - root - INFO - lr: 1.7191e-04 gnorm: 0.79 [10:45:48<11:15:15] +[titan] 2025-07-10 00:20:52,114 - root - INFO - step: 48890 loss: 16.3832 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.8513 +[titan] 2025-07-10 00:20:52,114 - root - INFO - lr: 1.7189e-04 gnorm: 0.90 [10:45:52<11:15:11] +[titan] 2025-07-10 00:20:56,047 - root - INFO - step: 48895 loss: 16.6136 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-10 00:20:56,047 - root - INFO - lr: 1.7187e-04 gnorm: 0.86 [10:45:56<11:15:07] +[titan] 2025-07-10 00:20:59,178 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:20:59,965 - root - INFO - step: 48900 loss: 16.5856 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.0212 +[titan] 2025-07-10 00:20:59,965 - root - INFO - lr: 1.7185e-04 gnorm: 0.78 [10:46:00<11:15:03] +[titan] 2025-07-10 00:21:03,892 - root - INFO - step: 48905 loss: 16.4042 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.8786 +[titan] 2025-07-10 00:21:03,892 - root - INFO - lr: 1.7183e-04 gnorm: 0.83 [10:46:04<11:14:59] +[titan] 2025-07-10 00:21:07,828 - root - INFO - step: 48910 loss: 16.3518 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.8323 +[titan] 2025-07-10 00:21:07,829 - root - INFO - lr: 1.7181e-04 gnorm: 0.82 [10:46:07<11:14:55] +[titan] 2025-07-10 00:21:11,751 - root - INFO - step: 48915 loss: 16.7186 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.1423 +[titan] 2025-07-10 00:21:11,751 - root - INFO - lr: 1.7179e-04 gnorm: 0.85 [10:46:11<11:14:51] +[titan] 2025-07-10 00:21:15,683 - root - INFO - step: 48920 loss: 16.5997 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 00:21:15,684 - root - INFO - lr: 1.7176e-04 gnorm: 0.78 [10:46:15<11:14:47] +[titan] 2025-07-10 00:21:19,610 - root - INFO - step: 48925 loss: 16.5767 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5531 global_avg_mtp_loss: 14.0237 +[titan] 2025-07-10 00:21:19,610 - root - INFO - lr: 1.7174e-04 gnorm: 0.76 [10:46:19<11:14:43] +[titan] 2025-07-10 00:21:23,552 - root - INFO - step: 48930 loss: 16.5456 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 14.0029 +[titan] 2025-07-10 00:21:23,553 - root - INFO - lr: 1.7172e-04 gnorm: 0.78 [10:46:23<11:14:39] +[titan] 2025-07-10 00:21:27,480 - root - INFO - step: 48935 loss: 16.2982 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.7956 +[titan] 2025-07-10 00:21:27,480 - root - INFO - lr: 1.7170e-04 gnorm: 0.78 [10:46:27<11:14:35] +[titan] 2025-07-10 00:21:31,390 - root - INFO - step: 48940 loss: 16.3711 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 00:21:31,390 - root - INFO - lr: 1.7168e-04 gnorm: 0.77 [10:46:31<11:14:31] +[titan] 2025-07-10 00:21:35,296 - root - INFO - step: 48945 loss: 16.6517 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0874 +[titan] 2025-07-10 00:21:35,296 - root - INFO - lr: 1.7166e-04 gnorm: 0.81 [10:46:35<11:14:27] +[titan] 2025-07-10 00:21:38,440 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:21:39,235 - root - INFO - step: 48950 loss: 16.6731 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.1065 +[titan] 2025-07-10 00:21:39,236 - root - INFO - lr: 1.7164e-04 gnorm: 0.86 [10:46:39<11:14:23] +[titan] 2025-07-10 00:21:43,157 - root - INFO - step: 48955 loss: 16.8180 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.2185 +[titan] 2025-07-10 00:21:43,157 - root - INFO - lr: 1.7161e-04 gnorm: 0.87 [10:46:43<11:14:19] +[titan] 2025-07-10 00:21:47,093 - root - INFO - step: 48960 loss: 16.5135 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9744 +[titan] 2025-07-10 00:21:47,093 - root - INFO - lr: 1.7159e-04 gnorm: 0.72 [10:46:47<11:14:15] +[titan] 2025-07-10 00:21:51,019 - root - INFO - step: 48965 loss: 16.5723 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 14.0222 +[titan] 2025-07-10 00:21:51,019 - root - INFO - lr: 1.7157e-04 gnorm: 0.84 [10:46:51<11:14:11] +[titan] 2025-07-10 00:21:54,941 - root - INFO - step: 48970 loss: 16.6153 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0632 +[titan] 2025-07-10 00:21:54,942 - root - INFO - lr: 1.7155e-04 gnorm: 0.83 [10:46:55<11:14:07] +[titan] 2025-07-10 00:21:58,868 - root - INFO - step: 48975 loss: 16.5588 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 14.0167 +[titan] 2025-07-10 00:21:58,868 - root - INFO - lr: 1.7153e-04 gnorm: 0.82 [10:46:59<11:14:03] +[titan] 2025-07-10 00:22:02,784 - root - INFO - step: 48980 loss: 16.6932 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1170 +[titan] 2025-07-10 00:22:02,784 - root - INFO - lr: 1.7151e-04 gnorm: 0.86 [10:47:02<11:13:59] +[titan] 2025-07-10 00:22:06,700 - root - INFO - step: 48985 loss: 16.7053 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5752 global_avg_mtp_loss: 14.1301 +[titan] 2025-07-10 00:22:06,700 - root - INFO - lr: 1.7149e-04 gnorm: 0.79 [10:47:06<11:13:55] +[titan] 2025-07-10 00:22:10,647 - root - INFO - step: 48990 loss: 16.4994 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9588 +[titan] 2025-07-10 00:22:10,647 - root - INFO - lr: 1.7146e-04 gnorm: 0.77 [10:47:10<11:13:51] +[titan] 2025-07-10 00:22:14,590 - root - INFO - step: 48995 loss: 16.5553 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 14.0103 +[titan] 2025-07-10 00:22:14,590 - root - INFO - lr: 1.7144e-04 gnorm: 0.80 [10:47:14<11:13:47] +[titan] 2025-07-10 00:22:17,731 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:22:18,520 - root - INFO - step: 49000 loss: 16.6196 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.0396 +[titan] 2025-07-10 00:22:18,521 - root - INFO - lr: 1.7142e-04 gnorm: 0.85 [10:47:18<11:13:43] +[titan] 2025-07-10 00:22:22,455 - root - INFO - step: 49005 loss: 16.4580 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9200 +[titan] 2025-07-10 00:22:22,455 - root - INFO - lr: 1.7140e-04 gnorm: 0.78 [10:47:22<11:13:39] +[titan] 2025-07-10 00:22:26,442 - root - INFO - step: 49010 loss: 16.0692 memory: 44.58GiB(31.99%) tps: 82,190 tflops: 283.65 mfu: 28.68% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.5959 +[titan] 2025-07-10 00:22:26,443 - root - INFO - lr: 1.7138e-04 gnorm: 0.85 [10:47:26<11:13:35] +[titan] 2025-07-10 00:22:30,345 - root - INFO - step: 49015 loss: 16.4400 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 00:22:30,345 - root - INFO - lr: 1.7136e-04 gnorm: 0.75 [10:47:30<11:13:31] +[titan] 2025-07-10 00:22:34,313 - root - INFO - step: 49020 loss: 16.2854 memory: 44.58GiB(31.99%) tps: 82,581 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.7853 +[titan] 2025-07-10 00:22:34,313 - root - INFO - lr: 1.7134e-04 gnorm: 0.79 [10:47:34<11:13:27] +[titan] 2025-07-10 00:22:38,226 - root - INFO - step: 49025 loss: 16.6084 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 14.0504 +[titan] 2025-07-10 00:22:38,226 - root - INFO - lr: 1.7131e-04 gnorm: 0.79 [10:47:38<11:13:23] +[titan] 2025-07-10 00:22:42,148 - root - INFO - step: 49030 loss: 16.4967 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 13.9430 +[titan] 2025-07-10 00:22:42,148 - root - INFO - lr: 1.7129e-04 gnorm: 0.78 [10:47:42<11:13:19] +[titan] 2025-07-10 00:22:46,059 - root - INFO - step: 49035 loss: 16.4087 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8896 +[titan] 2025-07-10 00:22:46,059 - root - INFO - lr: 1.7127e-04 gnorm: 0.82 [10:47:46<11:13:15] +[titan] 2025-07-10 00:22:49,949 - root - INFO - step: 49040 loss: 16.3478 memory: 44.58GiB(31.99%) tps: 84,255 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8409 +[titan] 2025-07-10 00:22:49,949 - root - INFO - lr: 1.7125e-04 gnorm: 0.91 [10:47:50<11:13:11] +[titan] 2025-07-10 00:22:53,847 - root - INFO - step: 49045 loss: 16.5237 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 13.9735 +[titan] 2025-07-10 00:22:53,847 - root - INFO - lr: 1.7123e-04 gnorm: 0.77 [10:47:53<11:13:07] +[titan] 2025-07-10 00:22:56,976 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:22:57,778 - root - INFO - step: 49050 loss: 16.4643 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-10 00:22:57,778 - root - INFO - lr: 1.7121e-04 gnorm: 0.79 [10:47:57<11:13:03] +[titan] 2025-07-10 00:23:01,713 - root - INFO - step: 49055 loss: 16.4124 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 00:23:01,713 - root - INFO - lr: 1.7119e-04 gnorm: 0.83 [10:48:01<11:12:59] +[titan] 2025-07-10 00:23:05,652 - root - INFO - step: 49060 loss: 16.6262 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0657 +[titan] 2025-07-10 00:23:05,652 - root - INFO - lr: 1.7116e-04 gnorm: 0.77 [10:48:05<11:12:55] +[titan] 2025-07-10 00:23:09,597 - root - INFO - step: 49065 loss: 16.6504 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.0804 +[titan] 2025-07-10 00:23:09,597 - root - INFO - lr: 1.7114e-04 gnorm: 0.78 [10:48:09<11:12:51] +[titan] 2025-07-10 00:23:13,489 - root - INFO - step: 49070 loss: 16.5446 memory: 44.58GiB(31.99%) tps: 84,198 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 13.9885 +[titan] 2025-07-10 00:23:13,489 - root - INFO - lr: 1.7112e-04 gnorm: 0.93 [10:48:13<11:12:47] +[titan] 2025-07-10 00:23:17,423 - root - INFO - step: 49075 loss: 16.6783 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5699 global_avg_mtp_loss: 14.1084 +[titan] 2025-07-10 00:23:17,423 - root - INFO - lr: 1.7110e-04 gnorm: 0.83 [10:48:17<11:12:43] +[titan] 2025-07-10 00:23:21,367 - root - INFO - step: 49080 loss: 16.5635 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 14.0137 +[titan] 2025-07-10 00:23:21,367 - root - INFO - lr: 1.7108e-04 gnorm: 0.77 [10:48:21<11:12:39] +[titan] 2025-07-10 00:23:25,293 - root - INFO - step: 49085 loss: 16.8092 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 14.2174 +[titan] 2025-07-10 00:23:25,293 - root - INFO - lr: 1.7106e-04 gnorm: 0.80 [10:48:25<11:12:35] +[titan] 2025-07-10 00:23:29,242 - root - INFO - step: 49090 loss: 16.7471 memory: 44.58GiB(31.99%) tps: 82,968 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.1669 +[titan] 2025-07-10 00:23:29,243 - root - INFO - lr: 1.7104e-04 gnorm: 0.79 [10:48:29<11:12:31] +[titan] 2025-07-10 00:23:33,160 - root - INFO - step: 49095 loss: 16.4742 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9498 +[titan] 2025-07-10 00:23:33,160 - root - INFO - lr: 1.7101e-04 gnorm: 0.76 [10:48:33<11:12:27] +[titan] 2025-07-10 00:23:36,272 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:23:37,060 - root - INFO - step: 49100 loss: 16.3874 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8677 +[titan] 2025-07-10 00:23:37,060 - root - INFO - lr: 1.7099e-04 gnorm: 0.75 [10:48:37<11:12:23] +[titan] 2025-07-10 00:23:40,962 - root - INFO - step: 49105 loss: 16.6181 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0516 +[titan] 2025-07-10 00:23:40,962 - root - INFO - lr: 1.7097e-04 gnorm: 0.80 [10:48:41<11:12:19] +[titan] 2025-07-10 00:23:44,930 - root - INFO - step: 49110 loss: 15.9726 memory: 44.58GiB(31.99%) tps: 82,581 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.5207 +[titan] 2025-07-10 00:23:44,930 - root - INFO - lr: 1.7095e-04 gnorm: 0.79 [10:48:45<11:12:15] +[titan] 2025-07-10 00:23:48,824 - root - INFO - step: 49115 loss: 16.5319 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 13.9850 +[titan] 2025-07-10 00:23:48,824 - root - INFO - lr: 1.7093e-04 gnorm: 0.85 [10:48:48<11:12:11] +[titan] 2025-07-10 00:23:52,747 - root - INFO - step: 49120 loss: 16.6884 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.1184 +[titan] 2025-07-10 00:23:52,747 - root - INFO - lr: 1.7091e-04 gnorm: 0.84 [10:48:52<11:12:07] +[titan] 2025-07-10 00:23:56,657 - root - INFO - step: 49125 loss: 16.5539 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-10 00:23:56,657 - root - INFO - lr: 1.7089e-04 gnorm: 0.79 [10:48:56<11:12:03] +[titan] 2025-07-10 00:24:00,574 - root - INFO - step: 49130 loss: 16.4939 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.9610 +[titan] 2025-07-10 00:24:00,574 - root - INFO - lr: 1.7087e-04 gnorm: 0.81 [10:49:00<11:11:59] +[titan] 2025-07-10 00:24:04,497 - root - INFO - step: 49135 loss: 16.2862 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 00:24:04,498 - root - INFO - lr: 1.7084e-04 gnorm: 0.79 [10:49:04<11:11:55] +[titan] 2025-07-10 00:24:08,454 - root - INFO - step: 49140 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 82,836 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.5769 global_avg_mtp_loss: 14.1248 +[titan] 2025-07-10 00:24:08,454 - root - INFO - lr: 1.7082e-04 gnorm: 0.81 [10:49:08<11:11:51] +[titan] 2025-07-10 00:24:12,355 - root - INFO - step: 49145 loss: 16.6093 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0396 +[titan] 2025-07-10 00:24:12,356 - root - INFO - lr: 1.7080e-04 gnorm: 0.84 [10:49:12<11:11:47] +[titan] 2025-07-10 00:24:15,557 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:24:16,353 - root - INFO - step: 49150 loss: 16.6574 memory: 44.58GiB(31.99%) tps: 81,971 tflops: 282.90 mfu: 28.60% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.0795 +[titan] 2025-07-10 00:24:16,354 - root - INFO - lr: 1.7078e-04 gnorm: 0.83 [10:49:16<11:11:43] +[titan] 2025-07-10 00:24:18,083 - root - INFO - Dumping profiler traces at step 49152 +[titan] 2025-07-10 00:24:18,115 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:24:20,479 - root - INFO - step: 49155 loss: 16.2523 memory: 44.58GiB(31.99%) tps: 79,436 tflops: 274.15 mfu: 27.72% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7600 +[titan] 2025-07-10 00:24:20,479 - root - INFO - lr: 1.7076e-04 gnorm: 0.85 [10:49:20<11:11:40] +[titan] 2025-07-10 00:24:24,380 - root - INFO - step: 49160 loss: 16.6273 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5702 global_avg_mtp_loss: 14.0571 +[titan] 2025-07-10 00:24:24,380 - root - INFO - lr: 1.7074e-04 gnorm: 0.80 [10:49:24<11:11:36] +[titan] 2025-07-10 00:24:28,304 - root - INFO - step: 49165 loss: 16.5971 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0437 +[titan] 2025-07-10 00:24:28,305 - root - INFO - lr: 1.7072e-04 gnorm: 0.83 [10:49:28<11:11:32] +[titan] 2025-07-10 00:24:32,236 - root - INFO - step: 49170 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9009 +[titan] 2025-07-10 00:24:32,236 - root - INFO - lr: 1.7069e-04 gnorm: 0.78 [10:49:32<11:11:28] +[titan] 2025-07-10 00:24:36,171 - root - INFO - step: 49175 loss: 16.6479 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0814 +[titan] 2025-07-10 00:24:36,171 - root - INFO - lr: 1.7067e-04 gnorm: 0.77 [10:49:36<11:11:24] +[titan] 2025-07-10 00:24:40,085 - root - INFO - step: 49180 loss: 16.3527 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8422 +[titan] 2025-07-10 00:24:40,085 - root - INFO - lr: 1.7065e-04 gnorm: 0.78 [10:49:40<11:11:20] +[titan] 2025-07-10 00:24:44,134 - root - INFO - step: 49185 loss: 16.6560 memory: 44.58GiB(31.99%) tps: 80,941 tflops: 279.34 mfu: 28.24% global_avg_ntp_loss: 2.5802 global_avg_mtp_loss: 14.0758 +[titan] 2025-07-10 00:24:44,134 - root - INFO - lr: 1.7063e-04 gnorm: 0.83 [10:49:44<11:11:16] +[titan] 2025-07-10 00:24:48,095 - root - INFO - step: 49190 loss: 16.6736 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.1092 +[titan] 2025-07-10 00:24:48,096 - root - INFO - lr: 1.7061e-04 gnorm: 0.88 [10:49:48<11:11:12] +[titan] 2025-07-10 00:24:52,000 - root - INFO - step: 49195 loss: 16.7070 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.1311 +[titan] 2025-07-10 00:24:52,000 - root - INFO - lr: 1.7059e-04 gnorm: 0.87 [10:49:52<11:11:08] +[titan] 2025-07-10 00:24:55,137 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:24:55,928 - root - INFO - step: 49200 loss: 16.5535 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 14.0099 +[titan] 2025-07-10 00:24:55,928 - root - INFO - lr: 1.7057e-04 gnorm: 0.90 [10:49:56<11:11:04] +[titan] 2025-07-10 00:24:59,834 - root - INFO - step: 49205 loss: 16.6869 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1165 +[titan] 2025-07-10 00:24:59,834 - root - INFO - lr: 1.7054e-04 gnorm: 0.80 [10:49:59<11:11:00] +[titan] 2025-07-10 00:25:03,735 - root - INFO - step: 49210 loss: 16.4427 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-10 00:25:03,735 - root - INFO - lr: 1.7052e-04 gnorm: 1.13 [10:50:03<11:10:56] +[titan] 2025-07-10 00:25:07,641 - root - INFO - step: 49215 loss: 16.5858 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 14.0383 +[titan] 2025-07-10 00:25:07,642 - root - INFO - lr: 1.7050e-04 gnorm: 0.83 [10:50:07<11:10:52] +[titan] 2025-07-10 00:25:11,565 - root - INFO - step: 49220 loss: 16.6135 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5818 global_avg_mtp_loss: 14.0317 +[titan] 2025-07-10 00:25:11,566 - root - INFO - lr: 1.7048e-04 gnorm: 0.81 [10:50:11<11:10:48] +[titan] 2025-07-10 00:25:15,481 - root - INFO - step: 49225 loss: 16.3989 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5159 global_avg_mtp_loss: 13.8830 +[titan] 2025-07-10 00:25:15,482 - root - INFO - lr: 1.7046e-04 gnorm: 0.82 [10:50:15<11:10:44] +[titan] 2025-07-10 00:25:19,443 - root - INFO - step: 49230 loss: 16.3300 memory: 44.58GiB(31.99%) tps: 82,721 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.8099 +[titan] 2025-07-10 00:25:19,443 - root - INFO - lr: 1.7044e-04 gnorm: 0.81 [10:50:19<11:10:40] +[titan] 2025-07-10 00:25:23,342 - root - INFO - step: 49235 loss: 16.7347 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1527 +[titan] 2025-07-10 00:25:23,343 - root - INFO - lr: 1.7042e-04 gnorm: 0.78 [10:50:23<11:10:36] +[titan] 2025-07-10 00:25:27,243 - root - INFO - step: 49240 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7831 +[titan] 2025-07-10 00:25:27,244 - root - INFO - lr: 1.7039e-04 gnorm: 0.81 [10:50:27<11:10:32] +[titan] 2025-07-10 00:25:31,138 - root - INFO - step: 49245 loss: 16.7043 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5763 global_avg_mtp_loss: 14.1280 +[titan] 2025-07-10 00:25:31,138 - root - INFO - lr: 1.7037e-04 gnorm: 0.80 [10:50:31<11:10:28] +[titan] 2025-07-10 00:25:34,265 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:25:35,055 - root - INFO - step: 49250 loss: 16.7347 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5903 global_avg_mtp_loss: 14.1444 +[titan] 2025-07-10 00:25:35,056 - root - INFO - lr: 1.7035e-04 gnorm: 0.80 [10:50:35<11:10:24] +[titan] 2025-07-10 00:25:38,967 - root - INFO - step: 49255 loss: 16.3563 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8430 +[titan] 2025-07-10 00:25:38,968 - root - INFO - lr: 1.7033e-04 gnorm: 0.79 [10:50:39<11:10:20] +[titan] 2025-07-10 00:25:42,900 - root - INFO - step: 49260 loss: 16.3978 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.8815 +[titan] 2025-07-10 00:25:42,900 - root - INFO - lr: 1.7031e-04 gnorm: 0.81 [10:50:43<11:10:16] +[titan] 2025-07-10 00:25:46,806 - root - INFO - step: 49265 loss: 16.4834 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9427 +[titan] 2025-07-10 00:25:46,806 - root - INFO - lr: 1.7029e-04 gnorm: 0.77 [10:50:46<11:10:12] +[titan] 2025-07-10 00:25:50,713 - root - INFO - step: 49270 loss: 16.4500 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.9362 +[titan] 2025-07-10 00:25:50,713 - root - INFO - lr: 1.7027e-04 gnorm: 0.78 [10:50:50<11:10:08] +[titan] 2025-07-10 00:25:54,614 - root - INFO - step: 49275 loss: 16.5883 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 14.0316 +[titan] 2025-07-10 00:25:54,614 - root - INFO - lr: 1.7024e-04 gnorm: 0.79 [10:50:54<11:10:03] +[titan] 2025-07-10 00:25:58,567 - root - INFO - step: 49280 loss: 16.3387 memory: 44.58GiB(31.99%) tps: 82,915 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8303 +[titan] 2025-07-10 00:25:58,567 - root - INFO - lr: 1.7022e-04 gnorm: 0.84 [10:50:58<11:10:00] +[titan] 2025-07-10 00:26:02,485 - root - INFO - step: 49285 loss: 16.5604 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0094 +[titan] 2025-07-10 00:26:02,485 - root - INFO - lr: 1.7020e-04 gnorm: 0.89 [10:51:02<11:09:55] +[titan] 2025-07-10 00:26:06,384 - root - INFO - step: 49290 loss: 16.6215 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5693 global_avg_mtp_loss: 14.0522 +[titan] 2025-07-10 00:26:06,384 - root - INFO - lr: 1.7018e-04 gnorm: 0.79 [10:51:06<11:09:51] +[titan] 2025-07-10 00:26:10,320 - root - INFO - step: 49295 loss: 16.5113 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9720 +[titan] 2025-07-10 00:26:10,321 - root - INFO - lr: 1.7016e-04 gnorm: 0.78 [10:51:10<11:09:47] +[titan] 2025-07-10 00:26:13,453 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:26:14,262 - root - INFO - step: 49300 loss: 16.4948 memory: 44.58GiB(31.99%) tps: 83,134 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.9678 +[titan] 2025-07-10 00:26:14,263 - root - INFO - lr: 1.7014e-04 gnorm: 0.80 [10:51:14<11:09:43] +[titan] 2025-07-10 00:26:18,168 - root - INFO - step: 49305 loss: 16.3490 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.8225 +[titan] 2025-07-10 00:26:18,168 - root - INFO - lr: 1.7012e-04 gnorm: 0.80 [10:51:18<11:09:39] +[titan] 2025-07-10 00:26:22,068 - root - INFO - step: 49310 loss: 16.6085 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0506 +[titan] 2025-07-10 00:26:22,068 - root - INFO - lr: 1.7009e-04 gnorm: 0.86 [10:51:22<11:09:35] +[titan] 2025-07-10 00:26:26,010 - root - INFO - step: 49315 loss: 16.4442 memory: 44.58GiB(31.99%) tps: 83,129 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9203 +[titan] 2025-07-10 00:26:26,011 - root - INFO - lr: 1.7007e-04 gnorm: 0.78 [10:51:26<11:09:31] +[titan] 2025-07-10 00:26:29,912 - root - INFO - step: 49320 loss: 16.6956 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 14.1165 +[titan] 2025-07-10 00:26:29,912 - root - INFO - lr: 1.7005e-04 gnorm: 0.82 [10:51:30<11:09:27] +[titan] 2025-07-10 00:26:33,806 - root - INFO - step: 49325 loss: 16.6829 memory: 44.58GiB(31.99%) tps: 84,162 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-10 00:26:33,806 - root - INFO - lr: 1.7003e-04 gnorm: 0.93 [10:51:33<11:09:23] +[titan] 2025-07-10 00:26:37,699 - root - INFO - step: 49330 loss: 16.5853 memory: 44.58GiB(31.99%) tps: 84,182 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0248 +[titan] 2025-07-10 00:26:37,699 - root - INFO - lr: 1.7001e-04 gnorm: 0.81 [10:51:37<11:09:19] +[titan] 2025-07-10 00:26:41,604 - root - INFO - step: 49335 loss: 16.5263 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 13.9746 +[titan] 2025-07-10 00:26:41,605 - root - INFO - lr: 1.6999e-04 gnorm: 0.80 [10:51:41<11:09:15] +[titan] 2025-07-10 00:26:45,559 - root - INFO - step: 49340 loss: 16.6928 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5656 global_avg_mtp_loss: 14.1272 +[titan] 2025-07-10 00:26:45,559 - root - INFO - lr: 1.6997e-04 gnorm: 0.80 [10:51:45<11:09:11] +[titan] 2025-07-10 00:26:49,505 - root - INFO - step: 49345 loss: 16.4701 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 13.9337 +[titan] 2025-07-10 00:26:49,505 - root - INFO - lr: 1.6994e-04 gnorm: 0.80 [10:51:49<11:09:07] +[titan] 2025-07-10 00:26:52,646 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:26:53,443 - root - INFO - step: 49350 loss: 16.7039 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5831 global_avg_mtp_loss: 14.1208 +[titan] 2025-07-10 00:26:53,443 - root - INFO - lr: 1.6992e-04 gnorm: 0.85 [10:51:53<11:09:03] +[titan] 2025-07-10 00:26:57,365 - root - INFO - step: 49355 loss: 16.6370 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.0669 +[titan] 2025-07-10 00:26:57,365 - root - INFO - lr: 1.6990e-04 gnorm: 0.81 [10:51:57<11:08:59] +[titan] 2025-07-10 00:27:01,283 - root - INFO - step: 49360 loss: 16.5433 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 14.0030 +[titan] 2025-07-10 00:27:01,283 - root - INFO - lr: 1.6988e-04 gnorm: 0.80 [10:52:01<11:08:55] +[titan] 2025-07-10 00:27:05,224 - root - INFO - step: 49365 loss: 16.4297 memory: 44.58GiB(31.99%) tps: 83,142 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.9006 +[titan] 2025-07-10 00:27:05,225 - root - INFO - lr: 1.6986e-04 gnorm: 0.77 [10:52:05<11:08:51] +[titan] 2025-07-10 00:27:09,189 - root - INFO - step: 49370 loss: 16.6844 memory: 44.58GiB(31.99%) tps: 82,663 tflops: 285.28 mfu: 28.85% global_avg_ntp_loss: 2.5841 global_avg_mtp_loss: 14.1003 +[titan] 2025-07-10 00:27:09,189 - root - INFO - lr: 1.6984e-04 gnorm: 0.80 [10:52:09<11:08:47] +[titan] 2025-07-10 00:27:13,124 - root - INFO - step: 49375 loss: 16.3388 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8258 +[titan] 2025-07-10 00:27:13,124 - root - INFO - lr: 1.6982e-04 gnorm: 0.81 [10:52:13<11:08:43] +[titan] 2025-07-10 00:27:17,029 - root - INFO - step: 49380 loss: 16.7921 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.6026 global_avg_mtp_loss: 14.1896 +[titan] 2025-07-10 00:27:17,029 - root - INFO - lr: 1.6979e-04 gnorm: 0.82 [10:52:17<11:08:39] +[titan] 2025-07-10 00:27:20,930 - root - INFO - step: 49385 loss: 16.6269 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0764 +[titan] 2025-07-10 00:27:20,930 - root - INFO - lr: 1.6977e-04 gnorm: 0.80 [10:52:21<11:08:35] +[titan] 2025-07-10 00:27:24,826 - root - INFO - step: 49390 loss: 16.6381 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0845 +[titan] 2025-07-10 00:27:24,826 - root - INFO - lr: 1.6975e-04 gnorm: 0.76 [10:52:24<11:08:31] +[titan] 2025-07-10 00:27:28,735 - root - INFO - step: 49395 loss: 16.7072 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.1374 +[titan] 2025-07-10 00:27:28,735 - root - INFO - lr: 1.6973e-04 gnorm: 0.79 [10:52:28<11:08:27] +[titan] 2025-07-10 00:27:31,863 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:27:32,666 - root - INFO - step: 49400 loss: 16.7899 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.2025 +[titan] 2025-07-10 00:27:32,667 - root - INFO - lr: 1.6971e-04 gnorm: 0.84 [10:52:32<11:08:23] +[titan] 2025-07-10 00:27:36,600 - root - INFO - step: 49405 loss: 16.6044 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0393 +[titan] 2025-07-10 00:27:36,600 - root - INFO - lr: 1.6969e-04 gnorm: 0.80 [10:52:36<11:08:19] +[titan] 2025-07-10 00:27:40,502 - root - INFO - step: 49410 loss: 16.2318 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7465 +[titan] 2025-07-10 00:27:40,502 - root - INFO - lr: 1.6967e-04 gnorm: 0.83 [10:52:40<11:08:15] +[titan] 2025-07-10 00:27:44,404 - root - INFO - step: 49415 loss: 16.3977 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 00:27:44,404 - root - INFO - lr: 1.6965e-04 gnorm: 0.78 [10:52:44<11:08:11] +[titan] 2025-07-10 00:27:48,317 - root - INFO - step: 49420 loss: 16.3956 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8736 +[titan] 2025-07-10 00:27:48,317 - root - INFO - lr: 1.6962e-04 gnorm: 0.85 [10:52:48<11:08:07] +[titan] 2025-07-10 00:27:52,254 - root - INFO - step: 49425 loss: 16.4232 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.9088 +[titan] 2025-07-10 00:27:52,254 - root - INFO - lr: 1.6960e-04 gnorm: 0.77 [10:52:52<11:08:03] +[titan] 2025-07-10 00:27:56,138 - root - INFO - step: 49430 loss: 16.4197 memory: 44.58GiB(31.99%) tps: 84,366 tflops: 291.16 mfu: 29.44% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.8897 +[titan] 2025-07-10 00:27:56,138 - root - INFO - lr: 1.6958e-04 gnorm: 0.82 [10:52:56<11:07:59] +[titan] 2025-07-10 00:28:00,038 - root - INFO - step: 49435 loss: 16.4263 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5334 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 00:28:00,038 - root - INFO - lr: 1.6956e-04 gnorm: 0.83 [10:53:00<11:07:55] +[titan] 2025-07-10 00:28:03,943 - root - INFO - step: 49440 loss: 16.6479 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0771 +[titan] 2025-07-10 00:28:03,944 - root - INFO - lr: 1.6954e-04 gnorm: 0.82 [10:53:04<11:07:51] +[titan] 2025-07-10 00:28:07,846 - root - INFO - step: 49445 loss: 16.3789 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.8626 +[titan] 2025-07-10 00:28:07,846 - root - INFO - lr: 1.6952e-04 gnorm: 1.58 [10:53:07<11:07:47] +[titan] 2025-07-10 00:28:11,005 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:28:11,802 - root - INFO - step: 49450 loss: 16.3751 memory: 44.58GiB(31.99%) tps: 82,852 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 00:28:11,802 - root - INFO - lr: 1.6950e-04 gnorm: 0.87 [10:53:11<11:07:43] +[titan] 2025-07-10 00:28:15,726 - root - INFO - step: 49455 loss: 16.4766 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9394 +[titan] 2025-07-10 00:28:15,726 - root - INFO - lr: 1.6947e-04 gnorm: 0.81 [10:53:15<11:07:39] +[titan] 2025-07-10 00:28:19,656 - root - INFO - step: 49460 loss: 16.4337 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 00:28:19,657 - root - INFO - lr: 1.6945e-04 gnorm: 0.80 [10:53:19<11:07:35] +[titan] 2025-07-10 00:28:23,624 - root - INFO - step: 49465 loss: 16.5252 memory: 44.58GiB(31.99%) tps: 82,591 tflops: 285.04 mfu: 28.82% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 13.9784 +[titan] 2025-07-10 00:28:23,625 - root - INFO - lr: 1.6943e-04 gnorm: 0.83 [10:53:23<11:07:31] +[titan] 2025-07-10 00:28:27,543 - root - INFO - step: 49470 loss: 16.6843 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.1106 +[titan] 2025-07-10 00:28:27,544 - root - INFO - lr: 1.6941e-04 gnorm: 0.81 [10:53:27<11:07:27] +[titan] 2025-07-10 00:28:31,451 - root - INFO - step: 49475 loss: 16.7516 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5779 global_avg_mtp_loss: 14.1737 +[titan] 2025-07-10 00:28:31,452 - root - INFO - lr: 1.6939e-04 gnorm: 0.84 [10:53:31<11:07:23] +[titan] 2025-07-10 00:28:35,353 - root - INFO - step: 49480 loss: 16.6031 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0441 +[titan] 2025-07-10 00:28:35,354 - root - INFO - lr: 1.6937e-04 gnorm: 0.78 [10:53:35<11:07:19] +[titan] 2025-07-10 00:28:39,251 - root - INFO - step: 49485 loss: 16.4399 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.9171 +[titan] 2025-07-10 00:28:39,252 - root - INFO - lr: 1.6935e-04 gnorm: 0.81 [10:53:39<11:07:15] +[titan] 2025-07-10 00:28:43,179 - root - INFO - step: 49490 loss: 16.7036 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1249 +[titan] 2025-07-10 00:28:43,179 - root - INFO - lr: 1.6932e-04 gnorm: 0.83 [10:53:43<11:07:11] +[titan] 2025-07-10 00:28:47,097 - root - INFO - step: 49495 loss: 16.3031 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.7983 +[titan] 2025-07-10 00:28:47,097 - root - INFO - lr: 1.6930e-04 gnorm: 0.85 [10:53:47<11:07:07] +[titan] 2025-07-10 00:28:50,224 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:28:51,009 - root - INFO - step: 49500 loss: 16.5316 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 13.9859 +[titan] 2025-07-10 00:28:51,009 - root - INFO - lr: 1.6928e-04 gnorm: 0.80 [10:53:51<11:07:03] +[titan] 2025-07-10 00:28:54,910 - root - INFO - step: 49505 loss: 16.8281 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6032 global_avg_mtp_loss: 14.2250 +[titan] 2025-07-10 00:28:54,910 - root - INFO - lr: 1.6926e-04 gnorm: 0.84 [10:53:55<11:06:59] +[titan] 2025-07-10 00:28:58,824 - root - INFO - step: 49510 loss: 16.5429 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 13.9843 +[titan] 2025-07-10 00:28:58,824 - root - INFO - lr: 1.6924e-04 gnorm: 0.82 [10:53:58<11:06:55] +[titan] 2025-07-10 00:29:02,751 - root - INFO - step: 49515 loss: 16.3715 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.8536 +[titan] 2025-07-10 00:29:02,751 - root - INFO - lr: 1.6922e-04 gnorm: 0.78 [10:54:02<11:06:51] +[titan] 2025-07-10 00:29:06,651 - root - INFO - step: 49520 loss: 16.4927 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9601 +[titan] 2025-07-10 00:29:06,652 - root - INFO - lr: 1.6920e-04 gnorm: 0.79 [10:54:06<11:06:47] +[titan] 2025-07-10 00:29:10,599 - root - INFO - step: 49525 loss: 16.4263 memory: 44.58GiB(31.99%) tps: 83,007 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9026 +[titan] 2025-07-10 00:29:10,600 - root - INFO - lr: 1.6917e-04 gnorm: 0.85 [10:54:10<11:06:43] +[titan] 2025-07-10 00:29:14,517 - root - INFO - step: 49530 loss: 16.6187 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 14.0562 +[titan] 2025-07-10 00:29:14,518 - root - INFO - lr: 1.6915e-04 gnorm: 0.81 [10:54:14<11:06:39] +[titan] 2025-07-10 00:29:18,459 - root - INFO - step: 49535 loss: 16.7335 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.1474 +[titan] 2025-07-10 00:29:18,459 - root - INFO - lr: 1.6913e-04 gnorm: 0.74 [10:54:18<11:06:35] +[titan] 2025-07-10 00:29:22,376 - root - INFO - step: 49540 loss: 16.3622 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.8223 +[titan] 2025-07-10 00:29:22,377 - root - INFO - lr: 1.6911e-04 gnorm: 0.78 [10:54:22<11:06:31] +[titan] 2025-07-10 00:29:26,287 - root - INFO - step: 49545 loss: 16.3685 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 00:29:26,287 - root - INFO - lr: 1.6909e-04 gnorm: 0.84 [10:54:26<11:06:27] +[titan] 2025-07-10 00:29:29,425 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:29:30,234 - root - INFO - step: 49550 loss: 16.7656 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5919 global_avg_mtp_loss: 14.1738 +[titan] 2025-07-10 00:29:30,234 - root - INFO - lr: 1.6907e-04 gnorm: 0.79 [10:54:30<11:06:23] +[titan] 2025-07-10 00:29:34,139 - root - INFO - step: 49555 loss: 16.4243 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 13.8699 +[titan] 2025-07-10 00:29:34,139 - root - INFO - lr: 1.6905e-04 gnorm: 0.77 [10:54:34<11:06:19] +[titan] 2025-07-10 00:29:38,048 - root - INFO - step: 49560 loss: 16.8172 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5963 global_avg_mtp_loss: 14.2209 +[titan] 2025-07-10 00:29:38,048 - root - INFO - lr: 1.6902e-04 gnorm: 0.76 [10:54:38<11:06:15] +[titan] 2025-07-10 00:29:41,955 - root - INFO - step: 49565 loss: 16.5530 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 14.0123 +[titan] 2025-07-10 00:29:41,955 - root - INFO - lr: 1.6900e-04 gnorm: 0.79 [10:54:42<11:06:11] +[titan] 2025-07-10 00:29:45,869 - root - INFO - step: 49570 loss: 16.6203 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.0437 +[titan] 2025-07-10 00:29:45,869 - root - INFO - lr: 1.6898e-04 gnorm: 0.78 [10:54:45<11:06:07] +[titan] 2025-07-10 00:29:49,798 - root - INFO - step: 49575 loss: 16.4082 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.8866 +[titan] 2025-07-10 00:29:49,799 - root - INFO - lr: 1.6896e-04 gnorm: 0.80 [10:54:49<11:06:03] +[titan] 2025-07-10 00:29:53,696 - root - INFO - step: 49580 loss: 16.3576 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8398 +[titan] 2025-07-10 00:29:53,696 - root - INFO - lr: 1.6894e-04 gnorm: 0.77 [10:54:53<11:05:59] +[titan] 2025-07-10 00:29:57,611 - root - INFO - step: 49585 loss: 16.3928 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5244 global_avg_mtp_loss: 13.8684 +[titan] 2025-07-10 00:29:57,611 - root - INFO - lr: 1.6892e-04 gnorm: 0.84 [10:54:57<11:05:55] +[titan] 2025-07-10 00:30:01,524 - root - INFO - step: 49590 loss: 16.2003 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7080 +[titan] 2025-07-10 00:30:01,525 - root - INFO - lr: 1.6890e-04 gnorm: 0.80 [10:55:01<11:05:51] +[titan] 2025-07-10 00:30:05,447 - root - INFO - step: 49595 loss: 16.6968 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1188 +[titan] 2025-07-10 00:30:05,447 - root - INFO - lr: 1.6887e-04 gnorm: 0.84 [10:55:05<11:05:47] +[titan] 2025-07-10 00:30:08,605 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:30:09,401 - root - INFO - step: 49600 loss: 16.3310 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5065 global_avg_mtp_loss: 13.8245 +[titan] 2025-07-10 00:30:09,401 - root - INFO - lr: 1.6885e-04 gnorm: 0.78 [10:55:09<11:05:43] +[titan] 2025-07-10 00:30:13,403 - root - INFO - step: 49605 loss: 16.7069 memory: 44.58GiB(31.99%) tps: 81,879 tflops: 282.58 mfu: 28.57% global_avg_ntp_loss: 2.5885 global_avg_mtp_loss: 14.1185 +[titan] 2025-07-10 00:30:13,404 - root - INFO - lr: 1.6883e-04 gnorm: 0.82 [10:55:13<11:05:39] +[titan] 2025-07-10 00:30:17,329 - root - INFO - step: 49610 loss: 16.9035 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.6084 global_avg_mtp_loss: 14.2951 +[titan] 2025-07-10 00:30:17,329 - root - INFO - lr: 1.6881e-04 gnorm: 0.82 [10:55:17<11:05:35] +[titan] 2025-07-10 00:30:21,239 - root - INFO - step: 49615 loss: 16.7487 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.1645 +[titan] 2025-07-10 00:30:21,240 - root - INFO - lr: 1.6879e-04 gnorm: 0.78 [10:55:21<11:05:31] +[titan] 2025-07-10 00:30:25,162 - root - INFO - step: 49620 loss: 16.6154 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 14.0587 +[titan] 2025-07-10 00:30:25,162 - root - INFO - lr: 1.6877e-04 gnorm: 0.85 [10:55:25<11:05:27] +[titan] 2025-07-10 00:30:29,059 - root - INFO - step: 49625 loss: 16.5816 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 14.0331 +[titan] 2025-07-10 00:30:29,060 - root - INFO - lr: 1.6875e-04 gnorm: 0.78 [10:55:29<11:05:23] +[titan] 2025-07-10 00:30:32,954 - root - INFO - step: 49630 loss: 16.6830 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5714 global_avg_mtp_loss: 14.1115 +[titan] 2025-07-10 00:30:32,954 - root - INFO - lr: 1.6872e-04 gnorm: 0.80 [10:55:33<11:05:19] +[titan] 2025-07-10 00:30:36,858 - root - INFO - step: 49635 loss: 16.3286 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8114 +[titan] 2025-07-10 00:30:36,858 - root - INFO - lr: 1.6870e-04 gnorm: 0.85 [10:55:36<11:05:15] +[titan] 2025-07-10 00:30:40,790 - root - INFO - step: 49640 loss: 16.5972 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0376 +[titan] 2025-07-10 00:30:40,790 - root - INFO - lr: 1.6868e-04 gnorm: 0.80 [10:55:40<11:05:11] +[titan] 2025-07-10 00:30:44,700 - root - INFO - step: 49645 loss: 16.4907 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9645 +[titan] 2025-07-10 00:30:44,701 - root - INFO - lr: 1.6866e-04 gnorm: 0.78 [10:55:44<11:05:07] +[titan] 2025-07-10 00:30:47,819 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:30:48,606 - root - INFO - step: 49650 loss: 16.3277 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8311 +[titan] 2025-07-10 00:30:48,607 - root - INFO - lr: 1.6864e-04 gnorm: 0.85 [10:55:48<11:05:03] +[titan] 2025-07-10 00:30:52,505 - root - INFO - step: 49655 loss: 16.6490 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0885 +[titan] 2025-07-10 00:30:52,506 - root - INFO - lr: 1.6862e-04 gnorm: 0.80 [10:55:52<11:04:59] +[titan] 2025-07-10 00:30:56,458 - root - INFO - step: 49660 loss: 16.6664 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.1035 +[titan] 2025-07-10 00:30:56,458 - root - INFO - lr: 1.6860e-04 gnorm: 1.03 [10:55:56<11:04:55] +[titan] 2025-07-10 00:30:59,831 - root - INFO - Dumping profiler traces at step 49664 +[titan] 2025-07-10 00:30:59,863 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:31:00,654 - root - INFO - step: 49665 loss: 16.6871 memory: 44.58GiB(31.99%) tps: 78,094 tflops: 269.51 mfu: 27.25% global_avg_ntp_loss: 2.5680 global_avg_mtp_loss: 14.1191 +[titan] 2025-07-10 00:31:00,654 - root - INFO - lr: 1.6857e-04 gnorm: 0.85 [10:56:00<11:04:51] +[titan] 2025-07-10 00:31:04,590 - root - INFO - step: 49670 loss: 16.6014 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0371 +[titan] 2025-07-10 00:31:04,590 - root - INFO - lr: 1.6855e-04 gnorm: 0.88 [10:56:04<11:04:47] +[titan] 2025-07-10 00:31:08,514 - root - INFO - step: 49675 loss: 16.2754 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7801 +[titan] 2025-07-10 00:31:08,515 - root - INFO - lr: 1.6853e-04 gnorm: 0.84 [10:56:08<11:04:43] +[titan] 2025-07-10 00:31:12,487 - root - INFO - step: 49680 loss: 16.4625 memory: 44.58GiB(31.99%) tps: 82,500 tflops: 284.72 mfu: 28.79% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 13.9230 +[titan] 2025-07-10 00:31:12,487 - root - INFO - lr: 1.6851e-04 gnorm: 0.80 [10:56:12<11:04:39] +[titan] 2025-07-10 00:31:16,401 - root - INFO - step: 49685 loss: 16.2155 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7297 +[titan] 2025-07-10 00:31:16,401 - root - INFO - lr: 1.6849e-04 gnorm: 0.93 [10:56:16<11:04:35] +[titan] 2025-07-10 00:31:20,300 - root - INFO - step: 49690 loss: 16.1544 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.6663 +[titan] 2025-07-10 00:31:20,301 - root - INFO - lr: 1.6847e-04 gnorm: 0.87 [10:56:20<11:04:31] +[titan] 2025-07-10 00:31:24,208 - root - INFO - step: 49695 loss: 16.5774 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.0051 +[titan] 2025-07-10 00:31:24,209 - root - INFO - lr: 1.6845e-04 gnorm: 0.82 [10:56:24<11:04:27] +[titan] 2025-07-10 00:31:27,353 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:31:28,154 - root - INFO - step: 49700 loss: 16.4169 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8942 +[titan] 2025-07-10 00:31:28,154 - root - INFO - lr: 1.6842e-04 gnorm: 0.82 [10:56:28<11:04:23] +[titan] 2025-07-10 00:31:32,052 - root - INFO - step: 49705 loss: 16.5459 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 13.9925 +[titan] 2025-07-10 00:31:32,053 - root - INFO - lr: 1.6840e-04 gnorm: 0.82 [10:56:32<11:04:19] +[titan] 2025-07-10 00:31:35,967 - root - INFO - step: 49710 loss: 16.3383 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8350 +[titan] 2025-07-10 00:31:35,967 - root - INFO - lr: 1.6838e-04 gnorm: 0.88 [10:56:36<11:04:15] +[titan] 2025-07-10 00:31:39,901 - root - INFO - step: 49715 loss: 16.7965 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5965 global_avg_mtp_loss: 14.2000 +[titan] 2025-07-10 00:31:39,902 - root - INFO - lr: 1.6836e-04 gnorm: 0.84 [10:56:39<11:04:11] +[titan] 2025-07-10 00:31:43,837 - root - INFO - step: 49720 loss: 16.2937 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.7956 +[titan] 2025-07-10 00:31:43,838 - root - INFO - lr: 1.6834e-04 gnorm: 0.83 [10:56:43<11:04:07] +[titan] 2025-07-10 00:31:47,758 - root - INFO - step: 49725 loss: 16.6327 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0735 +[titan] 2025-07-10 00:31:47,758 - root - INFO - lr: 1.6832e-04 gnorm: 0.77 [10:56:47<11:04:03] +[titan] 2025-07-10 00:31:51,676 - root - INFO - step: 49730 loss: 16.4949 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9632 +[titan] 2025-07-10 00:31:51,676 - root - INFO - lr: 1.6830e-04 gnorm: 0.81 [10:56:51<11:03:59] +[titan] 2025-07-10 00:31:55,632 - root - INFO - step: 49735 loss: 16.4647 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9332 +[titan] 2025-07-10 00:31:55,632 - root - INFO - lr: 1.6827e-04 gnorm: 0.87 [10:56:55<11:03:55] +[titan] 2025-07-10 00:31:59,555 - root - INFO - step: 49740 loss: 16.5048 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9680 +[titan] 2025-07-10 00:31:59,556 - root - INFO - lr: 1.6825e-04 gnorm: 0.86 [10:56:59<11:03:51] +[titan] 2025-07-10 00:32:03,526 - root - INFO - step: 49745 loss: 16.5296 memory: 44.58GiB(31.99%) tps: 82,537 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9854 +[titan] 2025-07-10 00:32:03,526 - root - INFO - lr: 1.6823e-04 gnorm: 0.84 [10:57:03<11:03:47] +[titan] 2025-07-10 00:32:06,647 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:32:07,438 - root - INFO - step: 49750 loss: 16.6420 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0730 +[titan] 2025-07-10 00:32:07,438 - root - INFO - lr: 1.6821e-04 gnorm: 0.81 [10:57:07<11:03:43] +[titan] 2025-07-10 00:32:11,357 - root - INFO - step: 49755 loss: 16.4494 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 00:32:11,357 - root - INFO - lr: 1.6819e-04 gnorm: 0.76 [10:57:11<11:03:39] +[titan] 2025-07-10 00:32:15,277 - root - INFO - step: 49760 loss: 16.5949 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0399 +[titan] 2025-07-10 00:32:15,277 - root - INFO - lr: 1.6817e-04 gnorm: 0.79 [10:57:15<11:03:35] +[titan] 2025-07-10 00:32:19,192 - root - INFO - step: 49765 loss: 16.5669 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.0141 +[titan] 2025-07-10 00:32:19,192 - root - INFO - lr: 1.6815e-04 gnorm: 0.79 [10:57:19<11:03:31] +[titan] 2025-07-10 00:32:23,090 - root - INFO - step: 49770 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0537 +[titan] 2025-07-10 00:32:23,090 - root - INFO - lr: 1.6812e-04 gnorm: 0.84 [10:57:23<11:03:27] +[titan] 2025-07-10 00:32:27,021 - root - INFO - step: 49775 loss: 16.3482 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8416 +[titan] 2025-07-10 00:32:27,022 - root - INFO - lr: 1.6810e-04 gnorm: 0.78 [10:57:27<11:03:23] +[titan] 2025-07-10 00:32:30,931 - root - INFO - step: 49780 loss: 16.2305 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7355 +[titan] 2025-07-10 00:32:30,931 - root - INFO - lr: 1.6808e-04 gnorm: 0.82 [10:57:31<11:03:19] +[titan] 2025-07-10 00:32:34,830 - root - INFO - step: 49785 loss: 16.3518 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 00:32:34,830 - root - INFO - lr: 1.6806e-04 gnorm: 0.78 [10:57:34<11:03:15] +[titan] 2025-07-10 00:32:38,772 - root - INFO - step: 49790 loss: 16.4502 memory: 44.58GiB(31.99%) tps: 83,129 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9279 +[titan] 2025-07-10 00:32:38,773 - root - INFO - lr: 1.6804e-04 gnorm: 0.82 [10:57:38<11:03:11] +[titan] 2025-07-10 00:32:42,708 - root - INFO - step: 49795 loss: 16.6673 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5631 global_avg_mtp_loss: 14.1042 +[titan] 2025-07-10 00:32:42,708 - root - INFO - lr: 1.6802e-04 gnorm: 0.79 [10:57:42<11:03:07] +[titan] 2025-07-10 00:32:45,834 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:32:46,621 - root - INFO - step: 49800 loss: 16.3907 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8760 +[titan] 2025-07-10 00:32:46,622 - root - INFO - lr: 1.6800e-04 gnorm: 0.76 [10:57:46<11:03:03] +[titan] 2025-07-10 00:32:50,531 - root - INFO - step: 49805 loss: 16.3064 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 00:32:50,532 - root - INFO - lr: 1.6797e-04 gnorm: 0.81 [10:57:50<11:02:59] +[titan] 2025-07-10 00:32:54,441 - root - INFO - step: 49810 loss: 16.5570 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 00:32:54,441 - root - INFO - lr: 1.6795e-04 gnorm: 0.78 [10:57:54<11:02:55] +[titan] 2025-07-10 00:32:58,350 - root - INFO - step: 49815 loss: 16.3688 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8578 +[titan] 2025-07-10 00:32:58,350 - root - INFO - lr: 1.6793e-04 gnorm: 0.80 [10:57:58<11:02:51] +[titan] 2025-07-10 00:33:02,256 - root - INFO - step: 49820 loss: 16.2677 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 00:33:02,256 - root - INFO - lr: 1.6791e-04 gnorm: 0.83 [10:58:02<11:02:47] +[titan] 2025-07-10 00:33:06,182 - root - INFO - step: 49825 loss: 16.3050 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 00:33:06,183 - root - INFO - lr: 1.6789e-04 gnorm: 0.82 [10:58:06<11:02:43] +[titan] 2025-07-10 00:33:10,097 - root - INFO - step: 49830 loss: 16.5117 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9760 +[titan] 2025-07-10 00:33:10,097 - root - INFO - lr: 1.6787e-04 gnorm: 0.82 [10:58:10<11:02:39] +[titan] 2025-07-10 00:33:14,238 - root - INFO - step: 49835 loss: 16.4062 memory: 44.58GiB(31.99%) tps: 79,140 tflops: 273.13 mfu: 27.62% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 00:33:14,238 - root - INFO - lr: 1.6785e-04 gnorm: 0.80 [10:58:14<11:02:35] +[titan] 2025-07-10 00:33:18,160 - root - INFO - step: 49840 loss: 16.6944 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.1046 +[titan] 2025-07-10 00:33:18,161 - root - INFO - lr: 1.6783e-04 gnorm: 0.80 [10:58:18<11:02:31] +[titan] 2025-07-10 00:33:22,113 - root - INFO - step: 49845 loss: 16.3936 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8727 +[titan] 2025-07-10 00:33:22,114 - root - INFO - lr: 1.6780e-04 gnorm: 0.86 [10:58:22<11:02:27] +[titan] 2025-07-10 00:33:25,222 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:33:26,013 - root - INFO - step: 49850 loss: 16.6458 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5688 global_avg_mtp_loss: 14.0770 +[titan] 2025-07-10 00:33:26,013 - root - INFO - lr: 1.6778e-04 gnorm: 0.82 [10:58:26<11:02:23] +[titan] 2025-07-10 00:33:29,984 - root - INFO - step: 49855 loss: 16.7495 memory: 44.58GiB(31.99%) tps: 82,523 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.6309 global_avg_mtp_loss: 14.1186 +[titan] 2025-07-10 00:33:29,984 - root - INFO - lr: 1.6776e-04 gnorm: 1.18 [10:58:30<11:02:19] +[titan] 2025-07-10 00:33:33,903 - root - INFO - step: 49860 loss: 16.5313 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 13.9789 +[titan] 2025-07-10 00:33:33,903 - root - INFO - lr: 1.6774e-04 gnorm: 0.82 [10:58:33<11:02:15] +[titan] 2025-07-10 00:33:37,821 - root - INFO - step: 49865 loss: 16.5766 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-10 00:33:37,821 - root - INFO - lr: 1.6772e-04 gnorm: 0.96 [10:58:37<11:02:11] +[titan] 2025-07-10 00:33:41,752 - root - INFO - step: 49870 loss: 16.7853 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5966 global_avg_mtp_loss: 14.1887 +[titan] 2025-07-10 00:33:41,753 - root - INFO - lr: 1.6770e-04 gnorm: 0.86 [10:58:41<11:02:07] +[titan] 2025-07-10 00:33:45,728 - root - INFO - step: 49875 loss: 16.6201 memory: 44.58GiB(31.99%) tps: 82,436 tflops: 284.50 mfu: 28.77% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.0507 +[titan] 2025-07-10 00:33:45,728 - root - INFO - lr: 1.6768e-04 gnorm: 0.84 [10:58:45<11:02:03] +[titan] 2025-07-10 00:33:49,649 - root - INFO - step: 49880 loss: 16.5968 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 14.0548 +[titan] 2025-07-10 00:33:49,649 - root - INFO - lr: 1.6765e-04 gnorm: 0.83 [10:58:49<11:01:59] +[titan] 2025-07-10 00:33:53,605 - root - INFO - step: 49885 loss: 16.5972 memory: 44.58GiB(31.99%) tps: 82,845 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 00:33:53,605 - root - INFO - lr: 1.6763e-04 gnorm: 0.80 [10:58:53<11:01:55] +[titan] 2025-07-10 00:33:57,524 - root - INFO - step: 49890 loss: 16.2424 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7471 +[titan] 2025-07-10 00:33:57,524 - root - INFO - lr: 1.6761e-04 gnorm: 0.84 [10:58:57<11:01:51] +[titan] 2025-07-10 00:34:01,474 - root - INFO - step: 49895 loss: 16.6284 memory: 44.58GiB(31.99%) tps: 82,969 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.0526 +[titan] 2025-07-10 00:34:01,474 - root - INFO - lr: 1.6759e-04 gnorm: 0.84 [10:59:01<11:01:47] +[titan] 2025-07-10 00:34:04,601 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:34:05,386 - root - INFO - step: 49900 loss: 16.4408 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 00:34:05,386 - root - INFO - lr: 1.6757e-04 gnorm: 0.86 [10:59:05<11:01:43] +[titan] 2025-07-10 00:34:09,292 - root - INFO - step: 49905 loss: 16.4664 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 13.9147 +[titan] 2025-07-10 00:34:09,292 - root - INFO - lr: 1.6755e-04 gnorm: 0.87 [10:59:09<11:01:39] +[titan] 2025-07-10 00:34:13,228 - root - INFO - step: 49910 loss: 16.7910 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5988 global_avg_mtp_loss: 14.1922 +[titan] 2025-07-10 00:34:13,228 - root - INFO - lr: 1.6753e-04 gnorm: 0.85 [10:59:13<11:01:35] +[titan] 2025-07-10 00:34:17,142 - root - INFO - step: 49915 loss: 16.5952 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 14.0307 +[titan] 2025-07-10 00:34:17,142 - root - INFO - lr: 1.6750e-04 gnorm: 0.82 [10:59:17<11:01:31] +[titan] 2025-07-10 00:34:21,063 - root - INFO - step: 49920 loss: 16.4603 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 00:34:21,063 - root - INFO - lr: 1.6748e-04 gnorm: 0.84 [10:59:21<11:01:27] +[titan] 2025-07-10 00:34:24,994 - root - INFO - step: 49925 loss: 16.6196 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 14.0622 +[titan] 2025-07-10 00:34:24,994 - root - INFO - lr: 1.6746e-04 gnorm: 0.85 [10:59:25<11:01:23] +[titan] 2025-07-10 00:34:28,900 - root - INFO - step: 49930 loss: 16.5819 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0361 +[titan] 2025-07-10 00:34:28,901 - root - INFO - lr: 1.6744e-04 gnorm: 0.78 [10:59:28<11:01:19] +[titan] 2025-07-10 00:34:32,803 - root - INFO - step: 49935 loss: 16.3753 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-10 00:34:32,804 - root - INFO - lr: 1.6742e-04 gnorm: 0.82 [10:59:32<11:01:15] +[titan] 2025-07-10 00:34:36,709 - root - INFO - step: 49940 loss: 16.4768 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9508 +[titan] 2025-07-10 00:34:36,709 - root - INFO - lr: 1.6740e-04 gnorm: 0.77 [10:59:36<11:01:11] +[titan] 2025-07-10 00:34:40,661 - root - INFO - step: 49945 loss: 16.4432 memory: 44.58GiB(31.99%) tps: 82,918 tflops: 286.17 mfu: 28.93% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9193 +[titan] 2025-07-10 00:34:40,661 - root - INFO - lr: 1.6738e-04 gnorm: 0.80 [10:59:40<11:01:07] +[titan] 2025-07-10 00:34:43,782 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:34:44,573 - root - INFO - step: 49950 loss: 16.5569 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 14.0095 +[titan] 2025-07-10 00:34:44,574 - root - INFO - lr: 1.6735e-04 gnorm: 0.78 [10:59:44<11:01:03] +[titan] 2025-07-10 00:34:48,480 - root - INFO - step: 49955 loss: 16.6067 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0497 +[titan] 2025-07-10 00:34:48,480 - root - INFO - lr: 1.6733e-04 gnorm: 0.87 [10:59:48<11:00:59] +[titan] 2025-07-10 00:34:52,408 - root - INFO - step: 49960 loss: 16.3173 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8189 +[titan] 2025-07-10 00:34:52,408 - root - INFO - lr: 1.6731e-04 gnorm: 0.80 [10:59:52<11:00:55] +[titan] 2025-07-10 00:34:56,345 - root - INFO - step: 49965 loss: 16.6377 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-10 00:34:56,345 - root - INFO - lr: 1.6729e-04 gnorm: 0.82 [10:59:56<11:00:51] +[titan] 2025-07-10 00:35:00,253 - root - INFO - step: 49970 loss: 16.4073 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8887 +[titan] 2025-07-10 00:35:00,253 - root - INFO - lr: 1.6727e-04 gnorm: 0.85 [11:00:00<11:00:47] +[titan] 2025-07-10 00:35:04,190 - root - INFO - step: 49975 loss: 16.4256 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 13.8779 +[titan] 2025-07-10 00:35:04,190 - root - INFO - lr: 1.6725e-04 gnorm: 0.86 [11:00:04<11:00:43] +[titan] 2025-07-10 00:35:08,127 - root - INFO - step: 49980 loss: 16.5908 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-10 00:35:08,127 - root - INFO - lr: 1.6723e-04 gnorm: 0.87 [11:00:08<11:00:39] +[titan] 2025-07-10 00:35:12,040 - root - INFO - step: 49985 loss: 16.5863 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-10 00:35:12,040 - root - INFO - lr: 1.6720e-04 gnorm: 0.86 [11:00:12<11:00:35] +[titan] 2025-07-10 00:35:15,966 - root - INFO - step: 49990 loss: 16.6812 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5863 global_avg_mtp_loss: 14.0949 +[titan] 2025-07-10 00:35:15,967 - root - INFO - lr: 1.6718e-04 gnorm: 0.81 [11:00:16<11:00:31] +[titan] 2025-07-10 00:35:19,886 - root - INFO - step: 49995 loss: 16.7560 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5898 global_avg_mtp_loss: 14.1662 +[titan] 2025-07-10 00:35:19,886 - root - INFO - lr: 1.6716e-04 gnorm: 0.84 [11:00:19<11:00:27] +[titan] 2025-07-10 00:35:23,039 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:35:23,832 - root - INFO - step: 50000 loss: 16.7004 memory: 44.58GiB(31.99%) tps: 83,049 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.1231 +[titan] 2025-07-10 00:35:23,832 - root - INFO - lr: 1.6714e-04 gnorm: 0.87 [11:00:23<11:00:23] +[titan] 2025-07-10 00:35:23,832 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 00:35:26,459 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 00:35:26,459 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.63 seconds. +[titan] 2025-07-10 00:36:21,193 - root - INFO - step: 50005 loss: 16.1523 memory: 44.58GiB(31.99%) tps: 5,713 tflops: 19.72 mfu: 1.99% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.6590 +[titan] 2025-07-10 00:36:21,193 - root - INFO - lr: 1.6712e-04 gnorm: 0.78 [11:01:21<11:01:13] +[titan] 2025-07-10 00:36:25,152 - root - INFO - step: 50010 loss: 16.5345 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-10 00:36:25,152 - root - INFO - lr: 1.6710e-04 gnorm: 0.87 [11:01:25<11:01:09] +[titan] 2025-07-10 00:36:29,101 - root - INFO - step: 50015 loss: 16.6388 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5946 global_avg_mtp_loss: 14.0442 +[titan] 2025-07-10 00:36:29,101 - root - INFO - lr: 1.6708e-04 gnorm: 0.85 [11:01:29<11:01:05] +[titan] 2025-07-10 00:36:33,058 - root - INFO - step: 50020 loss: 16.7872 memory: 44.58GiB(31.99%) tps: 82,812 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.5944 global_avg_mtp_loss: 14.1928 +[titan] 2025-07-10 00:36:33,058 - root - INFO - lr: 1.6705e-04 gnorm: 0.83 [11:01:33<11:01:01] +[titan] 2025-07-10 00:36:36,980 - root - INFO - step: 50025 loss: 16.5045 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9657 +[titan] 2025-07-10 00:36:36,980 - root - INFO - lr: 1.6703e-04 gnorm: 0.79 [11:01:37<11:00:57] +[titan] 2025-07-10 00:36:40,928 - root - INFO - step: 50030 loss: 16.5040 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.9699 +[titan] 2025-07-10 00:36:40,929 - root - INFO - lr: 1.6701e-04 gnorm: 0.78 [11:01:40<11:00:53] +[titan] 2025-07-10 00:36:44,852 - root - INFO - step: 50035 loss: 16.1981 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7142 +[titan] 2025-07-10 00:36:44,852 - root - INFO - lr: 1.6699e-04 gnorm: 0.79 [11:01:44<11:00:49] +[titan] 2025-07-10 00:36:48,794 - root - INFO - step: 50040 loss: 16.8096 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.2103 +[titan] 2025-07-10 00:36:48,794 - root - INFO - lr: 1.6697e-04 gnorm: 0.77 [11:01:48<11:00:45] +[titan] 2025-07-10 00:36:52,758 - root - INFO - step: 50045 loss: 16.4865 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9418 +[titan] 2025-07-10 00:36:52,758 - root - INFO - lr: 1.6695e-04 gnorm: 0.80 [11:01:52<11:00:41] +[titan] 2025-07-10 00:36:56,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:36:56,802 - root - INFO - step: 50050 loss: 16.5090 memory: 44.58GiB(31.99%) tps: 81,031 tflops: 279.65 mfu: 28.28% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9733 +[titan] 2025-07-10 00:36:56,803 - root - INFO - lr: 1.6693e-04 gnorm: 0.85 [11:01:56<11:00:37] +[titan] 2025-07-10 00:37:00,760 - root - INFO - step: 50055 loss: 16.4579 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9235 +[titan] 2025-07-10 00:37:00,760 - root - INFO - lr: 1.6690e-04 gnorm: 0.80 [11:02:00<11:00:33] +[titan] 2025-07-10 00:37:04,752 - root - INFO - step: 50060 loss: 16.2913 memory: 44.58GiB(31.99%) tps: 82,081 tflops: 283.27 mfu: 28.64% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.7879 +[titan] 2025-07-10 00:37:04,753 - root - INFO - lr: 1.6688e-04 gnorm: 0.87 [11:02:04<11:00:29] +[titan] 2025-07-10 00:37:08,742 - root - INFO - step: 50065 loss: 16.7448 memory: 44.58GiB(31.99%) tps: 82,134 tflops: 283.46 mfu: 28.66% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1575 +[titan] 2025-07-10 00:37:08,743 - root - INFO - lr: 1.6686e-04 gnorm: 0.77 [11:02:08<11:00:25] +[titan] 2025-07-10 00:37:12,713 - root - INFO - step: 50070 loss: 16.3732 memory: 44.58GiB(31.99%) tps: 82,528 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.5184 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-10 00:37:12,713 - root - INFO - lr: 1.6684e-04 gnorm: 0.91 [11:02:12<11:00:21] +[titan] 2025-07-10 00:37:16,663 - root - INFO - step: 50075 loss: 16.4642 memory: 44.58GiB(31.99%) tps: 82,962 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.9333 +[titan] 2025-07-10 00:37:16,664 - root - INFO - lr: 1.6682e-04 gnorm: 0.83 [11:02:16<11:00:17] +[titan] 2025-07-10 00:37:20,656 - root - INFO - step: 50080 loss: 16.3881 memory: 44.58GiB(31.99%) tps: 82,069 tflops: 283.23 mfu: 28.64% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.8599 +[titan] 2025-07-10 00:37:20,657 - root - INFO - lr: 1.6680e-04 gnorm: 0.81 [11:02:20<11:00:13] +[titan] 2025-07-10 00:37:24,600 - root - INFO - step: 50085 loss: 16.3913 memory: 44.58GiB(31.99%) tps: 83,093 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.8737 +[titan] 2025-07-10 00:37:24,601 - root - INFO - lr: 1.6678e-04 gnorm: 0.82 [11:02:24<11:00:09] +[titan] 2025-07-10 00:37:28,538 - root - INFO - step: 50090 loss: 16.4308 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9027 +[titan] 2025-07-10 00:37:28,538 - root - INFO - lr: 1.6675e-04 gnorm: 0.84 [11:02:28<11:00:05] +[titan] 2025-07-10 00:37:32,495 - root - INFO - step: 50095 loss: 16.6194 memory: 44.58GiB(31.99%) tps: 82,805 tflops: 285.77 mfu: 28.90% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0524 +[titan] 2025-07-10 00:37:32,496 - root - INFO - lr: 1.6673e-04 gnorm: 0.85 [11:02:32<11:00:01] +[titan] 2025-07-10 00:37:35,595 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:37:36,385 - root - INFO - step: 50100 loss: 16.4005 memory: 44.58GiB(31.99%) tps: 84,243 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8950 +[titan] 2025-07-10 00:37:36,386 - root - INFO - lr: 1.6671e-04 gnorm: 0.88 [11:02:36<10:59:57] +[titan] 2025-07-10 00:37:40,313 - root - INFO - step: 50105 loss: 16.6983 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5663 global_avg_mtp_loss: 14.1320 +[titan] 2025-07-10 00:37:40,313 - root - INFO - lr: 1.6669e-04 gnorm: 0.77 [11:02:40<10:59:53] +[titan] 2025-07-10 00:37:44,239 - root - INFO - step: 50110 loss: 16.3941 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8834 +[titan] 2025-07-10 00:37:44,239 - root - INFO - lr: 1.6667e-04 gnorm: 0.85 [11:02:44<10:59:49] +[titan] 2025-07-10 00:37:48,177 - root - INFO - step: 50115 loss: 16.4679 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9274 +[titan] 2025-07-10 00:37:48,177 - root - INFO - lr: 1.6665e-04 gnorm: 0.85 [11:02:48<10:59:45] +[titan] 2025-07-10 00:37:52,085 - root - INFO - step: 50120 loss: 16.5023 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9665 +[titan] 2025-07-10 00:37:52,085 - root - INFO - lr: 1.6663e-04 gnorm: 0.82 [11:02:52<10:59:41] +[titan] 2025-07-10 00:37:55,988 - root - INFO - step: 50125 loss: 16.5272 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 13.9792 +[titan] 2025-07-10 00:37:55,988 - root - INFO - lr: 1.6660e-04 gnorm: 0.82 [11:02:56<10:59:37] +[titan] 2025-07-10 00:37:59,905 - root - INFO - step: 50130 loss: 16.4611 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9388 +[titan] 2025-07-10 00:37:59,906 - root - INFO - lr: 1.6658e-04 gnorm: 0.80 [11:02:59<10:59:33] +[titan] 2025-07-10 00:38:03,813 - root - INFO - step: 50135 loss: 16.5724 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 14.0161 +[titan] 2025-07-10 00:38:03,813 - root - INFO - lr: 1.6656e-04 gnorm: 0.84 [11:03:03<10:59:29] +[titan] 2025-07-10 00:38:07,723 - root - INFO - step: 50140 loss: 16.5284 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 13.9930 +[titan] 2025-07-10 00:38:07,723 - root - INFO - lr: 1.6654e-04 gnorm: 0.79 [11:03:07<10:59:25] +[titan] 2025-07-10 00:38:11,640 - root - INFO - step: 50145 loss: 16.7156 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1260 +[titan] 2025-07-10 00:38:11,640 - root - INFO - lr: 1.6652e-04 gnorm: 0.92 [11:03:11<10:59:21] +[titan] 2025-07-10 00:38:14,799 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:38:15,600 - root - INFO - step: 50150 loss: 16.4373 memory: 44.58GiB(31.99%) tps: 82,762 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9132 +[titan] 2025-07-10 00:38:15,600 - root - INFO - lr: 1.6650e-04 gnorm: 0.88 [11:03:15<10:59:17] +[titan] 2025-07-10 00:38:19,518 - root - INFO - step: 50155 loss: 16.5578 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-10 00:38:19,518 - root - INFO - lr: 1.6648e-04 gnorm: 0.84 [11:03:19<10:59:13] +[titan] 2025-07-10 00:38:23,420 - root - INFO - step: 50160 loss: 16.7227 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.1303 +[titan] 2025-07-10 00:38:23,420 - root - INFO - lr: 1.6645e-04 gnorm: 0.81 [11:03:23<10:59:09] +[titan] 2025-07-10 00:38:27,321 - root - INFO - step: 50165 loss: 16.5085 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 13.9662 +[titan] 2025-07-10 00:38:27,321 - root - INFO - lr: 1.6643e-04 gnorm: 0.82 [11:03:27<10:59:05] +[titan] 2025-07-10 00:38:31,252 - root - INFO - step: 50170 loss: 16.5040 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 13.9580 +[titan] 2025-07-10 00:38:31,252 - root - INFO - lr: 1.6641e-04 gnorm: 0.81 [11:03:31<10:59:01] +[titan] 2025-07-10 00:38:35,252 - root - INFO - step: 50175 loss: 16.5392 memory: 44.58GiB(31.99%) tps: 81,925 tflops: 282.74 mfu: 28.59% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9881 +[titan] 2025-07-10 00:38:35,253 - root - INFO - lr: 1.6639e-04 gnorm: 0.81 [11:03:35<10:58:57] +[titan] 2025-07-10 00:38:36,189 - root - INFO - Dumping profiler traces at step 50176 +[titan] 2025-07-10 00:38:36,221 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:38:39,370 - root - INFO - step: 50180 loss: 16.2124 memory: 44.58GiB(31.99%) tps: 79,578 tflops: 274.64 mfu: 27.77% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.7207 +[titan] 2025-07-10 00:38:39,371 - root - INFO - lr: 1.6637e-04 gnorm: 0.82 [11:03:39<10:58:53] +[titan] 2025-07-10 00:38:43,277 - root - INFO - step: 50185 loss: 16.4884 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.9726 +[titan] 2025-07-10 00:38:43,277 - root - INFO - lr: 1.6635e-04 gnorm: 0.77 [11:03:43<10:58:49] +[titan] 2025-07-10 00:38:47,187 - root - INFO - step: 50190 loss: 16.4039 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.8828 +[titan] 2025-07-10 00:38:47,187 - root - INFO - lr: 1.6633e-04 gnorm: 0.80 [11:03:47<10:58:45] +[titan] 2025-07-10 00:38:51,106 - root - INFO - step: 50195 loss: 16.6004 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5492 global_avg_mtp_loss: 14.0512 +[titan] 2025-07-10 00:38:51,106 - root - INFO - lr: 1.6630e-04 gnorm: 0.82 [11:03:51<10:58:41] +[titan] 2025-07-10 00:38:54,234 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:38:55,031 - root - INFO - step: 50200 loss: 16.7796 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1945 +[titan] 2025-07-10 00:38:55,031 - root - INFO - lr: 1.6628e-04 gnorm: 0.82 [11:03:55<10:58:37] +[titan] 2025-07-10 00:38:58,992 - root - INFO - step: 50205 loss: 16.5486 memory: 44.58GiB(31.99%) tps: 82,740 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.5549 global_avg_mtp_loss: 13.9937 +[titan] 2025-07-10 00:38:58,992 - root - INFO - lr: 1.6626e-04 gnorm: 0.83 [11:03:59<10:58:33] +[titan] 2025-07-10 00:39:02,921 - root - INFO - step: 50210 loss: 16.5803 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0181 +[titan] 2025-07-10 00:39:02,921 - root - INFO - lr: 1.6624e-04 gnorm: 0.82 [11:04:02<10:58:29] +[titan] 2025-07-10 00:39:06,810 - root - INFO - step: 50215 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 84,270 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-10 00:39:06,810 - root - INFO - lr: 1.6622e-04 gnorm: 0.92 [11:04:06<10:58:25] +[titan] 2025-07-10 00:39:10,724 - root - INFO - step: 50220 loss: 16.4771 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9347 +[titan] 2025-07-10 00:39:10,724 - root - INFO - lr: 1.6620e-04 gnorm: 0.85 [11:04:10<10:58:21] +[titan] 2025-07-10 00:39:14,620 - root - INFO - step: 50225 loss: 16.4662 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.9371 +[titan] 2025-07-10 00:39:14,620 - root - INFO - lr: 1.6618e-04 gnorm: 0.88 [11:04:14<10:58:17] +[titan] 2025-07-10 00:39:18,559 - root - INFO - step: 50230 loss: 16.7488 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1667 +[titan] 2025-07-10 00:39:18,559 - root - INFO - lr: 1.6615e-04 gnorm: 0.84 [11:04:18<10:58:13] +[titan] 2025-07-10 00:39:22,481 - root - INFO - step: 50235 loss: 16.6930 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.1165 +[titan] 2025-07-10 00:39:22,482 - root - INFO - lr: 1.6613e-04 gnorm: 0.86 [11:04:22<10:58:09] +[titan] 2025-07-10 00:39:26,396 - root - INFO - step: 50240 loss: 16.6643 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 14.1023 +[titan] 2025-07-10 00:39:26,396 - root - INFO - lr: 1.6611e-04 gnorm: 0.87 [11:04:26<10:58:05] +[titan] 2025-07-10 00:39:30,309 - root - INFO - step: 50245 loss: 16.2783 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.7756 +[titan] 2025-07-10 00:39:30,310 - root - INFO - lr: 1.6609e-04 gnorm: 0.88 [11:04:30<10:58:01] +[titan] 2025-07-10 00:39:33,428 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:39:34,234 - root - INFO - step: 50250 loss: 16.4314 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.9157 +[titan] 2025-07-10 00:39:34,235 - root - INFO - lr: 1.6607e-04 gnorm: 0.89 [11:04:34<10:57:57] +[titan] 2025-07-10 00:39:38,162 - root - INFO - step: 50255 loss: 16.5361 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9932 +[titan] 2025-07-10 00:39:38,162 - root - INFO - lr: 1.6605e-04 gnorm: 0.83 [11:04:38<10:57:53] +[titan] 2025-07-10 00:39:42,064 - root - INFO - step: 50260 loss: 16.5552 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0058 +[titan] 2025-07-10 00:39:42,065 - root - INFO - lr: 1.6603e-04 gnorm: 0.82 [11:04:42<10:57:49] +[titan] 2025-07-10 00:39:45,960 - root - INFO - step: 50265 loss: 16.4646 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5244 global_avg_mtp_loss: 13.9402 +[titan] 2025-07-10 00:39:45,960 - root - INFO - lr: 1.6600e-04 gnorm: 0.78 [11:04:45<10:57:45] +[titan] 2025-07-10 00:39:49,864 - root - INFO - step: 50270 loss: 16.3524 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8428 +[titan] 2025-07-10 00:39:49,865 - root - INFO - lr: 1.6598e-04 gnorm: 0.74 [11:04:49<10:57:41] +[titan] 2025-07-10 00:39:53,775 - root - INFO - step: 50275 loss: 16.6612 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0887 +[titan] 2025-07-10 00:39:53,775 - root - INFO - lr: 1.6596e-04 gnorm: 0.80 [11:04:53<10:57:37] +[titan] 2025-07-10 00:39:57,722 - root - INFO - step: 50280 loss: 16.7749 memory: 44.58GiB(31.99%) tps: 83,033 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5949 global_avg_mtp_loss: 14.1800 +[titan] 2025-07-10 00:39:57,722 - root - INFO - lr: 1.6594e-04 gnorm: 0.83 [11:04:57<10:57:33] +[titan] 2025-07-10 00:40:01,646 - root - INFO - step: 50285 loss: 16.4187 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-10 00:40:01,646 - root - INFO - lr: 1.6592e-04 gnorm: 0.81 [11:05:01<10:57:29] +[titan] 2025-07-10 00:40:05,568 - root - INFO - step: 50290 loss: 16.5302 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 13.9790 +[titan] 2025-07-10 00:40:05,568 - root - INFO - lr: 1.6590e-04 gnorm: 0.79 [11:05:05<10:57:25] +[titan] 2025-07-10 00:40:09,466 - root - INFO - step: 50295 loss: 16.4633 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-10 00:40:09,466 - root - INFO - lr: 1.6588e-04 gnorm: 0.85 [11:05:09<10:57:21] +[titan] 2025-07-10 00:40:12,585 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:40:13,378 - root - INFO - step: 50300 loss: 16.2218 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.7174 +[titan] 2025-07-10 00:40:13,378 - root - INFO - lr: 1.6585e-04 gnorm: 0.84 [11:05:13<10:57:17] +[titan] 2025-07-10 00:40:17,294 - root - INFO - step: 50305 loss: 16.7734 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5873 global_avg_mtp_loss: 14.1860 +[titan] 2025-07-10 00:40:17,294 - root - INFO - lr: 1.6583e-04 gnorm: 0.85 [11:05:17<10:57:13] +[titan] 2025-07-10 00:40:21,229 - root - INFO - step: 50310 loss: 16.6048 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0420 +[titan] 2025-07-10 00:40:21,230 - root - INFO - lr: 1.6581e-04 gnorm: 0.78 [11:05:21<10:57:09] +[titan] 2025-07-10 00:40:25,174 - root - INFO - step: 50315 loss: 16.5350 memory: 44.58GiB(31.99%) tps: 83,082 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9908 +[titan] 2025-07-10 00:40:25,174 - root - INFO - lr: 1.6579e-04 gnorm: 0.80 [11:05:25<10:57:05] +[titan] 2025-07-10 00:40:29,108 - root - INFO - step: 50320 loss: 16.7520 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5795 global_avg_mtp_loss: 14.1725 +[titan] 2025-07-10 00:40:29,108 - root - INFO - lr: 1.6577e-04 gnorm: 0.79 [11:05:29<10:57:01] +[titan] 2025-07-10 00:40:33,073 - root - INFO - step: 50325 loss: 16.4258 memory: 44.58GiB(31.99%) tps: 82,652 tflops: 285.25 mfu: 28.84% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9024 +[titan] 2025-07-10 00:40:33,073 - root - INFO - lr: 1.6575e-04 gnorm: 0.81 [11:05:33<10:56:57] +[titan] 2025-07-10 00:40:36,991 - root - INFO - step: 50330 loss: 16.5483 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 13.9968 +[titan] 2025-07-10 00:40:36,992 - root - INFO - lr: 1.6573e-04 gnorm: 0.76 [11:05:37<10:56:53] +[titan] 2025-07-10 00:40:40,918 - root - INFO - step: 50335 loss: 16.6193 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 14.0613 +[titan] 2025-07-10 00:40:40,919 - root - INFO - lr: 1.6570e-04 gnorm: 0.81 [11:05:40<10:56:49] +[titan] 2025-07-10 00:40:44,827 - root - INFO - step: 50340 loss: 16.5736 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0186 +[titan] 2025-07-10 00:40:44,827 - root - INFO - lr: 1.6568e-04 gnorm: 0.84 [11:05:44<10:56:45] +[titan] 2025-07-10 00:40:48,793 - root - INFO - step: 50345 loss: 16.5562 memory: 44.58GiB(31.99%) tps: 82,630 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 14.0056 +[titan] 2025-07-10 00:40:48,793 - root - INFO - lr: 1.6566e-04 gnorm: 0.77 [11:05:48<10:56:41] +[titan] 2025-07-10 00:40:51,957 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:40:52,750 - root - INFO - step: 50350 loss: 16.2763 memory: 44.58GiB(31.99%) tps: 82,820 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.7721 +[titan] 2025-07-10 00:40:52,750 - root - INFO - lr: 1.6564e-04 gnorm: 0.83 [11:05:52<10:56:37] +[titan] 2025-07-10 00:40:56,658 - root - INFO - step: 50355 loss: 16.6576 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.0766 +[titan] 2025-07-10 00:40:56,658 - root - INFO - lr: 1.6562e-04 gnorm: 0.82 [11:05:56<10:56:33] +[titan] 2025-07-10 00:41:00,582 - root - INFO - step: 50360 loss: 16.5210 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9823 +[titan] 2025-07-10 00:41:00,582 - root - INFO - lr: 1.6560e-04 gnorm: 0.82 [11:06:00<10:56:29] +[titan] 2025-07-10 00:41:04,520 - root - INFO - step: 50365 loss: 16.5524 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 14.0084 +[titan] 2025-07-10 00:41:04,520 - root - INFO - lr: 1.6558e-04 gnorm: 0.83 [11:06:04<10:56:25] +[titan] 2025-07-10 00:41:08,478 - root - INFO - step: 50370 loss: 16.3432 memory: 44.58GiB(31.99%) tps: 82,783 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8406 +[titan] 2025-07-10 00:41:08,479 - root - INFO - lr: 1.6555e-04 gnorm: 0.82 [11:06:08<10:56:21] +[titan] 2025-07-10 00:41:12,419 - root - INFO - step: 50375 loss: 16.5534 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 14.0116 +[titan] 2025-07-10 00:41:12,419 - root - INFO - lr: 1.6553e-04 gnorm: 0.92 [11:06:12<10:56:17] +[titan] 2025-07-10 00:41:16,327 - root - INFO - step: 50380 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.6998 +[titan] 2025-07-10 00:41:16,327 - root - INFO - lr: 1.6551e-04 gnorm: 1.06 [11:06:16<10:56:13] +[titan] 2025-07-10 00:41:20,264 - root - INFO - step: 50385 loss: 16.6676 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-10 00:41:20,264 - root - INFO - lr: 1.6549e-04 gnorm: 0.86 [11:06:20<10:56:09] +[titan] 2025-07-10 00:41:24,181 - root - INFO - step: 50390 loss: 16.5399 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5545 global_avg_mtp_loss: 13.9854 +[titan] 2025-07-10 00:41:24,182 - root - INFO - lr: 1.6547e-04 gnorm: 0.88 [11:06:24<10:56:05] +[titan] 2025-07-10 00:41:28,154 - root - INFO - step: 50395 loss: 16.5232 memory: 44.58GiB(31.99%) tps: 82,485 tflops: 284.67 mfu: 28.78% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 13.9775 +[titan] 2025-07-10 00:41:28,155 - root - INFO - lr: 1.6545e-04 gnorm: 0.87 [11:06:28<10:56:01] +[titan] 2025-07-10 00:41:31,260 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:41:32,054 - root - INFO - step: 50400 loss: 16.3813 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8705 +[titan] 2025-07-10 00:41:32,055 - root - INFO - lr: 1.6543e-04 gnorm: 0.80 [11:06:32<10:55:57] +[titan] 2025-07-10 00:41:35,973 - root - INFO - step: 50405 loss: 16.3888 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.8641 +[titan] 2025-07-10 00:41:35,973 - root - INFO - lr: 1.6540e-04 gnorm: 0.82 [11:06:35<10:55:53] +[titan] 2025-07-10 00:41:39,887 - root - INFO - step: 50410 loss: 16.3910 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8759 +[titan] 2025-07-10 00:41:39,887 - root - INFO - lr: 1.6538e-04 gnorm: 0.82 [11:06:39<10:55:49] +[titan] 2025-07-10 00:41:43,802 - root - INFO - step: 50415 loss: 16.2532 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.7618 +[titan] 2025-07-10 00:41:43,803 - root - INFO - lr: 1.6536e-04 gnorm: 0.86 [11:06:43<10:55:45] +[titan] 2025-07-10 00:41:47,760 - root - INFO - step: 50420 loss: 16.4729 memory: 44.58GiB(31.99%) tps: 82,799 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 13.9273 +[titan] 2025-07-10 00:41:47,760 - root - INFO - lr: 1.6534e-04 gnorm: 0.79 [11:06:47<10:55:41] +[titan] 2025-07-10 00:41:51,654 - root - INFO - step: 50425 loss: 16.4549 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5431 global_avg_mtp_loss: 13.9118 +[titan] 2025-07-10 00:41:51,654 - root - INFO - lr: 1.6532e-04 gnorm: 0.81 [11:06:51<10:55:37] +[titan] 2025-07-10 00:41:55,555 - root - INFO - step: 50430 loss: 16.2891 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.7874 +[titan] 2025-07-10 00:41:55,556 - root - INFO - lr: 1.6530e-04 gnorm: 0.84 [11:06:55<10:55:33] +[titan] 2025-07-10 00:41:59,505 - root - INFO - step: 50435 loss: 16.5667 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0170 +[titan] 2025-07-10 00:41:59,506 - root - INFO - lr: 1.6528e-04 gnorm: 0.83 [11:06:59<10:55:29] +[titan] 2025-07-10 00:42:03,403 - root - INFO - step: 50440 loss: 16.6159 memory: 44.58GiB(31.99%) tps: 84,084 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0470 +[titan] 2025-07-10 00:42:03,403 - root - INFO - lr: 1.6525e-04 gnorm: 0.77 [11:07:03<10:55:25] +[titan] 2025-07-10 00:42:07,318 - root - INFO - step: 50445 loss: 16.5948 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5529 global_avg_mtp_loss: 14.0419 +[titan] 2025-07-10 00:42:07,319 - root - INFO - lr: 1.6523e-04 gnorm: 0.84 [11:07:07<10:55:21] +[titan] 2025-07-10 00:42:10,465 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:42:11,262 - root - INFO - step: 50450 loss: 16.3377 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.8199 +[titan] 2025-07-10 00:42:11,262 - root - INFO - lr: 1.6521e-04 gnorm: 0.78 [11:07:11<10:55:17] +[titan] 2025-07-10 00:42:15,190 - root - INFO - step: 50455 loss: 16.6382 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 14.0916 +[titan] 2025-07-10 00:42:15,190 - root - INFO - lr: 1.6519e-04 gnorm: 0.79 [11:07:15<10:55:13] +[titan] 2025-07-10 00:42:19,116 - root - INFO - step: 50460 loss: 16.4172 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.8909 +[titan] 2025-07-10 00:42:19,116 - root - INFO - lr: 1.6517e-04 gnorm: 0.79 [11:07:19<10:55:09] +[titan] 2025-07-10 00:42:23,046 - root - INFO - step: 50465 loss: 16.3954 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8720 +[titan] 2025-07-10 00:42:23,046 - root - INFO - lr: 1.6515e-04 gnorm: 0.82 [11:07:23<10:55:05] +[titan] 2025-07-10 00:42:26,955 - root - INFO - step: 50470 loss: 16.5833 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 14.0331 +[titan] 2025-07-10 00:42:26,955 - root - INFO - lr: 1.6513e-04 gnorm: 0.80 [11:07:26<10:55:01] +[titan] 2025-07-10 00:42:30,858 - root - INFO - step: 50475 loss: 16.3671 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-10 00:42:30,859 - root - INFO - lr: 1.6510e-04 gnorm: 0.79 [11:07:30<10:54:57] +[titan] 2025-07-10 00:42:34,785 - root - INFO - step: 50480 loss: 16.4349 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.9097 +[titan] 2025-07-10 00:42:34,785 - root - INFO - lr: 1.6508e-04 gnorm: 0.88 [11:07:34<10:54:53] +[titan] 2025-07-10 00:42:38,694 - root - INFO - step: 50485 loss: 16.5640 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 14.0211 +[titan] 2025-07-10 00:42:38,694 - root - INFO - lr: 1.6506e-04 gnorm: 0.81 [11:07:38<10:54:49] +[titan] 2025-07-10 00:42:42,605 - root - INFO - step: 50490 loss: 16.6909 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1079 +[titan] 2025-07-10 00:42:42,606 - root - INFO - lr: 1.6504e-04 gnorm: 0.84 [11:07:42<10:54:45] +[titan] 2025-07-10 00:42:46,518 - root - INFO - step: 50495 loss: 16.5553 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 14.0114 +[titan] 2025-07-10 00:42:46,518 - root - INFO - lr: 1.6502e-04 gnorm: 0.83 [11:07:46<10:54:40] +[titan] 2025-07-10 00:42:49,662 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:42:50,451 - root - INFO - step: 50500 loss: 16.4622 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9174 +[titan] 2025-07-10 00:42:50,452 - root - INFO - lr: 1.6500e-04 gnorm: 0.79 [11:07:50<10:54:36] +[titan] 2025-07-10 00:42:54,352 - root - INFO - step: 50505 loss: 16.6357 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0817 +[titan] 2025-07-10 00:42:54,353 - root - INFO - lr: 1.6498e-04 gnorm: 0.80 [11:07:54<10:54:32] +[titan] 2025-07-10 00:42:58,267 - root - INFO - step: 50510 loss: 16.6463 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0733 +[titan] 2025-07-10 00:42:58,267 - root - INFO - lr: 1.6496e-04 gnorm: 0.81 [11:07:58<10:54:28] +[titan] 2025-07-10 00:43:02,167 - root - INFO - step: 50515 loss: 16.4460 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.9208 +[titan] 2025-07-10 00:43:02,167 - root - INFO - lr: 1.6493e-04 gnorm: 0.80 [11:08:02<10:54:24] +[titan] 2025-07-10 00:43:06,075 - root - INFO - step: 50520 loss: 16.6015 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0374 +[titan] 2025-07-10 00:43:06,076 - root - INFO - lr: 1.6491e-04 gnorm: 0.83 [11:08:06<10:54:20] +[titan] 2025-07-10 00:43:09,978 - root - INFO - step: 50525 loss: 16.7454 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1650 +[titan] 2025-07-10 00:43:09,978 - root - INFO - lr: 1.6489e-04 gnorm: 0.81 [11:08:09<10:54:16] +[titan] 2025-07-10 00:43:13,884 - root - INFO - step: 50530 loss: 17.0474 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.6525 global_avg_mtp_loss: 14.3949 +[titan] 2025-07-10 00:43:13,885 - root - INFO - lr: 1.6487e-04 gnorm: 0.83 [11:08:13<10:54:12] +[titan] 2025-07-10 00:43:17,798 - root - INFO - step: 50535 loss: 16.5412 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-10 00:43:17,799 - root - INFO - lr: 1.6485e-04 gnorm: 0.83 [11:08:17<10:54:08] +[titan] 2025-07-10 00:43:21,730 - root - INFO - step: 50540 loss: 16.6554 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0897 +[titan] 2025-07-10 00:43:21,730 - root - INFO - lr: 1.6483e-04 gnorm: 0.83 [11:08:21<10:54:04] +[titan] 2025-07-10 00:43:25,652 - root - INFO - step: 50545 loss: 16.6438 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 14.0782 +[titan] 2025-07-10 00:43:25,653 - root - INFO - lr: 1.6481e-04 gnorm: 0.84 [11:08:25<10:54:00] +[titan] 2025-07-10 00:43:28,786 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:43:29,574 - root - INFO - step: 50550 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.6973 +[titan] 2025-07-10 00:43:29,575 - root - INFO - lr: 1.6478e-04 gnorm: 0.78 [11:08:29<10:53:56] +[titan] 2025-07-10 00:43:33,507 - root - INFO - step: 50555 loss: 16.1185 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.6378 +[titan] 2025-07-10 00:43:33,507 - root - INFO - lr: 1.6476e-04 gnorm: 0.87 [11:08:33<10:53:52] +[titan] 2025-07-10 00:43:37,437 - root - INFO - step: 50560 loss: 16.7003 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1159 +[titan] 2025-07-10 00:43:37,437 - root - INFO - lr: 1.6474e-04 gnorm: 0.80 [11:08:37<10:53:48] +[titan] 2025-07-10 00:43:41,342 - root - INFO - step: 50565 loss: 16.5443 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 14.0056 +[titan] 2025-07-10 00:43:41,343 - root - INFO - lr: 1.6472e-04 gnorm: 0.81 [11:08:41<10:53:44] +[titan] 2025-07-10 00:43:45,270 - root - INFO - step: 50570 loss: 16.6469 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.0666 +[titan] 2025-07-10 00:43:45,270 - root - INFO - lr: 1.6470e-04 gnorm: 0.78 [11:08:45<10:53:40] +[titan] 2025-07-10 00:43:49,159 - root - INFO - step: 50575 loss: 16.6296 memory: 44.58GiB(31.99%) tps: 84,259 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0597 +[titan] 2025-07-10 00:43:49,160 - root - INFO - lr: 1.6468e-04 gnorm: 0.78 [11:08:49<10:53:36] +[titan] 2025-07-10 00:43:53,096 - root - INFO - step: 50580 loss: 16.5843 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5539 global_avg_mtp_loss: 14.0304 +[titan] 2025-07-10 00:43:53,096 - root - INFO - lr: 1.6466e-04 gnorm: 0.82 [11:08:53<10:53:32] +[titan] 2025-07-10 00:43:57,009 - root - INFO - step: 50585 loss: 16.5564 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0128 +[titan] 2025-07-10 00:43:57,009 - root - INFO - lr: 1.6463e-04 gnorm: 0.81 [11:08:57<10:53:28] +[titan] 2025-07-10 00:44:00,914 - root - INFO - step: 50590 loss: 16.3291 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8143 +[titan] 2025-07-10 00:44:00,914 - root - INFO - lr: 1.6461e-04 gnorm: 0.82 [11:09:00<10:53:24] +[titan] 2025-07-10 00:44:04,842 - root - INFO - step: 50595 loss: 16.5778 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 14.0299 +[titan] 2025-07-10 00:44:04,842 - root - INFO - lr: 1.6459e-04 gnorm: 0.85 [11:09:04<10:53:20] +[titan] 2025-07-10 00:44:07,966 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:44:08,767 - root - INFO - step: 50600 loss: 16.4359 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.9178 +[titan] 2025-07-10 00:44:08,768 - root - INFO - lr: 1.6457e-04 gnorm: 0.84 [11:09:08<10:53:16] +[titan] 2025-07-10 00:44:12,689 - root - INFO - step: 50605 loss: 16.5662 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5605 global_avg_mtp_loss: 14.0057 +[titan] 2025-07-10 00:44:12,690 - root - INFO - lr: 1.6455e-04 gnorm: 0.87 [11:09:12<10:53:12] +[titan] 2025-07-10 00:44:16,615 - root - INFO - step: 50610 loss: 16.4370 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9055 +[titan] 2025-07-10 00:44:16,615 - root - INFO - lr: 1.6453e-04 gnorm: 0.76 [11:09:16<10:53:08] +[titan] 2025-07-10 00:44:20,577 - root - INFO - step: 50615 loss: 16.5882 memory: 44.58GiB(31.99%) tps: 82,707 tflops: 285.44 mfu: 28.86% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0345 +[titan] 2025-07-10 00:44:20,577 - root - INFO - lr: 1.6451e-04 gnorm: 0.81 [11:09:20<10:53:04] +[titan] 2025-07-10 00:44:24,476 - root - INFO - step: 50620 loss: 16.2995 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.7913 +[titan] 2025-07-10 00:44:24,476 - root - INFO - lr: 1.6448e-04 gnorm: 0.87 [11:09:24<10:53:00] +[titan] 2025-07-10 00:44:28,404 - root - INFO - step: 50625 loss: 16.5968 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5701 global_avg_mtp_loss: 14.0267 +[titan] 2025-07-10 00:44:28,405 - root - INFO - lr: 1.6446e-04 gnorm: 0.83 [11:09:28<10:52:56] +[titan] 2025-07-10 00:44:32,313 - root - INFO - step: 50630 loss: 16.2426 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7625 +[titan] 2025-07-10 00:44:32,313 - root - INFO - lr: 1.6444e-04 gnorm: 0.81 [11:09:32<10:52:52] +[titan] 2025-07-10 00:44:36,229 - root - INFO - step: 50635 loss: 16.5916 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-10 00:44:36,229 - root - INFO - lr: 1.6442e-04 gnorm: 0.80 [11:09:36<10:52:48] +[titan] 2025-07-10 00:44:40,150 - root - INFO - step: 50640 loss: 16.7352 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.1598 +[titan] 2025-07-10 00:44:40,150 - root - INFO - lr: 1.6440e-04 gnorm: 2.93 [11:09:40<10:52:44] +[titan] 2025-07-10 00:44:44,095 - root - INFO - step: 50645 loss: 16.5289 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9835 +[titan] 2025-07-10 00:44:44,095 - root - INFO - lr: 1.6438e-04 gnorm: 0.85 [11:09:44<10:52:40] +[titan] 2025-07-10 00:44:47,236 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:44:48,030 - root - INFO - step: 50650 loss: 16.5470 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 13.9955 +[titan] 2025-07-10 00:44:48,030 - root - INFO - lr: 1.6436e-04 gnorm: 0.79 [11:09:48<10:52:36] +[titan] 2025-07-10 00:44:51,942 - root - INFO - step: 50655 loss: 16.6566 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5827 global_avg_mtp_loss: 14.0739 +[titan] 2025-07-10 00:44:51,942 - root - INFO - lr: 1.6433e-04 gnorm: 0.85 [11:09:51<10:52:32] +[titan] 2025-07-10 00:44:55,850 - root - INFO - step: 50660 loss: 16.7550 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5771 global_avg_mtp_loss: 14.1779 +[titan] 2025-07-10 00:44:55,850 - root - INFO - lr: 1.6431e-04 gnorm: 0.83 [11:09:55<10:52:28] +[titan] 2025-07-10 00:44:59,778 - root - INFO - step: 50665 loss: 16.3602 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8560 +[titan] 2025-07-10 00:44:59,778 - root - INFO - lr: 1.6429e-04 gnorm: 0.77 [11:09:59<10:52:24] +[titan] 2025-07-10 00:45:03,719 - root - INFO - step: 50670 loss: 16.7871 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5923 global_avg_mtp_loss: 14.1948 +[titan] 2025-07-10 00:45:03,719 - root - INFO - lr: 1.6427e-04 gnorm: 0.83 [11:10:03<10:52:20] +[titan] 2025-07-10 00:45:07,626 - root - INFO - step: 50675 loss: 16.7675 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1784 +[titan] 2025-07-10 00:45:07,627 - root - INFO - lr: 1.6425e-04 gnorm: 0.80 [11:10:07<10:52:16] +[titan] 2025-07-10 00:45:11,554 - root - INFO - step: 50680 loss: 16.3196 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8133 +[titan] 2025-07-10 00:45:11,554 - root - INFO - lr: 1.6423e-04 gnorm: 0.88 [11:10:11<10:52:12] +[titan] 2025-07-10 00:45:15,541 - root - INFO - step: 50685 loss: 16.4688 memory: 44.58GiB(31.99%) tps: 82,196 tflops: 283.67 mfu: 28.68% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9321 +[titan] 2025-07-10 00:45:15,541 - root - INFO - lr: 1.6421e-04 gnorm: 0.84 [11:10:15<10:52:08] +[titan] 2025-07-10 00:45:18,049 - root - INFO - Dumping profiler traces at step 50688 +[titan] 2025-07-10 00:45:18,084 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:45:19,649 - root - INFO - step: 50690 loss: 16.5949 memory: 44.58GiB(31.99%) tps: 79,773 tflops: 275.31 mfu: 27.84% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0416 +[titan] 2025-07-10 00:45:19,649 - root - INFO - lr: 1.6418e-04 gnorm: 0.80 [11:10:19<10:52:04] +[titan] 2025-07-10 00:45:23,609 - root - INFO - step: 50695 loss: 16.6950 memory: 44.58GiB(31.99%) tps: 82,750 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.1311 +[titan] 2025-07-10 00:45:23,609 - root - INFO - lr: 1.6416e-04 gnorm: 0.84 [11:10:23<10:52:00] +[titan] 2025-07-10 00:45:26,754 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:45:27,574 - root - INFO - step: 50700 loss: 16.4921 memory: 44.58GiB(31.99%) tps: 82,656 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.9670 +[titan] 2025-07-10 00:45:27,574 - root - INFO - lr: 1.6414e-04 gnorm: 0.80 [11:10:27<10:51:56] +[titan] 2025-07-10 00:45:31,481 - root - INFO - step: 50705 loss: 16.6900 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5700 global_avg_mtp_loss: 14.1200 +[titan] 2025-07-10 00:45:31,482 - root - INFO - lr: 1.6412e-04 gnorm: 0.91 [11:10:31<10:51:52] +[titan] 2025-07-10 00:45:35,413 - root - INFO - step: 50710 loss: 16.6600 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0927 +[titan] 2025-07-10 00:45:35,414 - root - INFO - lr: 1.6410e-04 gnorm: 0.82 [11:10:35<10:51:48] +[titan] 2025-07-10 00:45:39,345 - root - INFO - step: 50715 loss: 16.6648 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5741 global_avg_mtp_loss: 14.0907 +[titan] 2025-07-10 00:45:39,346 - root - INFO - lr: 1.6408e-04 gnorm: 0.78 [11:10:39<10:51:44] +[titan] 2025-07-10 00:45:43,271 - root - INFO - step: 50720 loss: 16.6227 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0621 +[titan] 2025-07-10 00:45:43,271 - root - INFO - lr: 1.6406e-04 gnorm: 0.82 [11:10:43<10:51:40] +[titan] 2025-07-10 00:45:47,175 - root - INFO - step: 50725 loss: 16.4427 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 00:45:47,175 - root - INFO - lr: 1.6403e-04 gnorm: 0.81 [11:10:47<10:51:36] +[titan] 2025-07-10 00:45:51,074 - root - INFO - step: 50730 loss: 16.5182 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5375 global_avg_mtp_loss: 13.9807 +[titan] 2025-07-10 00:45:51,075 - root - INFO - lr: 1.6401e-04 gnorm: 0.79 [11:10:51<10:51:32] +[titan] 2025-07-10 00:45:55,007 - root - INFO - step: 50735 loss: 16.5681 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 14.0245 +[titan] 2025-07-10 00:45:55,007 - root - INFO - lr: 1.6399e-04 gnorm: 0.79 [11:10:54<10:51:28] +[titan] 2025-07-10 00:45:58,956 - root - INFO - step: 50740 loss: 16.3444 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.8338 +[titan] 2025-07-10 00:45:58,956 - root - INFO - lr: 1.6397e-04 gnorm: 0.79 [11:10:58<10:51:24] +[titan] 2025-07-10 00:46:02,870 - root - INFO - step: 50745 loss: 16.8329 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.6105 global_avg_mtp_loss: 14.2224 +[titan] 2025-07-10 00:46:02,870 - root - INFO - lr: 1.6395e-04 gnorm: 0.80 [11:11:02<10:51:20] +[titan] 2025-07-10 00:46:05,981 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:46:06,794 - root - INFO - step: 50750 loss: 16.4993 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9734 +[titan] 2025-07-10 00:46:06,794 - root - INFO - lr: 1.6393e-04 gnorm: 0.81 [11:11:06<10:51:16] +[titan] 2025-07-10 00:46:10,727 - root - INFO - step: 50755 loss: 16.5028 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9582 +[titan] 2025-07-10 00:46:10,727 - root - INFO - lr: 1.6391e-04 gnorm: 0.79 [11:11:10<10:51:12] +[titan] 2025-07-10 00:46:14,711 - root - INFO - step: 50760 loss: 16.2267 memory: 44.58GiB(31.99%) tps: 82,261 tflops: 283.90 mfu: 28.71% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.7359 +[titan] 2025-07-10 00:46:14,711 - root - INFO - lr: 1.6388e-04 gnorm: 0.81 [11:11:14<10:51:08] +[titan] 2025-07-10 00:46:18,636 - root - INFO - step: 50765 loss: 16.3843 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 00:46:18,636 - root - INFO - lr: 1.6386e-04 gnorm: 0.80 [11:11:18<10:51:04] +[titan] 2025-07-10 00:46:22,546 - root - INFO - step: 50770 loss: 16.6359 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0719 +[titan] 2025-07-10 00:46:22,546 - root - INFO - lr: 1.6384e-04 gnorm: 0.86 [11:11:22<10:51:00] +[titan] 2025-07-10 00:46:26,446 - root - INFO - step: 50775 loss: 16.6877 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 14.1230 +[titan] 2025-07-10 00:46:26,446 - root - INFO - lr: 1.6382e-04 gnorm: 0.74 [11:11:26<10:50:56] +[titan] 2025-07-10 00:46:30,364 - root - INFO - step: 50780 loss: 16.2587 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.7566 +[titan] 2025-07-10 00:46:30,364 - root - INFO - lr: 1.6380e-04 gnorm: 0.81 [11:11:30<10:50:52] +[titan] 2025-07-10 00:46:34,278 - root - INFO - step: 50785 loss: 16.4106 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.8908 +[titan] 2025-07-10 00:46:34,278 - root - INFO - lr: 1.6378e-04 gnorm: 0.78 [11:11:34<10:50:48] +[titan] 2025-07-10 00:46:38,183 - root - INFO - step: 50790 loss: 16.3624 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.8545 +[titan] 2025-07-10 00:46:38,184 - root - INFO - lr: 1.6376e-04 gnorm: 0.84 [11:11:38<10:50:44] +[titan] 2025-07-10 00:46:42,091 - root - INFO - step: 50795 loss: 16.7524 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.1692 +[titan] 2025-07-10 00:46:42,091 - root - INFO - lr: 1.6373e-04 gnorm: 0.80 [11:11:42<10:50:40] +[titan] 2025-07-10 00:46:45,204 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:46:45,996 - root - INFO - step: 50800 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9106 +[titan] 2025-07-10 00:46:45,996 - root - INFO - lr: 1.6371e-04 gnorm: 0.85 [11:11:45<10:50:36] +[titan] 2025-07-10 00:46:49,929 - root - INFO - step: 50805 loss: 16.9441 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.6287 global_avg_mtp_loss: 14.3153 +[titan] 2025-07-10 00:46:49,929 - root - INFO - lr: 1.6369e-04 gnorm: 0.80 [11:11:49<10:50:32] +[titan] 2025-07-10 00:46:53,825 - root - INFO - step: 50810 loss: 16.4654 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9173 +[titan] 2025-07-10 00:46:53,825 - root - INFO - lr: 1.6367e-04 gnorm: 0.79 [11:11:53<10:50:28] +[titan] 2025-07-10 00:46:57,732 - root - INFO - step: 50815 loss: 16.5555 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5381 global_avg_mtp_loss: 14.0174 +[titan] 2025-07-10 00:46:57,732 - root - INFO - lr: 1.6365e-04 gnorm: 0.85 [11:11:57<10:50:24] +[titan] 2025-07-10 00:47:01,629 - root - INFO - step: 50820 loss: 16.7519 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.1803 +[titan] 2025-07-10 00:47:01,630 - root - INFO - lr: 1.6363e-04 gnorm: 0.87 [11:12:01<10:50:20] +[titan] 2025-07-10 00:47:05,538 - root - INFO - step: 50825 loss: 16.2369 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.7450 +[titan] 2025-07-10 00:47:05,538 - root - INFO - lr: 1.6361e-04 gnorm: 0.87 [11:12:05<10:50:16] +[titan] 2025-07-10 00:47:09,454 - root - INFO - step: 50830 loss: 16.4053 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.8793 +[titan] 2025-07-10 00:47:09,454 - root - INFO - lr: 1.6358e-04 gnorm: 0.80 [11:12:09<10:50:12] +[titan] 2025-07-10 00:47:13,408 - root - INFO - step: 50835 loss: 16.6459 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0838 +[titan] 2025-07-10 00:47:13,408 - root - INFO - lr: 1.6356e-04 gnorm: 0.85 [11:12:13<10:50:08] +[titan] 2025-07-10 00:47:17,320 - root - INFO - step: 50840 loss: 16.8741 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.6207 global_avg_mtp_loss: 14.2534 +[titan] 2025-07-10 00:47:17,320 - root - INFO - lr: 1.6354e-04 gnorm: 0.82 [11:12:17<10:50:04] +[titan] 2025-07-10 00:47:21,253 - root - INFO - step: 50845 loss: 16.5743 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0223 +[titan] 2025-07-10 00:47:21,254 - root - INFO - lr: 1.6352e-04 gnorm: 0.78 [11:12:21<10:50:00] +[titan] 2025-07-10 00:47:24,370 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:47:25,154 - root - INFO - step: 50850 loss: 16.5410 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 13.9976 +[titan] 2025-07-10 00:47:25,154 - root - INFO - lr: 1.6350e-04 gnorm: 0.82 [11:12:25<10:49:56] +[titan] 2025-07-10 00:47:29,108 - root - INFO - step: 50855 loss: 16.4548 memory: 44.58GiB(31.99%) tps: 82,881 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.9238 +[titan] 2025-07-10 00:47:29,108 - root - INFO - lr: 1.6348e-04 gnorm: 0.90 [11:12:29<10:49:52] +[titan] 2025-07-10 00:47:33,032 - root - INFO - step: 50860 loss: 16.6104 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5525 global_avg_mtp_loss: 14.0579 +[titan] 2025-07-10 00:47:33,032 - root - INFO - lr: 1.6346e-04 gnorm: 0.81 [11:12:33<10:49:48] +[titan] 2025-07-10 00:47:36,971 - root - INFO - step: 50865 loss: 16.5522 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 14.0155 +[titan] 2025-07-10 00:47:36,971 - root - INFO - lr: 1.6343e-04 gnorm: 0.76 [11:12:36<10:49:44] +[titan] 2025-07-10 00:47:40,887 - root - INFO - step: 50870 loss: 16.5291 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 13.9715 +[titan] 2025-07-10 00:47:40,888 - root - INFO - lr: 1.6341e-04 gnorm: 0.79 [11:12:40<10:49:40] +[titan] 2025-07-10 00:47:44,809 - root - INFO - step: 50875 loss: 16.6578 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5626 global_avg_mtp_loss: 14.0953 +[titan] 2025-07-10 00:47:44,809 - root - INFO - lr: 1.6339e-04 gnorm: 0.81 [11:12:44<10:49:36] +[titan] 2025-07-10 00:47:48,721 - root - INFO - step: 50880 loss: 16.6127 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5561 global_avg_mtp_loss: 14.0567 +[titan] 2025-07-10 00:47:48,721 - root - INFO - lr: 1.6337e-04 gnorm: 0.85 [11:12:48<10:49:32] +[titan] 2025-07-10 00:47:52,637 - root - INFO - step: 50885 loss: 16.8481 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.2370 +[titan] 2025-07-10 00:47:52,637 - root - INFO - lr: 1.6335e-04 gnorm: 0.97 [11:12:52<10:49:28] +[titan] 2025-07-10 00:47:56,550 - root - INFO - step: 50890 loss: 16.4903 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9529 +[titan] 2025-07-10 00:47:56,551 - root - INFO - lr: 1.6333e-04 gnorm: 0.83 [11:12:56<10:49:24] +[titan] 2025-07-10 00:48:00,462 - root - INFO - step: 50895 loss: 16.2813 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.7866 +[titan] 2025-07-10 00:48:00,463 - root - INFO - lr: 1.6331e-04 gnorm: 0.84 [11:13:00<10:49:20] +[titan] 2025-07-10 00:48:03,578 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:48:04,366 - root - INFO - step: 50900 loss: 16.3840 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 00:48:04,367 - root - INFO - lr: 1.6328e-04 gnorm: 0.82 [11:13:04<10:49:16] +[titan] 2025-07-10 00:48:08,298 - root - INFO - step: 50905 loss: 16.5228 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9789 +[titan] 2025-07-10 00:48:08,298 - root - INFO - lr: 1.6326e-04 gnorm: 0.89 [11:13:08<10:49:12] +[titan] 2025-07-10 00:48:12,254 - root - INFO - step: 50910 loss: 16.3030 memory: 44.58GiB(31.99%) tps: 82,836 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.8102 +[titan] 2025-07-10 00:48:12,254 - root - INFO - lr: 1.6324e-04 gnorm: 0.77 [11:13:12<10:49:08] +[titan] 2025-07-10 00:48:16,155 - root - INFO - step: 50915 loss: 16.3912 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8709 +[titan] 2025-07-10 00:48:16,156 - root - INFO - lr: 1.6322e-04 gnorm: 0.84 [11:13:16<10:49:04] +[titan] 2025-07-10 00:48:20,050 - root - INFO - step: 50920 loss: 16.4859 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9512 +[titan] 2025-07-10 00:48:20,050 - root - INFO - lr: 1.6320e-04 gnorm: 0.80 [11:13:20<10:49:00] +[titan] 2025-07-10 00:48:23,949 - root - INFO - step: 50925 loss: 16.4225 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.8906 +[titan] 2025-07-10 00:48:23,950 - root - INFO - lr: 1.6318e-04 gnorm: 0.80 [11:13:23<10:48:56] +[titan] 2025-07-10 00:48:27,850 - root - INFO - step: 50930 loss: 16.2831 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.7778 +[titan] 2025-07-10 00:48:27,851 - root - INFO - lr: 1.6316e-04 gnorm: 0.75 [11:13:27<10:48:52] +[titan] 2025-07-10 00:48:31,745 - root - INFO - step: 50935 loss: 16.5444 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 13.9912 +[titan] 2025-07-10 00:48:31,746 - root - INFO - lr: 1.6313e-04 gnorm: 0.80 [11:13:31<10:48:48] +[titan] 2025-07-10 00:48:35,666 - root - INFO - step: 50940 loss: 16.2319 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.7279 +[titan] 2025-07-10 00:48:35,666 - root - INFO - lr: 1.6311e-04 gnorm: 0.94 [11:13:35<10:48:44] +[titan] 2025-07-10 00:48:39,557 - root - INFO - step: 50945 loss: 16.3665 memory: 44.58GiB(31.99%) tps: 84,215 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 00:48:39,557 - root - INFO - lr: 1.6309e-04 gnorm: 0.82 [11:13:39<10:48:40] +[titan] 2025-07-10 00:48:42,699 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:48:43,490 - root - INFO - step: 50950 loss: 16.3402 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.8206 +[titan] 2025-07-10 00:48:43,490 - root - INFO - lr: 1.6307e-04 gnorm: 0.92 [11:13:43<10:48:36] +[titan] 2025-07-10 00:48:47,418 - root - INFO - step: 50955 loss: 16.7266 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5879 global_avg_mtp_loss: 14.1387 +[titan] 2025-07-10 00:48:47,419 - root - INFO - lr: 1.6305e-04 gnorm: 0.77 [11:13:47<10:48:31] +[titan] 2025-07-10 00:48:51,333 - root - INFO - step: 50960 loss: 16.3256 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8071 +[titan] 2025-07-10 00:48:51,333 - root - INFO - lr: 1.6303e-04 gnorm: 0.82 [11:13:51<10:48:27] +[titan] 2025-07-10 00:48:55,272 - root - INFO - step: 50965 loss: 16.4464 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9075 +[titan] 2025-07-10 00:48:55,272 - root - INFO - lr: 1.6301e-04 gnorm: 0.84 [11:13:55<10:48:23] +[titan] 2025-07-10 00:48:59,198 - root - INFO - step: 50970 loss: 16.3884 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8677 +[titan] 2025-07-10 00:48:59,198 - root - INFO - lr: 1.6298e-04 gnorm: 0.82 [11:13:59<10:48:19] +[titan] 2025-07-10 00:49:03,109 - root - INFO - step: 50975 loss: 16.4743 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.9561 +[titan] 2025-07-10 00:49:03,109 - root - INFO - lr: 1.6296e-04 gnorm: 0.83 [11:14:03<10:48:15] +[titan] 2025-07-10 00:49:07,004 - root - INFO - step: 50980 loss: 16.4109 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.8841 +[titan] 2025-07-10 00:49:07,004 - root - INFO - lr: 1.6294e-04 gnorm: 0.80 [11:14:06<10:48:11] +[titan] 2025-07-10 00:49:10,910 - root - INFO - step: 50985 loss: 16.5441 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9935 +[titan] 2025-07-10 00:49:10,910 - root - INFO - lr: 1.6292e-04 gnorm: 0.86 [11:14:10<10:48:07] +[titan] 2025-07-10 00:49:14,850 - root - INFO - step: 50990 loss: 16.5285 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9834 +[titan] 2025-07-10 00:49:14,851 - root - INFO - lr: 1.6290e-04 gnorm: 0.80 [11:14:14<10:48:03] +[titan] 2025-07-10 00:49:18,762 - root - INFO - step: 50995 loss: 16.5335 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9887 +[titan] 2025-07-10 00:49:18,762 - root - INFO - lr: 1.6288e-04 gnorm: 0.76 [11:14:18<10:47:59] +[titan] 2025-07-10 00:49:21,890 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:49:22,691 - root - INFO - step: 51000 loss: 16.5345 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.9884 +[titan] 2025-07-10 00:49:22,691 - root - INFO - lr: 1.6286e-04 gnorm: 0.80 [11:14:22<10:47:55] +[titan] 2025-07-10 00:49:26,601 - root - INFO - step: 51005 loss: 16.4689 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 13.9201 +[titan] 2025-07-10 00:49:26,602 - root - INFO - lr: 1.6283e-04 gnorm: 0.84 [11:14:26<10:47:51] +[titan] 2025-07-10 00:49:30,521 - root - INFO - step: 51010 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0558 +[titan] 2025-07-10 00:49:30,521 - root - INFO - lr: 1.6281e-04 gnorm: 0.81 [11:14:30<10:47:47] +[titan] 2025-07-10 00:49:34,435 - root - INFO - step: 51015 loss: 16.3778 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8609 +[titan] 2025-07-10 00:49:34,435 - root - INFO - lr: 1.6279e-04 gnorm: 0.86 [11:14:34<10:47:43] +[titan] 2025-07-10 00:49:38,393 - root - INFO - step: 51020 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 82,780 tflops: 285.69 mfu: 28.89% global_avg_ntp_loss: 2.5149 global_avg_mtp_loss: 13.8844 +[titan] 2025-07-10 00:49:38,394 - root - INFO - lr: 1.6277e-04 gnorm: 0.85 [11:14:38<10:47:39] +[titan] 2025-07-10 00:49:42,334 - root - INFO - step: 51025 loss: 16.4853 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9556 +[titan] 2025-07-10 00:49:42,335 - root - INFO - lr: 1.6275e-04 gnorm: 0.81 [11:14:42<10:47:35] +[titan] 2025-07-10 00:49:46,268 - root - INFO - step: 51030 loss: 16.6759 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5724 global_avg_mtp_loss: 14.1035 +[titan] 2025-07-10 00:49:46,268 - root - INFO - lr: 1.6273e-04 gnorm: 0.79 [11:14:46<10:47:31] +[titan] 2025-07-10 00:49:50,180 - root - INFO - step: 51035 loss: 16.3807 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8613 +[titan] 2025-07-10 00:49:50,181 - root - INFO - lr: 1.6271e-04 gnorm: 0.87 [11:14:50<10:47:27] +[titan] 2025-07-10 00:49:54,098 - root - INFO - step: 51040 loss: 16.1033 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.6179 +[titan] 2025-07-10 00:49:54,098 - root - INFO - lr: 1.6268e-04 gnorm: 0.84 [11:14:54<10:47:23] +[titan] 2025-07-10 00:49:58,010 - root - INFO - step: 51045 loss: 16.5593 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0000 +[titan] 2025-07-10 00:49:58,010 - root - INFO - lr: 1.6266e-04 gnorm: 0.88 [11:14:57<10:47:19] +[titan] 2025-07-10 00:50:01,138 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:50:01,928 - root - INFO - step: 51050 loss: 16.6681 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0985 +[titan] 2025-07-10 00:50:01,928 - root - INFO - lr: 1.6264e-04 gnorm: 0.79 [11:15:01<10:47:15] +[titan] 2025-07-10 00:50:05,893 - root - INFO - step: 51055 loss: 16.6482 memory: 44.58GiB(31.99%) tps: 82,644 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 14.0985 +[titan] 2025-07-10 00:50:05,893 - root - INFO - lr: 1.6262e-04 gnorm: 0.81 [11:15:05<10:47:11] +[titan] 2025-07-10 00:50:09,802 - root - INFO - step: 51060 loss: 16.7461 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5977 global_avg_mtp_loss: 14.1484 +[titan] 2025-07-10 00:50:09,803 - root - INFO - lr: 1.6260e-04 gnorm: 0.79 [11:15:09<10:47:07] +[titan] 2025-07-10 00:50:13,726 - root - INFO - step: 51065 loss: 16.5588 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 14.0154 +[titan] 2025-07-10 00:50:13,726 - root - INFO - lr: 1.6258e-04 gnorm: 0.80 [11:15:13<10:47:03] +[titan] 2025-07-10 00:50:17,645 - root - INFO - step: 51070 loss: 16.3335 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8201 +[titan] 2025-07-10 00:50:17,645 - root - INFO - lr: 1.6256e-04 gnorm: 0.80 [11:15:17<10:46:59] +[titan] 2025-07-10 00:50:21,553 - root - INFO - step: 51075 loss: 16.2108 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7229 +[titan] 2025-07-10 00:50:21,553 - root - INFO - lr: 1.6253e-04 gnorm: 0.80 [11:15:21<10:46:55] +[titan] 2025-07-10 00:50:25,488 - root - INFO - step: 51080 loss: 16.2519 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7620 +[titan] 2025-07-10 00:50:25,488 - root - INFO - lr: 1.6251e-04 gnorm: 0.82 [11:15:25<10:46:51] +[titan] 2025-07-10 00:50:29,446 - root - INFO - step: 51085 loss: 16.5830 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 14.0264 +[titan] 2025-07-10 00:50:29,446 - root - INFO - lr: 1.6249e-04 gnorm: 0.80 [11:15:29<10:46:47] +[titan] 2025-07-10 00:50:33,373 - root - INFO - step: 51090 loss: 16.3822 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 00:50:33,373 - root - INFO - lr: 1.6247e-04 gnorm: 0.77 [11:15:33<10:46:43] +[titan] 2025-07-10 00:50:37,294 - root - INFO - step: 51095 loss: 16.3679 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8642 +[titan] 2025-07-10 00:50:37,294 - root - INFO - lr: 1.6245e-04 gnorm: 0.89 [11:15:37<10:46:39] +[titan] 2025-07-10 00:50:40,410 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:50:41,198 - root - INFO - step: 51100 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.8686 +[titan] 2025-07-10 00:50:41,198 - root - INFO - lr: 1.6243e-04 gnorm: 0.82 [11:15:41<10:46:35] +[titan] 2025-07-10 00:50:45,130 - root - INFO - step: 51105 loss: 16.3653 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8548 +[titan] 2025-07-10 00:50:45,130 - root - INFO - lr: 1.6241e-04 gnorm: 0.82 [11:15:45<10:46:31] +[titan] 2025-07-10 00:50:49,053 - root - INFO - step: 51110 loss: 16.5288 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9814 +[titan] 2025-07-10 00:50:49,053 - root - INFO - lr: 1.6238e-04 gnorm: 0.84 [11:15:48<10:46:27] +[titan] 2025-07-10 00:50:52,968 - root - INFO - step: 51115 loss: 16.1260 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.6473 +[titan] 2025-07-10 00:50:52,969 - root - INFO - lr: 1.6236e-04 gnorm: 0.82 [11:15:52<10:46:23] +[titan] 2025-07-10 00:50:56,909 - root - INFO - step: 51120 loss: 16.5402 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9955 +[titan] 2025-07-10 00:50:56,909 - root - INFO - lr: 1.6234e-04 gnorm: 0.83 [11:15:56<10:46:19] +[titan] 2025-07-10 00:51:00,808 - root - INFO - step: 51125 loss: 16.4599 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.9435 +[titan] 2025-07-10 00:51:00,809 - root - INFO - lr: 1.6232e-04 gnorm: 0.84 [11:16:00<10:46:15] +[titan] 2025-07-10 00:51:04,757 - root - INFO - step: 51130 loss: 16.1499 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.6707 +[titan] 2025-07-10 00:51:04,757 - root - INFO - lr: 1.6230e-04 gnorm: 0.83 [11:16:04<10:46:11] +[titan] 2025-07-10 00:51:08,690 - root - INFO - step: 51135 loss: 16.2738 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.7773 +[titan] 2025-07-10 00:51:08,690 - root - INFO - lr: 1.6228e-04 gnorm: 0.87 [11:16:08<10:46:07] +[titan] 2025-07-10 00:51:12,620 - root - INFO - step: 51140 loss: 16.3468 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 00:51:12,620 - root - INFO - lr: 1.6226e-04 gnorm: 0.83 [11:16:12<10:46:03] +[titan] 2025-07-10 00:51:16,531 - root - INFO - step: 51145 loss: 16.6034 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 14.0507 +[titan] 2025-07-10 00:51:16,532 - root - INFO - lr: 1.6223e-04 gnorm: 0.82 [11:16:16<10:45:59] +[titan] 2025-07-10 00:51:19,666 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:51:20,459 - root - INFO - step: 51150 loss: 16.6679 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5635 global_avg_mtp_loss: 14.1044 +[titan] 2025-07-10 00:51:20,460 - root - INFO - lr: 1.6221e-04 gnorm: 0.78 [11:16:20<10:45:55] +[titan] 2025-07-10 00:51:24,359 - root - INFO - step: 51155 loss: 16.2761 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7800 +[titan] 2025-07-10 00:51:24,359 - root - INFO - lr: 1.6219e-04 gnorm: 0.79 [11:16:24<10:45:51] +[titan] 2025-07-10 00:51:28,290 - root - INFO - step: 51160 loss: 16.8457 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.2433 +[titan] 2025-07-10 00:51:28,290 - root - INFO - lr: 1.6217e-04 gnorm: 0.78 [11:16:28<10:45:47] +[titan] 2025-07-10 00:51:32,192 - root - INFO - step: 51165 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.8471 +[titan] 2025-07-10 00:51:32,192 - root - INFO - lr: 1.6215e-04 gnorm: 0.80 [11:16:32<10:45:43] +[titan] 2025-07-10 00:51:36,100 - root - INFO - step: 51170 loss: 16.8445 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.6252 global_avg_mtp_loss: 14.2193 +[titan] 2025-07-10 00:51:36,100 - root - INFO - lr: 1.6213e-04 gnorm: 0.94 [11:16:36<10:45:39] +[titan] 2025-07-10 00:51:40,003 - root - INFO - step: 51175 loss: 16.3369 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 00:51:40,003 - root - INFO - lr: 1.6211e-04 gnorm: 0.83 [11:16:39<10:45:35] +[titan] 2025-07-10 00:51:43,915 - root - INFO - step: 51180 loss: 16.5632 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0085 +[titan] 2025-07-10 00:51:43,915 - root - INFO - lr: 1.6209e-04 gnorm: 0.85 [11:16:43<10:45:31] +[titan] 2025-07-10 00:51:47,843 - root - INFO - step: 51185 loss: 16.3684 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8524 +[titan] 2025-07-10 00:51:47,844 - root - INFO - lr: 1.6206e-04 gnorm: 0.79 [11:16:47<10:45:27] +[titan] 2025-07-10 00:51:51,852 - root - INFO - step: 51190 loss: 16.2222 memory: 44.58GiB(31.99%) tps: 81,756 tflops: 282.15 mfu: 28.53% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.7312 +[titan] 2025-07-10 00:51:51,852 - root - INFO - lr: 1.6204e-04 gnorm: 0.79 [11:16:51<10:45:23] +[titan] 2025-07-10 00:51:55,782 - root - INFO - step: 51195 loss: 16.4934 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.9493 +[titan] 2025-07-10 00:51:55,782 - root - INFO - lr: 1.6202e-04 gnorm: 0.81 [11:16:55<10:45:19] +[titan] 2025-07-10 00:51:58,981 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:51:59,777 - root - INFO - step: 51200 loss: 16.6137 memory: 44.58GiB(31.99%) tps: 82,045 tflops: 283.15 mfu: 28.63% global_avg_ntp_loss: 2.5559 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-10 00:51:59,777 - root - INFO - lr: 1.6200e-04 gnorm: 0.85 [11:16:59<10:45:15] +[titan] 2025-07-10 00:51:59,931 - root - INFO - Dumping profiler traces at step 51200 +[titan] 2025-07-10 00:51:59,965 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:52:03,903 - root - INFO - step: 51205 loss: 16.3772 memory: 44.58GiB(31.99%) tps: 79,409 tflops: 274.05 mfu: 27.71% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.8308 +[titan] 2025-07-10 00:52:03,904 - root - INFO - lr: 1.6198e-04 gnorm: 0.85 [11:17:03<10:45:11] +[titan] 2025-07-10 00:52:07,819 - root - INFO - step: 51210 loss: 16.3249 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 00:52:07,819 - root - INFO - lr: 1.6196e-04 gnorm: 0.77 [11:17:07<10:45:07] +[titan] 2025-07-10 00:52:11,753 - root - INFO - step: 51215 loss: 16.4573 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9270 +[titan] 2025-07-10 00:52:11,753 - root - INFO - lr: 1.6194e-04 gnorm: 0.78 [11:17:11<10:45:03] +[titan] 2025-07-10 00:52:15,651 - root - INFO - step: 51220 loss: 16.6021 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0387 +[titan] 2025-07-10 00:52:15,652 - root - INFO - lr: 1.6191e-04 gnorm: 0.81 [11:17:15<10:44:59] +[titan] 2025-07-10 00:52:19,559 - root - INFO - step: 51225 loss: 16.3907 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.8792 +[titan] 2025-07-10 00:52:19,559 - root - INFO - lr: 1.6189e-04 gnorm: 0.83 [11:17:19<10:44:55] +[titan] 2025-07-10 00:52:23,483 - root - INFO - step: 51230 loss: 16.5482 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 13.9954 +[titan] 2025-07-10 00:52:23,483 - root - INFO - lr: 1.6187e-04 gnorm: 0.77 [11:17:23<10:44:51] +[titan] 2025-07-10 00:52:27,392 - root - INFO - step: 51235 loss: 16.4858 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9445 +[titan] 2025-07-10 00:52:27,392 - root - INFO - lr: 1.6185e-04 gnorm: 0.80 [11:17:27<10:44:47] +[titan] 2025-07-10 00:52:31,318 - root - INFO - step: 51240 loss: 16.6139 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0569 +[titan] 2025-07-10 00:52:31,319 - root - INFO - lr: 1.6183e-04 gnorm: 0.75 [11:17:31<10:44:43] +[titan] 2025-07-10 00:52:35,240 - root - INFO - step: 51245 loss: 16.4569 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9244 +[titan] 2025-07-10 00:52:35,240 - root - INFO - lr: 1.6181e-04 gnorm: 0.80 [11:17:35<10:44:39] +[titan] 2025-07-10 00:52:38,368 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:52:39,162 - root - INFO - step: 51250 loss: 16.4899 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 13.9419 +[titan] 2025-07-10 00:52:39,162 - root - INFO - lr: 1.6179e-04 gnorm: 0.80 [11:17:39<10:44:35] +[titan] 2025-07-10 00:52:43,116 - root - INFO - step: 51255 loss: 16.3930 memory: 44.58GiB(31.99%) tps: 82,865 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8760 +[titan] 2025-07-10 00:52:43,117 - root - INFO - lr: 1.6176e-04 gnorm: 0.80 [11:17:43<10:44:31] +[titan] 2025-07-10 00:52:47,093 - root - INFO - step: 51260 loss: 16.5834 memory: 44.58GiB(31.99%) tps: 82,411 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-10 00:52:47,093 - root - INFO - lr: 1.6174e-04 gnorm: 0.79 [11:17:47<10:44:27] +[titan] 2025-07-10 00:52:51,011 - root - INFO - step: 51265 loss: 16.4548 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 13.9145 +[titan] 2025-07-10 00:52:51,011 - root - INFO - lr: 1.6172e-04 gnorm: 0.80 [11:17:50<10:44:23] +[titan] 2025-07-10 00:52:54,928 - root - INFO - step: 51270 loss: 16.6804 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5745 global_avg_mtp_loss: 14.1059 +[titan] 2025-07-10 00:52:54,928 - root - INFO - lr: 1.6170e-04 gnorm: 0.83 [11:17:54<10:44:19] +[titan] 2025-07-10 00:52:58,826 - root - INFO - step: 51275 loss: 16.4583 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.9251 +[titan] 2025-07-10 00:52:58,826 - root - INFO - lr: 1.6168e-04 gnorm: 0.87 [11:17:58<10:44:15] +[titan] 2025-07-10 00:53:02,753 - root - INFO - step: 51280 loss: 16.6623 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.0918 +[titan] 2025-07-10 00:53:02,753 - root - INFO - lr: 1.6166e-04 gnorm: 0.83 [11:18:02<10:44:11] +[titan] 2025-07-10 00:53:06,692 - root - INFO - step: 51285 loss: 16.6152 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0615 +[titan] 2025-07-10 00:53:06,692 - root - INFO - lr: 1.6164e-04 gnorm: 0.83 [11:18:06<10:44:07] +[titan] 2025-07-10 00:53:10,600 - root - INFO - step: 51290 loss: 16.4470 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-10 00:53:10,600 - root - INFO - lr: 1.6161e-04 gnorm: 0.83 [11:18:10<10:44:03] +[titan] 2025-07-10 00:53:14,517 - root - INFO - step: 51295 loss: 16.2461 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7511 +[titan] 2025-07-10 00:53:14,517 - root - INFO - lr: 1.6159e-04 gnorm: 0.82 [11:18:14<10:43:59] +[titan] 2025-07-10 00:53:17,641 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:53:18,427 - root - INFO - step: 51300 loss: 17.0353 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.6442 global_avg_mtp_loss: 14.3911 +[titan] 2025-07-10 00:53:18,427 - root - INFO - lr: 1.6157e-04 gnorm: 0.89 [11:18:18<10:43:55] +[titan] 2025-07-10 00:53:22,351 - root - INFO - step: 51305 loss: 16.5393 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 14.0057 +[titan] 2025-07-10 00:53:22,351 - root - INFO - lr: 1.6155e-04 gnorm: 0.81 [11:18:22<10:43:51] +[titan] 2025-07-10 00:53:26,277 - root - INFO - step: 51310 loss: 16.6841 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.1209 +[titan] 2025-07-10 00:53:26,277 - root - INFO - lr: 1.6153e-04 gnorm: 0.83 [11:18:26<10:43:47] +[titan] 2025-07-10 00:53:30,231 - root - INFO - step: 51315 loss: 16.4903 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5392 global_avg_mtp_loss: 13.9511 +[titan] 2025-07-10 00:53:30,231 - root - INFO - lr: 1.6151e-04 gnorm: 0.90 [11:18:30<10:43:43] +[titan] 2025-07-10 00:53:34,172 - root - INFO - step: 51320 loss: 16.3269 memory: 44.58GiB(31.99%) tps: 83,156 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8187 +[titan] 2025-07-10 00:53:34,172 - root - INFO - lr: 1.6149e-04 gnorm: 0.77 [11:18:34<10:43:39] +[titan] 2025-07-10 00:53:38,111 - root - INFO - step: 51325 loss: 16.5613 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 14.0197 +[titan] 2025-07-10 00:53:38,111 - root - INFO - lr: 1.6146e-04 gnorm: 0.77 [11:18:38<10:43:35] +[titan] 2025-07-10 00:53:42,039 - root - INFO - step: 51330 loss: 16.6293 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.0545 +[titan] 2025-07-10 00:53:42,039 - root - INFO - lr: 1.6144e-04 gnorm: 0.83 [11:18:41<10:43:31] +[titan] 2025-07-10 00:53:45,971 - root - INFO - step: 51335 loss: 16.3573 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8539 +[titan] 2025-07-10 00:53:45,971 - root - INFO - lr: 1.6142e-04 gnorm: 0.81 [11:18:45<10:43:27] +[titan] 2025-07-10 00:53:49,924 - root - INFO - step: 51340 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 82,907 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-10 00:53:49,924 - root - INFO - lr: 1.6140e-04 gnorm: 0.83 [11:18:49<10:43:23] +[titan] 2025-07-10 00:53:53,840 - root - INFO - step: 51345 loss: 16.5871 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 14.0459 +[titan] 2025-07-10 00:53:53,840 - root - INFO - lr: 1.6138e-04 gnorm: 0.79 [11:18:53<10:43:19] +[titan] 2025-07-10 00:53:57,028 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:53:57,834 - root - INFO - step: 51350 loss: 16.3553 memory: 44.58GiB(31.99%) tps: 82,046 tflops: 283.15 mfu: 28.63% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8470 +[titan] 2025-07-10 00:53:57,834 - root - INFO - lr: 1.6136e-04 gnorm: 0.81 [11:18:57<10:43:15] +[titan] 2025-07-10 00:54:01,763 - root - INFO - step: 51355 loss: 16.6047 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0503 +[titan] 2025-07-10 00:54:01,763 - root - INFO - lr: 1.6134e-04 gnorm: 0.81 [11:19:01<10:43:11] +[titan] 2025-07-10 00:54:05,704 - root - INFO - step: 51360 loss: 16.7237 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1322 +[titan] 2025-07-10 00:54:05,705 - root - INFO - lr: 1.6131e-04 gnorm: 0.87 [11:19:05<10:43:07] +[titan] 2025-07-10 00:54:09,638 - root - INFO - step: 51365 loss: 16.7109 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.1398 +[titan] 2025-07-10 00:54:09,638 - root - INFO - lr: 1.6129e-04 gnorm: 0.84 [11:19:09<10:43:03] +[titan] 2025-07-10 00:54:13,638 - root - INFO - step: 51370 loss: 16.3067 memory: 44.58GiB(31.99%) tps: 81,915 tflops: 282.70 mfu: 28.58% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.7893 +[titan] 2025-07-10 00:54:13,638 - root - INFO - lr: 1.6127e-04 gnorm: 0.81 [11:19:13<10:42:59] +[titan] 2025-07-10 00:54:17,629 - root - INFO - step: 51375 loss: 16.6180 memory: 44.58GiB(31.99%) tps: 82,114 tflops: 283.39 mfu: 28.65% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0640 +[titan] 2025-07-10 00:54:17,629 - root - INFO - lr: 1.6125e-04 gnorm: 0.80 [11:19:17<10:42:55] +[titan] 2025-07-10 00:54:21,567 - root - INFO - step: 51380 loss: 16.4134 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8944 +[titan] 2025-07-10 00:54:21,567 - root - INFO - lr: 1.6123e-04 gnorm: 0.81 [11:19:21<10:42:51] +[titan] 2025-07-10 00:54:25,543 - root - INFO - step: 51385 loss: 16.5497 memory: 44.58GiB(31.99%) tps: 82,424 tflops: 284.46 mfu: 28.76% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 14.0062 +[titan] 2025-07-10 00:54:25,543 - root - INFO - lr: 1.6121e-04 gnorm: 0.81 [11:19:25<10:42:47] +[titan] 2025-07-10 00:54:29,506 - root - INFO - step: 51390 loss: 16.6426 memory: 44.58GiB(31.99%) tps: 82,691 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0692 +[titan] 2025-07-10 00:54:29,506 - root - INFO - lr: 1.6119e-04 gnorm: 0.79 [11:19:29<10:42:43] +[titan] 2025-07-10 00:54:33,433 - root - INFO - step: 51395 loss: 16.5450 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 13.9950 +[titan] 2025-07-10 00:54:33,434 - root - INFO - lr: 1.6116e-04 gnorm: 0.81 [11:19:33<10:42:39] +[titan] 2025-07-10 00:54:36,540 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:54:37,329 - root - INFO - step: 51400 loss: 16.2765 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.7782 +[titan] 2025-07-10 00:54:37,329 - root - INFO - lr: 1.6114e-04 gnorm: 0.97 [11:19:37<10:42:35] +[titan] 2025-07-10 00:54:41,252 - root - INFO - step: 51405 loss: 16.5316 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 13.9833 +[titan] 2025-07-10 00:54:41,253 - root - INFO - lr: 1.6112e-04 gnorm: 0.81 [11:19:41<10:42:31] +[titan] 2025-07-10 00:54:45,208 - root - INFO - step: 51410 loss: 16.5204 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5509 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 00:54:45,208 - root - INFO - lr: 1.6110e-04 gnorm: 0.79 [11:19:45<10:42:27] +[titan] 2025-07-10 00:54:49,131 - root - INFO - step: 51415 loss: 16.3657 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 00:54:49,131 - root - INFO - lr: 1.6108e-04 gnorm: 0.84 [11:19:49<10:42:23] +[titan] 2025-07-10 00:54:53,049 - root - INFO - step: 51420 loss: 16.6910 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.1325 +[titan] 2025-07-10 00:54:53,049 - root - INFO - lr: 1.6106e-04 gnorm: 0.78 [11:19:52<10:42:19] +[titan] 2025-07-10 00:54:56,977 - root - INFO - step: 51425 loss: 16.4317 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5458 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 00:54:56,978 - root - INFO - lr: 1.6104e-04 gnorm: 0.77 [11:19:56<10:42:15] +[titan] 2025-07-10 00:55:00,887 - root - INFO - step: 51430 loss: 16.2648 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-10 00:55:00,887 - root - INFO - lr: 1.6101e-04 gnorm: 0.83 [11:20:00<10:42:11] +[titan] 2025-07-10 00:55:04,803 - root - INFO - step: 51435 loss: 16.2604 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7769 +[titan] 2025-07-10 00:55:04,803 - root - INFO - lr: 1.6099e-04 gnorm: 0.85 [11:20:04<10:42:07] +[titan] 2025-07-10 00:55:08,721 - root - INFO - step: 51440 loss: 16.4006 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.8753 +[titan] 2025-07-10 00:55:08,721 - root - INFO - lr: 1.6097e-04 gnorm: 0.84 [11:20:08<10:42:03] +[titan] 2025-07-10 00:55:12,649 - root - INFO - step: 51445 loss: 16.4951 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9460 +[titan] 2025-07-10 00:55:12,650 - root - INFO - lr: 1.6095e-04 gnorm: 0.87 [11:20:12<10:41:59] +[titan] 2025-07-10 00:55:15,766 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:55:16,560 - root - INFO - step: 51450 loss: 16.5456 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 13.9985 +[titan] 2025-07-10 00:55:16,560 - root - INFO - lr: 1.6093e-04 gnorm: 0.79 [11:20:16<10:41:55] +[titan] 2025-07-10 00:55:20,488 - root - INFO - step: 51455 loss: 16.2713 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.7803 +[titan] 2025-07-10 00:55:20,488 - root - INFO - lr: 1.6091e-04 gnorm: 0.81 [11:20:20<10:41:51] +[titan] 2025-07-10 00:55:24,394 - root - INFO - step: 51460 loss: 16.6405 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0840 +[titan] 2025-07-10 00:55:24,394 - root - INFO - lr: 1.6089e-04 gnorm: 0.82 [11:20:24<10:41:47] +[titan] 2025-07-10 00:55:28,302 - root - INFO - step: 51465 loss: 16.2364 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 00:55:28,303 - root - INFO - lr: 1.6086e-04 gnorm: 0.82 [11:20:28<10:41:43] +[titan] 2025-07-10 00:55:32,223 - root - INFO - step: 51470 loss: 16.5312 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 13.9628 +[titan] 2025-07-10 00:55:32,224 - root - INFO - lr: 1.6084e-04 gnorm: 0.79 [11:20:32<10:41:39] +[titan] 2025-07-10 00:55:36,154 - root - INFO - step: 51475 loss: 16.5683 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5492 global_avg_mtp_loss: 14.0192 +[titan] 2025-07-10 00:55:36,154 - root - INFO - lr: 1.6082e-04 gnorm: 0.86 [11:20:36<10:41:35] +[titan] 2025-07-10 00:55:40,055 - root - INFO - step: 51480 loss: 16.5025 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 13.9467 +[titan] 2025-07-10 00:55:40,055 - root - INFO - lr: 1.6080e-04 gnorm: 0.87 [11:20:39<10:41:31] +[titan] 2025-07-10 00:55:43,960 - root - INFO - step: 51485 loss: 16.4667 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 13.9243 +[titan] 2025-07-10 00:55:43,960 - root - INFO - lr: 1.6078e-04 gnorm: 0.84 [11:20:43<10:41:27] +[titan] 2025-07-10 00:55:47,918 - root - INFO - step: 51490 loss: 16.5402 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 14.0035 +[titan] 2025-07-10 00:55:47,918 - root - INFO - lr: 1.6076e-04 gnorm: 0.92 [11:20:47<10:41:23] +[titan] 2025-07-10 00:55:51,831 - root - INFO - step: 51495 loss: 16.5606 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 14.0128 +[titan] 2025-07-10 00:55:51,832 - root - INFO - lr: 1.6074e-04 gnorm: 0.89 [11:20:51<10:41:19] +[titan] 2025-07-10 00:55:54,965 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:55:55,779 - root - INFO - step: 51500 loss: 16.4050 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.8736 +[titan] 2025-07-10 00:55:55,780 - root - INFO - lr: 1.6071e-04 gnorm: 0.85 [11:20:55<10:41:15] +[titan] 2025-07-10 00:55:59,708 - root - INFO - step: 51505 loss: 16.7044 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5935 global_avg_mtp_loss: 14.1109 +[titan] 2025-07-10 00:55:59,708 - root - INFO - lr: 1.6069e-04 gnorm: 0.88 [11:20:59<10:41:11] +[titan] 2025-07-10 00:56:03,671 - root - INFO - step: 51510 loss: 16.6763 memory: 44.58GiB(31.99%) tps: 82,687 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.5812 global_avg_mtp_loss: 14.0951 +[titan] 2025-07-10 00:56:03,671 - root - INFO - lr: 1.6067e-04 gnorm: 0.87 [11:21:03<10:41:07] +[titan] 2025-07-10 00:56:07,589 - root - INFO - step: 51515 loss: 16.2829 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.7797 +[titan] 2025-07-10 00:56:07,589 - root - INFO - lr: 1.6065e-04 gnorm: 0.87 [11:21:07<10:41:03] +[titan] 2025-07-10 00:56:11,520 - root - INFO - step: 51520 loss: 16.4474 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9055 +[titan] 2025-07-10 00:56:11,520 - root - INFO - lr: 1.6063e-04 gnorm: 0.85 [11:21:11<10:40:59] +[titan] 2025-07-10 00:56:15,436 - root - INFO - step: 51525 loss: 16.5766 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0180 +[titan] 2025-07-10 00:56:15,437 - root - INFO - lr: 1.6061e-04 gnorm: 0.86 [11:21:15<10:40:55] +[titan] 2025-07-10 00:56:19,358 - root - INFO - step: 51530 loss: 16.2713 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.7665 +[titan] 2025-07-10 00:56:19,358 - root - INFO - lr: 1.6059e-04 gnorm: 0.81 [11:21:19<10:40:51] +[titan] 2025-07-10 00:56:23,271 - root - INFO - step: 51535 loss: 16.6845 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5738 global_avg_mtp_loss: 14.1107 +[titan] 2025-07-10 00:56:23,272 - root - INFO - lr: 1.6056e-04 gnorm: 0.75 [11:21:23<10:40:47] +[titan] 2025-07-10 00:56:27,190 - root - INFO - step: 51540 loss: 16.3742 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-10 00:56:27,190 - root - INFO - lr: 1.6054e-04 gnorm: 0.78 [11:21:27<10:40:43] +[titan] 2025-07-10 00:56:31,090 - root - INFO - step: 51545 loss: 16.5370 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5515 global_avg_mtp_loss: 13.9855 +[titan] 2025-07-10 00:56:31,090 - root - INFO - lr: 1.6052e-04 gnorm: 0.82 [11:21:30<10:40:39] +[titan] 2025-07-10 00:56:34,207 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:56:34,996 - root - INFO - step: 51550 loss: 16.5023 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9660 +[titan] 2025-07-10 00:56:34,996 - root - INFO - lr: 1.6050e-04 gnorm: 0.85 [11:21:34<10:40:35] +[titan] 2025-07-10 00:56:38,913 - root - INFO - step: 51555 loss: 16.4195 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.8967 +[titan] 2025-07-10 00:56:38,913 - root - INFO - lr: 1.6048e-04 gnorm: 0.84 [11:21:38<10:40:31] +[titan] 2025-07-10 00:56:42,856 - root - INFO - step: 51560 loss: 16.4902 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-10 00:56:42,856 - root - INFO - lr: 1.6046e-04 gnorm: 0.80 [11:21:42<10:40:27] +[titan] 2025-07-10 00:56:46,788 - root - INFO - step: 51565 loss: 16.2290 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7361 +[titan] 2025-07-10 00:56:46,789 - root - INFO - lr: 1.6044e-04 gnorm: 0.80 [11:21:46<10:40:23] +[titan] 2025-07-10 00:56:50,699 - root - INFO - step: 51570 loss: 16.4144 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.8860 +[titan] 2025-07-10 00:56:50,700 - root - INFO - lr: 1.6041e-04 gnorm: 0.81 [11:21:50<10:40:19] +[titan] 2025-07-10 00:56:54,646 - root - INFO - step: 51575 loss: 16.6929 memory: 44.58GiB(31.99%) tps: 83,033 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1167 +[titan] 2025-07-10 00:56:54,647 - root - INFO - lr: 1.6039e-04 gnorm: 0.81 [11:21:54<10:40:15] +[titan] 2025-07-10 00:56:58,580 - root - INFO - step: 51580 loss: 16.6237 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0564 +[titan] 2025-07-10 00:56:58,580 - root - INFO - lr: 1.6037e-04 gnorm: 0.81 [11:21:58<10:40:11] +[titan] 2025-07-10 00:57:02,483 - root - INFO - step: 51585 loss: 16.3610 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8490 +[titan] 2025-07-10 00:57:02,483 - root - INFO - lr: 1.6035e-04 gnorm: 0.79 [11:22:02<10:40:07] +[titan] 2025-07-10 00:57:06,411 - root - INFO - step: 51590 loss: 16.7941 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5921 global_avg_mtp_loss: 14.2021 +[titan] 2025-07-10 00:57:06,411 - root - INFO - lr: 1.6033e-04 gnorm: 0.82 [11:22:06<10:40:03] +[titan] 2025-07-10 00:57:10,327 - root - INFO - step: 51595 loss: 16.3710 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8651 +[titan] 2025-07-10 00:57:10,328 - root - INFO - lr: 1.6031e-04 gnorm: 0.83 [11:22:10<10:39:59] +[titan] 2025-07-10 00:57:13,447 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:57:14,239 - root - INFO - step: 51600 loss: 16.5063 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9613 +[titan] 2025-07-10 00:57:14,240 - root - INFO - lr: 1.6029e-04 gnorm: 0.78 [11:22:14<10:39:55] +[titan] 2025-07-10 00:57:18,156 - root - INFO - step: 51605 loss: 16.2749 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.7709 +[titan] 2025-07-10 00:57:18,157 - root - INFO - lr: 1.6027e-04 gnorm: 0.79 [11:22:18<10:39:51] +[titan] 2025-07-10 00:57:22,063 - root - INFO - step: 51610 loss: 16.4337 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.8990 +[titan] 2025-07-10 00:57:22,064 - root - INFO - lr: 1.6024e-04 gnorm: 0.84 [11:22:21<10:39:47] +[titan] 2025-07-10 00:57:25,982 - root - INFO - step: 51615 loss: 16.6271 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0639 +[titan] 2025-07-10 00:57:25,982 - root - INFO - lr: 1.6022e-04 gnorm: 0.84 [11:22:25<10:39:43] +[titan] 2025-07-10 00:57:29,895 - root - INFO - step: 51620 loss: 16.2856 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.7689 +[titan] 2025-07-10 00:57:29,895 - root - INFO - lr: 1.6020e-04 gnorm: 0.83 [11:22:29<10:39:39] +[titan] 2025-07-10 00:57:33,818 - root - INFO - step: 51625 loss: 16.3254 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.8129 +[titan] 2025-07-10 00:57:33,818 - root - INFO - lr: 1.6018e-04 gnorm: 0.81 [11:22:33<10:39:35] +[titan] 2025-07-10 00:57:37,760 - root - INFO - step: 51630 loss: 16.4236 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.8891 +[titan] 2025-07-10 00:57:37,760 - root - INFO - lr: 1.6016e-04 gnorm: 0.80 [11:22:37<10:39:31] +[titan] 2025-07-10 00:57:41,676 - root - INFO - step: 51635 loss: 16.5068 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9589 +[titan] 2025-07-10 00:57:41,676 - root - INFO - lr: 1.6014e-04 gnorm: 0.77 [11:22:41<10:39:27] +[titan] 2025-07-10 00:57:45,614 - root - INFO - step: 51640 loss: 16.2903 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7943 +[titan] 2025-07-10 00:57:45,614 - root - INFO - lr: 1.6012e-04 gnorm: 0.82 [11:22:45<10:39:23] +[titan] 2025-07-10 00:57:49,522 - root - INFO - step: 51645 loss: 16.2444 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7526 +[titan] 2025-07-10 00:57:49,522 - root - INFO - lr: 1.6009e-04 gnorm: 0.82 [11:22:49<10:39:19] +[titan] 2025-07-10 00:57:52,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:57:53,500 - root - INFO - step: 51650 loss: 16.1340 memory: 44.58GiB(31.99%) tps: 82,377 tflops: 284.30 mfu: 28.75% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.6615 +[titan] 2025-07-10 00:57:53,500 - root - INFO - lr: 1.6007e-04 gnorm: 0.84 [11:22:53<10:39:15] +[titan] 2025-07-10 00:57:57,427 - root - INFO - step: 51655 loss: 16.8427 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5998 global_avg_mtp_loss: 14.2429 +[titan] 2025-07-10 00:57:57,428 - root - INFO - lr: 1.6005e-04 gnorm: 0.84 [11:22:57<10:39:11] +[titan] 2025-07-10 00:58:01,367 - root - INFO - step: 51660 loss: 16.4999 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.02% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9572 +[titan] 2025-07-10 00:58:01,368 - root - INFO - lr: 1.6003e-04 gnorm: 0.83 [11:23:01<10:39:07] +[titan] 2025-07-10 00:58:05,326 - root - INFO - step: 51665 loss: 16.6140 memory: 44.58GiB(31.99%) tps: 82,777 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 14.0582 +[titan] 2025-07-10 00:58:05,327 - root - INFO - lr: 1.6001e-04 gnorm: 0.79 [11:23:05<10:39:03] +[titan] 2025-07-10 00:58:09,238 - root - INFO - step: 51670 loss: 16.4555 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9145 +[titan] 2025-07-10 00:58:09,238 - root - INFO - lr: 1.5999e-04 gnorm: 0.82 [11:23:09<10:38:59] +[titan] 2025-07-10 00:58:13,145 - root - INFO - step: 51675 loss: 16.3720 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8565 +[titan] 2025-07-10 00:58:13,145 - root - INFO - lr: 1.5997e-04 gnorm: 0.80 [11:23:13<10:38:55] +[titan] 2025-07-10 00:58:17,052 - root - INFO - step: 51680 loss: 16.6505 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0790 +[titan] 2025-07-10 00:58:17,052 - root - INFO - lr: 1.5994e-04 gnorm: 0.79 [11:23:16<10:38:51] +[titan] 2025-07-10 00:58:20,959 - root - INFO - step: 51685 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9319 +[titan] 2025-07-10 00:58:20,960 - root - INFO - lr: 1.5992e-04 gnorm: 0.77 [11:23:20<10:38:47] +[titan] 2025-07-10 00:58:24,868 - root - INFO - step: 51690 loss: 16.4530 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9261 +[titan] 2025-07-10 00:58:24,868 - root - INFO - lr: 1.5990e-04 gnorm: 0.78 [11:23:24<10:38:43] +[titan] 2025-07-10 00:58:28,774 - root - INFO - step: 51695 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9112 +[titan] 2025-07-10 00:58:28,775 - root - INFO - lr: 1.5988e-04 gnorm: 0.79 [11:23:28<10:38:39] +[titan] 2025-07-10 00:58:31,898 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:58:32,693 - root - INFO - step: 51700 loss: 16.5609 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0138 +[titan] 2025-07-10 00:58:32,693 - root - INFO - lr: 1.5986e-04 gnorm: 0.83 [11:23:32<10:38:35] +[titan] 2025-07-10 00:58:36,653 - root - INFO - step: 51705 loss: 16.6032 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0499 +[titan] 2025-07-10 00:58:36,653 - root - INFO - lr: 1.5984e-04 gnorm: 0.76 [11:23:36<10:38:31] +[titan] 2025-07-10 00:58:40,638 - root - INFO - step: 51710 loss: 16.1441 memory: 44.58GiB(31.99%) tps: 82,228 tflops: 283.78 mfu: 28.69% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.6637 +[titan] 2025-07-10 00:58:40,639 - root - INFO - lr: 1.5982e-04 gnorm: 0.76 [11:23:40<10:38:27] +[titan] 2025-07-10 00:58:42,365 - root - INFO - Dumping profiler traces at step 51712 +[titan] 2025-07-10 00:58:42,397 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 00:58:44,767 - root - INFO - step: 51715 loss: 16.6515 memory: 44.58GiB(31.99%) tps: 79,368 tflops: 273.91 mfu: 27.70% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-10 00:58:44,768 - root - INFO - lr: 1.5979e-04 gnorm: 0.80 [11:23:44<10:38:23] +[titan] 2025-07-10 00:58:48,690 - root - INFO - step: 51720 loss: 16.3834 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8771 +[titan] 2025-07-10 00:58:48,690 - root - INFO - lr: 1.5977e-04 gnorm: 0.77 [11:23:48<10:38:19] +[titan] 2025-07-10 00:58:52,605 - root - INFO - step: 51725 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5749 global_avg_mtp_loss: 14.0723 +[titan] 2025-07-10 00:58:52,605 - root - INFO - lr: 1.5975e-04 gnorm: 0.87 [11:23:52<10:38:15] +[titan] 2025-07-10 00:58:56,539 - root - INFO - step: 51730 loss: 16.6057 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-10 00:58:56,540 - root - INFO - lr: 1.5973e-04 gnorm: 0.83 [11:23:56<10:38:11] +[titan] 2025-07-10 00:59:00,485 - root - INFO - step: 51735 loss: 16.3456 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8392 +[titan] 2025-07-10 00:59:00,485 - root - INFO - lr: 1.5971e-04 gnorm: 0.81 [11:24:00<10:38:07] +[titan] 2025-07-10 00:59:04,387 - root - INFO - step: 51740 loss: 16.7706 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.6088 global_avg_mtp_loss: 14.1618 +[titan] 2025-07-10 00:59:04,387 - root - INFO - lr: 1.5969e-04 gnorm: 0.82 [11:24:04<10:38:03] +[titan] 2025-07-10 00:59:08,286 - root - INFO - step: 51745 loss: 16.5158 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9793 +[titan] 2025-07-10 00:59:08,286 - root - INFO - lr: 1.5967e-04 gnorm: 0.81 [11:24:08<10:37:59] +[titan] 2025-07-10 00:59:11,416 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:59:12,201 - root - INFO - step: 51750 loss: 16.5603 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0003 +[titan] 2025-07-10 00:59:12,202 - root - INFO - lr: 1.5964e-04 gnorm: 0.80 [11:24:12<10:37:55] +[titan] 2025-07-10 00:59:16,122 - root - INFO - step: 51755 loss: 16.4985 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9598 +[titan] 2025-07-10 00:59:16,123 - root - INFO - lr: 1.5962e-04 gnorm: 0.80 [11:24:16<10:37:51] +[titan] 2025-07-10 00:59:20,022 - root - INFO - step: 51760 loss: 16.3251 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8269 +[titan] 2025-07-10 00:59:20,022 - root - INFO - lr: 1.5960e-04 gnorm: 0.78 [11:24:19<10:37:47] +[titan] 2025-07-10 00:59:23,935 - root - INFO - step: 51765 loss: 16.5955 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 14.0443 +[titan] 2025-07-10 00:59:23,935 - root - INFO - lr: 1.5958e-04 gnorm: 0.80 [11:24:23<10:37:43] +[titan] 2025-07-10 00:59:27,865 - root - INFO - step: 51770 loss: 16.6654 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0956 +[titan] 2025-07-10 00:59:27,865 - root - INFO - lr: 1.5956e-04 gnorm: 0.81 [11:24:27<10:37:39] +[titan] 2025-07-10 00:59:31,808 - root - INFO - step: 51775 loss: 16.5400 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 13.9865 +[titan] 2025-07-10 00:59:31,808 - root - INFO - lr: 1.5954e-04 gnorm: 0.83 [11:24:31<10:37:35] +[titan] 2025-07-10 00:59:35,725 - root - INFO - step: 51780 loss: 16.7232 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.1589 +[titan] 2025-07-10 00:59:35,726 - root - INFO - lr: 1.5952e-04 gnorm: 0.80 [11:24:35<10:37:31] +[titan] 2025-07-10 00:59:39,657 - root - INFO - step: 51785 loss: 16.4772 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9415 +[titan] 2025-07-10 00:59:39,658 - root - INFO - lr: 1.5949e-04 gnorm: 0.76 [11:24:39<10:37:27] +[titan] 2025-07-10 00:59:43,581 - root - INFO - step: 51790 loss: 16.5576 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 13.9988 +[titan] 2025-07-10 00:59:43,581 - root - INFO - lr: 1.5947e-04 gnorm: 0.81 [11:24:43<10:37:23] +[titan] 2025-07-10 00:59:47,508 - root - INFO - step: 51795 loss: 16.6774 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5633 global_avg_mtp_loss: 14.1141 +[titan] 2025-07-10 00:59:47,508 - root - INFO - lr: 1.5945e-04 gnorm: 0.83 [11:24:47<10:37:19] +[titan] 2025-07-10 00:59:50,622 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 00:59:51,407 - root - INFO - step: 51800 loss: 16.5953 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.0315 +[titan] 2025-07-10 00:59:51,408 - root - INFO - lr: 1.5943e-04 gnorm: 0.80 [11:24:51<10:37:15] +[titan] 2025-07-10 00:59:55,373 - root - INFO - step: 51805 loss: 16.4570 memory: 44.58GiB(31.99%) tps: 82,626 tflops: 285.16 mfu: 28.83% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9159 +[titan] 2025-07-10 00:59:55,374 - root - INFO - lr: 1.5941e-04 gnorm: 0.78 [11:24:55<10:37:11] +[titan] 2025-07-10 00:59:59,313 - root - INFO - step: 51810 loss: 16.5779 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0297 +[titan] 2025-07-10 00:59:59,314 - root - INFO - lr: 1.5939e-04 gnorm: 0.83 [11:24:59<10:37:07] +[titan] 2025-07-10 01:00:03,242 - root - INFO - step: 51815 loss: 16.5528 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 14.0087 +[titan] 2025-07-10 01:00:03,243 - root - INFO - lr: 1.5937e-04 gnorm: 0.79 [11:25:03<10:37:03] +[titan] 2025-07-10 01:00:07,157 - root - INFO - step: 51820 loss: 16.7579 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5882 global_avg_mtp_loss: 14.1697 +[titan] 2025-07-10 01:00:07,157 - root - INFO - lr: 1.5934e-04 gnorm: 0.81 [11:25:07<10:36:59] +[titan] 2025-07-10 01:00:11,057 - root - INFO - step: 51825 loss: 16.7553 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5935 global_avg_mtp_loss: 14.1618 +[titan] 2025-07-10 01:00:11,058 - root - INFO - lr: 1.5932e-04 gnorm: 0.79 [11:25:10<10:36:55] +[titan] 2025-07-10 01:00:14,974 - root - INFO - step: 51830 loss: 16.4526 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.9311 +[titan] 2025-07-10 01:00:14,974 - root - INFO - lr: 1.5930e-04 gnorm: 0.80 [11:25:14<10:36:51] +[titan] 2025-07-10 01:00:18,947 - root - INFO - step: 51835 loss: 16.3364 memory: 44.58GiB(31.99%) tps: 82,481 tflops: 284.66 mfu: 28.78% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8387 +[titan] 2025-07-10 01:00:18,947 - root - INFO - lr: 1.5928e-04 gnorm: 0.80 [11:25:18<10:36:47] +[titan] 2025-07-10 01:00:22,850 - root - INFO - step: 51840 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.7770 +[titan] 2025-07-10 01:00:22,850 - root - INFO - lr: 1.5926e-04 gnorm: 0.82 [11:25:22<10:36:43] +[titan] 2025-07-10 01:00:26,754 - root - INFO - step: 51845 loss: 16.6006 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.0285 +[titan] 2025-07-10 01:00:26,754 - root - INFO - lr: 1.5924e-04 gnorm: 0.83 [11:25:26<10:36:39] +[titan] 2025-07-10 01:00:29,869 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:00:30,662 - root - INFO - step: 51850 loss: 16.4710 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9343 +[titan] 2025-07-10 01:00:30,662 - root - INFO - lr: 1.5922e-04 gnorm: 0.85 [11:25:30<10:36:35] +[titan] 2025-07-10 01:00:34,570 - root - INFO - step: 51855 loss: 16.5099 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9754 +[titan] 2025-07-10 01:00:34,571 - root - INFO - lr: 1.5919e-04 gnorm: 0.83 [11:25:34<10:36:31] +[titan] 2025-07-10 01:00:38,487 - root - INFO - step: 51860 loss: 16.4057 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.8754 +[titan] 2025-07-10 01:00:38,487 - root - INFO - lr: 1.5917e-04 gnorm: 0.86 [11:25:38<10:36:27] +[titan] 2025-07-10 01:00:42,393 - root - INFO - step: 51865 loss: 16.4065 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8806 +[titan] 2025-07-10 01:00:42,393 - root - INFO - lr: 1.5915e-04 gnorm: 0.85 [11:25:42<10:36:23] +[titan] 2025-07-10 01:00:46,358 - root - INFO - step: 51870 loss: 16.9087 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.6242 global_avg_mtp_loss: 14.2845 +[titan] 2025-07-10 01:00:46,358 - root - INFO - lr: 1.5913e-04 gnorm: 0.84 [11:25:46<10:36:19] +[titan] 2025-07-10 01:00:50,299 - root - INFO - step: 51875 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0164 +[titan] 2025-07-10 01:00:50,299 - root - INFO - lr: 1.5911e-04 gnorm: 0.78 [11:25:50<10:36:15] +[titan] 2025-07-10 01:00:54,203 - root - INFO - step: 51880 loss: 16.6664 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0925 +[titan] 2025-07-10 01:00:54,203 - root - INFO - lr: 1.5909e-04 gnorm: 0.79 [11:25:54<10:36:11] +[titan] 2025-07-10 01:00:58,141 - root - INFO - step: 51885 loss: 16.7177 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.1459 +[titan] 2025-07-10 01:00:58,142 - root - INFO - lr: 1.5907e-04 gnorm: 0.85 [11:25:58<10:36:07] +[titan] 2025-07-10 01:01:02,066 - root - INFO - step: 51890 loss: 16.4665 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.9436 +[titan] 2025-07-10 01:01:02,066 - root - INFO - lr: 1.5905e-04 gnorm: 0.77 [11:26:01<10:36:03] +[titan] 2025-07-10 01:01:06,024 - root - INFO - step: 51895 loss: 16.6342 memory: 44.58GiB(31.99%) tps: 82,808 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0771 +[titan] 2025-07-10 01:01:06,024 - root - INFO - lr: 1.5902e-04 gnorm: 0.81 [11:26:05<10:35:59] +[titan] 2025-07-10 01:01:09,137 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:01:09,927 - root - INFO - step: 51900 loss: 16.7832 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.2079 +[titan] 2025-07-10 01:01:09,927 - root - INFO - lr: 1.5900e-04 gnorm: 0.80 [11:26:09<10:35:55] +[titan] 2025-07-10 01:01:13,834 - root - INFO - step: 51905 loss: 16.3825 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.8563 +[titan] 2025-07-10 01:01:13,834 - root - INFO - lr: 1.5898e-04 gnorm: 0.84 [11:26:13<10:35:51] +[titan] 2025-07-10 01:01:17,752 - root - INFO - step: 51910 loss: 16.4623 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9357 +[titan] 2025-07-10 01:01:17,752 - root - INFO - lr: 1.5896e-04 gnorm: 0.79 [11:26:17<10:35:47] +[titan] 2025-07-10 01:01:21,659 - root - INFO - step: 51915 loss: 16.4817 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9583 +[titan] 2025-07-10 01:01:21,659 - root - INFO - lr: 1.5894e-04 gnorm: 0.82 [11:26:21<10:35:43] +[titan] 2025-07-10 01:01:25,564 - root - INFO - step: 51920 loss: 16.5518 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0043 +[titan] 2025-07-10 01:01:25,564 - root - INFO - lr: 1.5892e-04 gnorm: 0.80 [11:26:25<10:35:39] +[titan] 2025-07-10 01:01:29,509 - root - INFO - step: 51925 loss: 16.7334 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1546 +[titan] 2025-07-10 01:01:29,510 - root - INFO - lr: 1.5890e-04 gnorm: 0.78 [11:26:29<10:35:35] +[titan] 2025-07-10 01:01:33,411 - root - INFO - step: 51930 loss: 16.3265 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.8186 +[titan] 2025-07-10 01:01:33,412 - root - INFO - lr: 1.5887e-04 gnorm: 0.88 [11:26:33<10:35:31] +[titan] 2025-07-10 01:01:37,317 - root - INFO - step: 51935 loss: 16.3105 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8121 +[titan] 2025-07-10 01:01:37,317 - root - INFO - lr: 1.5885e-04 gnorm: 0.80 [11:26:37<10:35:27] +[titan] 2025-07-10 01:01:41,267 - root - INFO - step: 51940 loss: 16.3491 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5180 global_avg_mtp_loss: 13.8311 +[titan] 2025-07-10 01:01:41,267 - root - INFO - lr: 1.5883e-04 gnorm: 0.80 [11:26:41<10:35:23] +[titan] 2025-07-10 01:01:45,215 - root - INFO - step: 51945 loss: 16.4861 memory: 44.58GiB(31.99%) tps: 83,010 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 13.9444 +[titan] 2025-07-10 01:01:45,215 - root - INFO - lr: 1.5881e-04 gnorm: 0.80 [11:26:45<10:35:19] +[titan] 2025-07-10 01:01:48,336 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:01:49,134 - root - INFO - step: 51950 loss: 16.4363 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9080 +[titan] 2025-07-10 01:01:49,134 - root - INFO - lr: 1.5879e-04 gnorm: 0.80 [11:26:49<10:35:15] +[titan] 2025-07-10 01:01:53,068 - root - INFO - step: 51955 loss: 16.4411 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9211 +[titan] 2025-07-10 01:01:53,068 - root - INFO - lr: 1.5877e-04 gnorm: 0.81 [11:26:52<10:35:11] +[titan] 2025-07-10 01:01:56,998 - root - INFO - step: 51960 loss: 16.5660 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0106 +[titan] 2025-07-10 01:01:56,998 - root - INFO - lr: 1.5875e-04 gnorm: 0.81 [11:26:56<10:35:07] +[titan] 2025-07-10 01:02:00,923 - root - INFO - step: 51965 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 01:02:00,923 - root - INFO - lr: 1.5872e-04 gnorm: 0.82 [11:27:00<10:35:03] +[titan] 2025-07-10 01:02:04,829 - root - INFO - step: 51970 loss: 16.4235 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.9156 +[titan] 2025-07-10 01:02:04,829 - root - INFO - lr: 1.5870e-04 gnorm: 0.88 [11:27:04<10:34:59] +[titan] 2025-07-10 01:02:08,737 - root - INFO - step: 51975 loss: 16.5937 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.0293 +[titan] 2025-07-10 01:02:08,737 - root - INFO - lr: 1.5868e-04 gnorm: 0.81 [11:27:08<10:34:55] +[titan] 2025-07-10 01:02:12,658 - root - INFO - step: 51980 loss: 16.4850 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9506 +[titan] 2025-07-10 01:02:12,659 - root - INFO - lr: 1.5866e-04 gnorm: 0.80 [11:27:12<10:34:51] +[titan] 2025-07-10 01:02:16,569 - root - INFO - step: 51985 loss: 16.6327 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.0624 +[titan] 2025-07-10 01:02:16,569 - root - INFO - lr: 1.5864e-04 gnorm: 0.79 [11:27:16<10:34:47] +[titan] 2025-07-10 01:02:20,476 - root - INFO - step: 51990 loss: 16.0885 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6227 +[titan] 2025-07-10 01:02:20,476 - root - INFO - lr: 1.5862e-04 gnorm: 0.88 [11:27:20<10:34:43] +[titan] 2025-07-10 01:02:24,375 - root - INFO - step: 51995 loss: 16.3679 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8586 +[titan] 2025-07-10 01:02:24,375 - root - INFO - lr: 1.5860e-04 gnorm: 0.92 [11:27:24<10:34:39] +[titan] 2025-07-10 01:02:27,508 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:02:28,300 - root - INFO - step: 52000 loss: 16.3457 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-10 01:02:28,300 - root - INFO - lr: 1.5857e-04 gnorm: 0.80 [11:27:28<10:34:35] +[titan] 2025-07-10 01:02:32,225 - root - INFO - step: 52005 loss: 16.5038 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5484 global_avg_mtp_loss: 13.9554 +[titan] 2025-07-10 01:02:32,225 - root - INFO - lr: 1.5855e-04 gnorm: 0.82 [11:27:32<10:34:31] +[titan] 2025-07-10 01:02:36,132 - root - INFO - step: 52010 loss: 16.7227 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5913 global_avg_mtp_loss: 14.1315 +[titan] 2025-07-10 01:02:36,133 - root - INFO - lr: 1.5853e-04 gnorm: 0.97 [11:27:35<10:34:27] +[titan] 2025-07-10 01:02:40,109 - root - INFO - step: 52015 loss: 16.3406 memory: 44.58GiB(31.99%) tps: 82,415 tflops: 284.43 mfu: 28.76% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.8137 +[titan] 2025-07-10 01:02:40,109 - root - INFO - lr: 1.5851e-04 gnorm: 0.83 [11:27:39<10:34:23] +[titan] 2025-07-10 01:02:44,024 - root - INFO - step: 52020 loss: 16.6015 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5584 global_avg_mtp_loss: 14.0431 +[titan] 2025-07-10 01:02:44,024 - root - INFO - lr: 1.5849e-04 gnorm: 0.84 [11:27:43<10:34:19] +[titan] 2025-07-10 01:02:47,942 - root - INFO - step: 52025 loss: 16.3439 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8362 +[titan] 2025-07-10 01:02:47,943 - root - INFO - lr: 1.5847e-04 gnorm: 0.79 [11:27:47<10:34:15] +[titan] 2025-07-10 01:02:51,846 - root - INFO - step: 52030 loss: 16.1376 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.6595 +[titan] 2025-07-10 01:02:51,846 - root - INFO - lr: 1.5845e-04 gnorm: 0.84 [11:27:51<10:34:11] +[titan] 2025-07-10 01:02:55,794 - root - INFO - step: 52035 loss: 16.6631 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5801 global_avg_mtp_loss: 14.0831 +[titan] 2025-07-10 01:02:55,794 - root - INFO - lr: 1.5842e-04 gnorm: 0.79 [11:27:55<10:34:07] +[titan] 2025-07-10 01:02:59,717 - root - INFO - step: 52040 loss: 16.5021 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 13.9481 +[titan] 2025-07-10 01:02:59,717 - root - INFO - lr: 1.5840e-04 gnorm: 0.88 [11:27:59<10:34:03] +[titan] 2025-07-10 01:03:03,650 - root - INFO - step: 52045 loss: 16.4602 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9228 +[titan] 2025-07-10 01:03:03,650 - root - INFO - lr: 1.5838e-04 gnorm: 0.81 [11:28:03<10:33:59] +[titan] 2025-07-10 01:03:06,789 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:03:07,583 - root - INFO - step: 52050 loss: 16.6378 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0679 +[titan] 2025-07-10 01:03:07,583 - root - INFO - lr: 1.5836e-04 gnorm: 0.93 [11:28:07<10:33:55] +[titan] 2025-07-10 01:03:11,542 - root - INFO - step: 52055 loss: 16.4578 memory: 44.58GiB(31.99%) tps: 82,772 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9264 +[titan] 2025-07-10 01:03:11,543 - root - INFO - lr: 1.5834e-04 gnorm: 0.81 [11:28:11<10:33:51] +[titan] 2025-07-10 01:03:15,508 - root - INFO - step: 52060 loss: 16.2806 memory: 44.58GiB(31.99%) tps: 82,634 tflops: 285.18 mfu: 28.84% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.7873 +[titan] 2025-07-10 01:03:15,509 - root - INFO - lr: 1.5832e-04 gnorm: 0.81 [11:28:15<10:33:47] +[titan] 2025-07-10 01:03:19,424 - root - INFO - step: 52065 loss: 16.5466 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0008 +[titan] 2025-07-10 01:03:19,425 - root - INFO - lr: 1.5830e-04 gnorm: 0.83 [11:28:19<10:33:43] +[titan] 2025-07-10 01:03:23,379 - root - INFO - step: 52070 loss: 16.2241 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.7283 +[titan] 2025-07-10 01:03:23,380 - root - INFO - lr: 1.5827e-04 gnorm: 0.83 [11:28:23<10:33:39] +[titan] 2025-07-10 01:03:27,298 - root - INFO - step: 52075 loss: 16.6402 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5626 global_avg_mtp_loss: 14.0776 +[titan] 2025-07-10 01:03:27,298 - root - INFO - lr: 1.5825e-04 gnorm: 0.85 [11:28:27<10:33:35] +[titan] 2025-07-10 01:03:31,206 - root - INFO - step: 52080 loss: 16.6553 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0924 +[titan] 2025-07-10 01:03:31,206 - root - INFO - lr: 1.5823e-04 gnorm: 0.83 [11:28:31<10:33:31] +[titan] 2025-07-10 01:03:35,114 - root - INFO - step: 52085 loss: 16.4965 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9513 +[titan] 2025-07-10 01:03:35,114 - root - INFO - lr: 1.5821e-04 gnorm: 0.81 [11:28:34<10:33:27] +[titan] 2025-07-10 01:03:39,021 - root - INFO - step: 52090 loss: 16.5485 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0029 +[titan] 2025-07-10 01:03:39,022 - root - INFO - lr: 1.5819e-04 gnorm: 0.80 [11:28:38<10:33:23] +[titan] 2025-07-10 01:03:42,934 - root - INFO - step: 52095 loss: 16.6768 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.1014 +[titan] 2025-07-10 01:03:42,935 - root - INFO - lr: 1.5817e-04 gnorm: 0.90 [11:28:42<10:33:19] +[titan] 2025-07-10 01:03:46,062 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:03:46,849 - root - INFO - step: 52100 loss: 16.2262 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7339 +[titan] 2025-07-10 01:03:46,850 - root - INFO - lr: 1.5815e-04 gnorm: 0.83 [11:28:46<10:33:15] +[titan] 2025-07-10 01:03:50,777 - root - INFO - step: 52105 loss: 16.6937 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.1159 +[titan] 2025-07-10 01:03:50,778 - root - INFO - lr: 1.5813e-04 gnorm: 0.83 [11:28:50<10:33:11] +[titan] 2025-07-10 01:03:54,699 - root - INFO - step: 52110 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8288 +[titan] 2025-07-10 01:03:54,700 - root - INFO - lr: 1.5810e-04 gnorm: 0.82 [11:28:54<10:33:07] +[titan] 2025-07-10 01:03:58,617 - root - INFO - step: 52115 loss: 16.7213 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.1530 +[titan] 2025-07-10 01:03:58,617 - root - INFO - lr: 1.5808e-04 gnorm: 0.80 [11:28:58<10:33:03] +[titan] 2025-07-10 01:04:02,540 - root - INFO - step: 52120 loss: 16.5787 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5446 global_avg_mtp_loss: 14.0341 +[titan] 2025-07-10 01:04:02,540 - root - INFO - lr: 1.5806e-04 gnorm: 0.80 [11:29:02<10:32:59] +[titan] 2025-07-10 01:04:06,480 - root - INFO - step: 52125 loss: 16.6289 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 14.0732 +[titan] 2025-07-10 01:04:06,480 - root - INFO - lr: 1.5804e-04 gnorm: 0.88 [11:29:06<10:32:55] +[titan] 2025-07-10 01:04:10,382 - root - INFO - step: 52130 loss: 16.7200 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.1261 +[titan] 2025-07-10 01:04:10,382 - root - INFO - lr: 1.5802e-04 gnorm: 0.86 [11:29:10<10:32:51] +[titan] 2025-07-10 01:04:14,291 - root - INFO - step: 52135 loss: 16.7457 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.1517 +[titan] 2025-07-10 01:04:14,291 - root - INFO - lr: 1.5800e-04 gnorm: 0.82 [11:29:14<10:32:47] +[titan] 2025-07-10 01:04:18,205 - root - INFO - step: 52140 loss: 16.4446 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9119 +[titan] 2025-07-10 01:04:18,206 - root - INFO - lr: 1.5798e-04 gnorm: 0.81 [11:29:18<10:32:43] +[titan] 2025-07-10 01:04:22,137 - root - INFO - step: 52145 loss: 16.4899 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.9672 +[titan] 2025-07-10 01:04:22,137 - root - INFO - lr: 1.5795e-04 gnorm: 0.77 [11:29:21<10:32:39] +[titan] 2025-07-10 01:04:25,258 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:04:26,051 - root - INFO - step: 52150 loss: 16.4365 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.8985 +[titan] 2025-07-10 01:04:26,051 - root - INFO - lr: 1.5793e-04 gnorm: 0.83 [11:29:25<10:32:35] +[titan] 2025-07-10 01:04:29,963 - root - INFO - step: 52155 loss: 16.3739 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8610 +[titan] 2025-07-10 01:04:29,964 - root - INFO - lr: 1.5791e-04 gnorm: 0.88 [11:29:29<10:32:31] +[titan] 2025-07-10 01:04:33,882 - root - INFO - step: 52160 loss: 16.4248 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.8959 +[titan] 2025-07-10 01:04:33,882 - root - INFO - lr: 1.5789e-04 gnorm: 0.85 [11:29:33<10:32:27] +[titan] 2025-07-10 01:04:37,830 - root - INFO - step: 52165 loss: 16.5025 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 13.9561 +[titan] 2025-07-10 01:04:37,830 - root - INFO - lr: 1.5787e-04 gnorm: 0.83 [11:29:37<10:32:23] +[titan] 2025-07-10 01:04:41,741 - root - INFO - step: 52170 loss: 16.4434 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-10 01:04:41,741 - root - INFO - lr: 1.5785e-04 gnorm: 0.81 [11:29:41<10:32:19] +[titan] 2025-07-10 01:04:45,705 - root - INFO - step: 52175 loss: 16.4420 memory: 44.58GiB(31.99%) tps: 82,680 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9061 +[titan] 2025-07-10 01:04:45,705 - root - INFO - lr: 1.5783e-04 gnorm: 0.82 [11:29:45<10:32:15] +[titan] 2025-07-10 01:04:49,599 - root - INFO - step: 52180 loss: 16.5427 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 14.0011 +[titan] 2025-07-10 01:04:49,599 - root - INFO - lr: 1.5780e-04 gnorm: 0.78 [11:29:49<10:32:11] +[titan] 2025-07-10 01:04:53,497 - root - INFO - step: 52185 loss: 16.4479 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9160 +[titan] 2025-07-10 01:04:53,498 - root - INFO - lr: 1.5778e-04 gnorm: 0.81 [11:29:53<10:32:07] +[titan] 2025-07-10 01:04:57,418 - root - INFO - step: 52190 loss: 16.3863 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5180 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 01:04:57,419 - root - INFO - lr: 1.5776e-04 gnorm: 0.79 [11:29:57<10:32:03] +[titan] 2025-07-10 01:05:01,341 - root - INFO - step: 52195 loss: 16.3165 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 01:05:01,341 - root - INFO - lr: 1.5774e-04 gnorm: 0.81 [11:30:01<10:31:59] +[titan] 2025-07-10 01:05:04,492 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:05:05,314 - root - INFO - step: 52200 loss: 16.5448 memory: 44.58GiB(31.99%) tps: 82,492 tflops: 284.69 mfu: 28.79% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 14.0064 +[titan] 2025-07-10 01:05:05,314 - root - INFO - lr: 1.5772e-04 gnorm: 0.80 [11:30:05<10:31:55] +[titan] 2025-07-10 01:05:09,222 - root - INFO - step: 52205 loss: 16.7244 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.1398 +[titan] 2025-07-10 01:05:09,222 - root - INFO - lr: 1.5770e-04 gnorm: 0.79 [11:30:09<10:31:51] +[titan] 2025-07-10 01:05:13,133 - root - INFO - step: 52210 loss: 16.5643 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 14.0226 +[titan] 2025-07-10 01:05:13,133 - root - INFO - lr: 1.5768e-04 gnorm: 0.83 [11:30:12<10:31:47] +[titan] 2025-07-10 01:05:17,049 - root - INFO - step: 52215 loss: 16.6142 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0428 +[titan] 2025-07-10 01:05:17,050 - root - INFO - lr: 1.5765e-04 gnorm: 0.79 [11:30:16<10:31:43] +[titan] 2025-07-10 01:05:20,986 - root - INFO - step: 52220 loss: 16.5961 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5567 global_avg_mtp_loss: 14.0394 +[titan] 2025-07-10 01:05:20,987 - root - INFO - lr: 1.5763e-04 gnorm: 0.78 [11:30:20<10:31:39] +[titan] 2025-07-10 01:05:24,390 - root - INFO - Dumping profiler traces at step 52224 +[titan] 2025-07-10 01:05:24,421 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:05:25,216 - root - INFO - step: 52225 loss: 16.5750 memory: 44.58GiB(31.99%) tps: 77,484 tflops: 267.41 mfu: 27.04% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 14.0248 +[titan] 2025-07-10 01:05:25,216 - root - INFO - lr: 1.5761e-04 gnorm: 0.80 [11:30:25<10:31:35] +[titan] 2025-07-10 01:05:29,120 - root - INFO - step: 52230 loss: 16.4038 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.8778 +[titan] 2025-07-10 01:05:29,121 - root - INFO - lr: 1.5759e-04 gnorm: 0.79 [11:30:28<10:31:31] +[titan] 2025-07-10 01:05:33,044 - root - INFO - step: 52235 loss: 16.5142 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-10 01:05:33,044 - root - INFO - lr: 1.5757e-04 gnorm: 0.88 [11:30:32<10:31:27] +[titan] 2025-07-10 01:05:36,973 - root - INFO - step: 52240 loss: 16.5308 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5392 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-10 01:05:36,973 - root - INFO - lr: 1.5755e-04 gnorm: 0.81 [11:30:36<10:31:23] +[titan] 2025-07-10 01:05:40,890 - root - INFO - step: 52245 loss: 16.6900 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.1180 +[titan] 2025-07-10 01:05:40,890 - root - INFO - lr: 1.5753e-04 gnorm: 0.78 [11:30:40<10:31:19] +[titan] 2025-07-10 01:05:44,036 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:05:44,834 - root - INFO - step: 52250 loss: 16.3751 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.8494 +[titan] 2025-07-10 01:05:44,834 - root - INFO - lr: 1.5750e-04 gnorm: 0.82 [11:30:44<10:31:15] +[titan] 2025-07-10 01:05:48,738 - root - INFO - step: 52255 loss: 16.5600 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 14.0183 +[titan] 2025-07-10 01:05:48,738 - root - INFO - lr: 1.5748e-04 gnorm: 0.81 [11:30:48<10:31:11] +[titan] 2025-07-10 01:05:52,664 - root - INFO - step: 52260 loss: 16.4483 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9166 +[titan] 2025-07-10 01:05:52,664 - root - INFO - lr: 1.5746e-04 gnorm: 0.90 [11:30:52<10:31:07] +[titan] 2025-07-10 01:05:56,596 - root - INFO - step: 52265 loss: 16.2962 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.7904 +[titan] 2025-07-10 01:05:56,596 - root - INFO - lr: 1.5744e-04 gnorm: 0.88 [11:30:56<10:31:03] +[titan] 2025-07-10 01:06:00,506 - root - INFO - step: 52270 loss: 16.7962 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.6008 global_avg_mtp_loss: 14.1955 +[titan] 2025-07-10 01:06:00,506 - root - INFO - lr: 1.5742e-04 gnorm: 0.81 [11:31:00<10:30:59] +[titan] 2025-07-10 01:06:04,402 - root - INFO - step: 52275 loss: 16.5395 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-10 01:06:04,403 - root - INFO - lr: 1.5740e-04 gnorm: 0.83 [11:31:04<10:30:55] +[titan] 2025-07-10 01:06:08,359 - root - INFO - step: 52280 loss: 16.5167 memory: 44.58GiB(31.99%) tps: 82,833 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 01:06:08,359 - root - INFO - lr: 1.5738e-04 gnorm: 0.84 [11:31:08<10:30:51] +[titan] 2025-07-10 01:06:12,259 - root - INFO - step: 52285 loss: 16.5549 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 14.0177 +[titan] 2025-07-10 01:06:12,259 - root - INFO - lr: 1.5736e-04 gnorm: 0.78 [11:31:12<10:30:47] +[titan] 2025-07-10 01:06:16,157 - root - INFO - step: 52290 loss: 16.4733 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.9458 +[titan] 2025-07-10 01:06:16,157 - root - INFO - lr: 1.5733e-04 gnorm: 0.84 [11:31:15<10:30:43] +[titan] 2025-07-10 01:06:20,110 - root - INFO - step: 52295 loss: 16.5681 memory: 44.58GiB(31.99%) tps: 82,890 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 14.0157 +[titan] 2025-07-10 01:06:20,111 - root - INFO - lr: 1.5731e-04 gnorm: 0.84 [11:31:19<10:30:39] +[titan] 2025-07-10 01:06:23,227 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:06:24,013 - root - INFO - step: 52300 loss: 16.4635 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9420 +[titan] 2025-07-10 01:06:24,013 - root - INFO - lr: 1.5729e-04 gnorm: 0.81 [11:31:23<10:30:35] +[titan] 2025-07-10 01:06:27,918 - root - INFO - step: 52305 loss: 16.3806 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8667 +[titan] 2025-07-10 01:06:27,918 - root - INFO - lr: 1.5727e-04 gnorm: 0.83 [11:31:27<10:30:31] +[titan] 2025-07-10 01:06:31,831 - root - INFO - step: 52310 loss: 16.6989 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5907 global_avg_mtp_loss: 14.1082 +[titan] 2025-07-10 01:06:31,832 - root - INFO - lr: 1.5725e-04 gnorm: 0.86 [11:31:31<10:30:27] +[titan] 2025-07-10 01:06:35,824 - root - INFO - step: 52315 loss: 16.3379 memory: 44.58GiB(31.99%) tps: 82,070 tflops: 283.24 mfu: 28.64% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8248 +[titan] 2025-07-10 01:06:35,825 - root - INFO - lr: 1.5723e-04 gnorm: 0.90 [11:31:35<10:30:23] +[titan] 2025-07-10 01:06:39,765 - root - INFO - step: 52320 loss: 16.7340 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1492 +[titan] 2025-07-10 01:06:39,766 - root - INFO - lr: 1.5721e-04 gnorm: 0.85 [11:31:39<10:30:19] +[titan] 2025-07-10 01:06:43,690 - root - INFO - step: 52325 loss: 16.6661 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0985 +[titan] 2025-07-10 01:06:43,690 - root - INFO - lr: 1.5718e-04 gnorm: 0.82 [11:31:43<10:30:15] +[titan] 2025-07-10 01:06:47,621 - root - INFO - step: 52330 loss: 16.6191 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5653 global_avg_mtp_loss: 14.0538 +[titan] 2025-07-10 01:06:47,621 - root - INFO - lr: 1.5716e-04 gnorm: 0.81 [11:31:47<10:30:11] +[titan] 2025-07-10 01:06:51,558 - root - INFO - step: 52335 loss: 16.9154 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.6123 global_avg_mtp_loss: 14.3030 +[titan] 2025-07-10 01:06:51,558 - root - INFO - lr: 1.5714e-04 gnorm: 0.82 [11:31:51<10:30:07] +[titan] 2025-07-10 01:06:55,463 - root - INFO - step: 52340 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.8676 +[titan] 2025-07-10 01:06:55,463 - root - INFO - lr: 1.5712e-04 gnorm: 0.84 [11:31:55<10:30:03] +[titan] 2025-07-10 01:06:59,410 - root - INFO - step: 52345 loss: 16.4463 memory: 44.58GiB(31.99%) tps: 83,033 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5375 global_avg_mtp_loss: 13.9088 +[titan] 2025-07-10 01:06:59,410 - root - INFO - lr: 1.5710e-04 gnorm: 0.81 [11:31:59<10:29:59] +[titan] 2025-07-10 01:07:02,578 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:07:03,370 - root - INFO - step: 52350 loss: 16.3639 memory: 44.58GiB(31.99%) tps: 82,757 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.8621 +[titan] 2025-07-10 01:07:03,370 - root - INFO - lr: 1.5708e-04 gnorm: 0.83 [11:32:03<10:29:55] +[titan] 2025-07-10 01:07:07,284 - root - INFO - step: 52355 loss: 16.5473 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0001 +[titan] 2025-07-10 01:07:07,284 - root - INFO - lr: 1.5706e-04 gnorm: 0.81 [11:32:07<10:29:51] +[titan] 2025-07-10 01:07:11,171 - root - INFO - step: 52360 loss: 16.4068 memory: 44.58GiB(31.99%) tps: 84,299 tflops: 290.93 mfu: 29.42% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.8815 +[titan] 2025-07-10 01:07:11,172 - root - INFO - lr: 1.5703e-04 gnorm: 0.82 [11:32:11<10:29:47] +[titan] 2025-07-10 01:07:15,094 - root - INFO - step: 52365 loss: 16.6449 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 14.0808 +[titan] 2025-07-10 01:07:15,094 - root - INFO - lr: 1.5701e-04 gnorm: 0.80 [11:32:14<10:29:43] +[titan] 2025-07-10 01:07:19,050 - root - INFO - step: 52370 loss: 16.5977 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-10 01:07:19,050 - root - INFO - lr: 1.5699e-04 gnorm: 0.84 [11:32:18<10:29:39] +[titan] 2025-07-10 01:07:22,947 - root - INFO - step: 52375 loss: 16.5640 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5464 global_avg_mtp_loss: 14.0175 +[titan] 2025-07-10 01:07:22,947 - root - INFO - lr: 1.5697e-04 gnorm: 0.81 [11:32:22<10:29:35] +[titan] 2025-07-10 01:07:26,907 - root - INFO - step: 52380 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 82,758 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8041 +[titan] 2025-07-10 01:07:26,907 - root - INFO - lr: 1.5695e-04 gnorm: 0.85 [11:32:26<10:29:31] +[titan] 2025-07-10 01:07:30,832 - root - INFO - step: 52385 loss: 16.6052 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0542 +[titan] 2025-07-10 01:07:30,832 - root - INFO - lr: 1.5693e-04 gnorm: 0.82 [11:32:30<10:29:27] +[titan] 2025-07-10 01:07:34,741 - root - INFO - step: 52390 loss: 16.4315 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.8936 +[titan] 2025-07-10 01:07:34,741 - root - INFO - lr: 1.5691e-04 gnorm: 0.87 [11:32:34<10:29:23] +[titan] 2025-07-10 01:07:38,661 - root - INFO - step: 52395 loss: 16.5098 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5462 global_avg_mtp_loss: 13.9636 +[titan] 2025-07-10 01:07:38,661 - root - INFO - lr: 1.5688e-04 gnorm: 0.86 [11:32:38<10:29:19] +[titan] 2025-07-10 01:07:41,791 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:07:42,588 - root - INFO - step: 52400 loss: 16.8256 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5991 global_avg_mtp_loss: 14.2265 +[titan] 2025-07-10 01:07:42,588 - root - INFO - lr: 1.5686e-04 gnorm: 0.82 [11:32:42<10:29:15] +[titan] 2025-07-10 01:07:46,552 - root - INFO - step: 52405 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 14.0238 +[titan] 2025-07-10 01:07:46,552 - root - INFO - lr: 1.5684e-04 gnorm: 0.81 [11:32:46<10:29:11] +[titan] 2025-07-10 01:07:50,452 - root - INFO - step: 52410 loss: 16.2342 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7416 +[titan] 2025-07-10 01:07:50,452 - root - INFO - lr: 1.5682e-04 gnorm: 0.85 [11:32:50<10:29:07] +[titan] 2025-07-10 01:07:54,367 - root - INFO - step: 52415 loss: 16.7002 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5647 global_avg_mtp_loss: 14.1355 +[titan] 2025-07-10 01:07:54,368 - root - INFO - lr: 1.5680e-04 gnorm: 0.85 [11:32:54<10:29:03] +[titan] 2025-07-10 01:07:58,300 - root - INFO - step: 52420 loss: 16.2113 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7237 +[titan] 2025-07-10 01:07:58,300 - root - INFO - lr: 1.5678e-04 gnorm: 0.86 [11:32:58<10:28:59] +[titan] 2025-07-10 01:08:02,212 - root - INFO - step: 52425 loss: 16.5863 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5468 global_avg_mtp_loss: 14.0395 +[titan] 2025-07-10 01:08:02,212 - root - INFO - lr: 1.5676e-04 gnorm: 0.82 [11:33:02<10:28:55] +[titan] 2025-07-10 01:08:06,156 - root - INFO - step: 52430 loss: 16.3776 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8726 +[titan] 2025-07-10 01:08:06,156 - root - INFO - lr: 1.5674e-04 gnorm: 0.83 [11:33:05<10:28:51] +[titan] 2025-07-10 01:08:10,061 - root - INFO - step: 52435 loss: 16.5149 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-10 01:08:10,062 - root - INFO - lr: 1.5671e-04 gnorm: 0.81 [11:33:09<10:28:47] +[titan] 2025-07-10 01:08:13,979 - root - INFO - step: 52440 loss: 16.4345 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.9049 +[titan] 2025-07-10 01:08:13,979 - root - INFO - lr: 1.5669e-04 gnorm: 0.84 [11:33:13<10:28:43] +[titan] 2025-07-10 01:08:17,889 - root - INFO - step: 52445 loss: 16.6561 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.0887 +[titan] 2025-07-10 01:08:17,890 - root - INFO - lr: 1.5667e-04 gnorm: 0.81 [11:33:17<10:28:39] +[titan] 2025-07-10 01:08:21,008 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:08:21,796 - root - INFO - step: 52450 loss: 16.4024 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8904 +[titan] 2025-07-10 01:08:21,796 - root - INFO - lr: 1.5665e-04 gnorm: 0.80 [11:33:21<10:28:35] +[titan] 2025-07-10 01:08:25,710 - root - INFO - step: 52455 loss: 16.4886 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-10 01:08:25,710 - root - INFO - lr: 1.5663e-04 gnorm: 0.78 [11:33:25<10:28:31] +[titan] 2025-07-10 01:08:29,608 - root - INFO - step: 52460 loss: 16.6131 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0540 +[titan] 2025-07-10 01:08:29,608 - root - INFO - lr: 1.5661e-04 gnorm: 0.85 [11:33:29<10:28:27] +[titan] 2025-07-10 01:08:33,580 - root - INFO - step: 52465 loss: 16.4697 memory: 44.58GiB(31.99%) tps: 82,509 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.9356 +[titan] 2025-07-10 01:08:33,580 - root - INFO - lr: 1.5659e-04 gnorm: 0.86 [11:33:33<10:28:23] +[titan] 2025-07-10 01:08:37,499 - root - INFO - step: 52470 loss: 16.6363 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.0718 +[titan] 2025-07-10 01:08:37,499 - root - INFO - lr: 1.5656e-04 gnorm: 0.79 [11:33:37<10:28:19] +[titan] 2025-07-10 01:08:41,436 - root - INFO - step: 52475 loss: 16.5241 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 13.9815 +[titan] 2025-07-10 01:08:41,436 - root - INFO - lr: 1.5654e-04 gnorm: 0.82 [11:33:41<10:28:15] +[titan] 2025-07-10 01:08:45,343 - root - INFO - step: 52480 loss: 16.3160 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8144 +[titan] 2025-07-10 01:08:45,343 - root - INFO - lr: 1.5652e-04 gnorm: 0.81 [11:33:45<10:28:11] +[titan] 2025-07-10 01:08:49,265 - root - INFO - step: 52485 loss: 16.6049 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0411 +[titan] 2025-07-10 01:08:49,265 - root - INFO - lr: 1.5650e-04 gnorm: 0.81 [11:33:49<10:28:07] +[titan] 2025-07-10 01:08:53,186 - root - INFO - step: 52490 loss: 16.5535 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 14.0165 +[titan] 2025-07-10 01:08:53,186 - root - INFO - lr: 1.5648e-04 gnorm: 0.78 [11:33:53<10:28:03] +[titan] 2025-07-10 01:08:57,131 - root - INFO - step: 52495 loss: 16.6788 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5728 global_avg_mtp_loss: 14.1061 +[titan] 2025-07-10 01:08:57,131 - root - INFO - lr: 1.5646e-04 gnorm: 0.79 [11:33:56<10:27:59] +[titan] 2025-07-10 01:09:00,236 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:09:01,033 - root - INFO - step: 52500 loss: 16.6415 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5645 global_avg_mtp_loss: 14.0770 +[titan] 2025-07-10 01:09:01,033 - root - INFO - lr: 1.5644e-04 gnorm: 0.82 [11:34:00<10:27:55] +[titan] 2025-07-10 01:09:04,945 - root - INFO - step: 52505 loss: 16.5496 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0039 +[titan] 2025-07-10 01:09:04,945 - root - INFO - lr: 1.5641e-04 gnorm: 0.85 [11:34:04<10:27:51] +[titan] 2025-07-10 01:09:08,861 - root - INFO - step: 52510 loss: 16.6113 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0592 +[titan] 2025-07-10 01:09:08,861 - root - INFO - lr: 1.5639e-04 gnorm: 0.85 [11:34:08<10:27:47] +[titan] 2025-07-10 01:09:12,804 - root - INFO - step: 52515 loss: 16.3865 memory: 44.58GiB(31.99%) tps: 83,104 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8714 +[titan] 2025-07-10 01:09:12,805 - root - INFO - lr: 1.5637e-04 gnorm: 0.82 [11:34:12<10:27:43] +[titan] 2025-07-10 01:09:16,709 - root - INFO - step: 52520 loss: 16.5376 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9983 +[titan] 2025-07-10 01:09:16,710 - root - INFO - lr: 1.5635e-04 gnorm: 0.78 [11:34:16<10:27:39] +[titan] 2025-07-10 01:09:20,627 - root - INFO - step: 52525 loss: 16.4044 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8892 +[titan] 2025-07-10 01:09:20,627 - root - INFO - lr: 1.5633e-04 gnorm: 0.80 [11:34:20<10:27:35] +[titan] 2025-07-10 01:09:24,540 - root - INFO - step: 52530 loss: 16.1628 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.6866 +[titan] 2025-07-10 01:09:24,540 - root - INFO - lr: 1.5631e-04 gnorm: 0.85 [11:34:24<10:27:30] +[titan] 2025-07-10 01:09:28,492 - root - INFO - step: 52535 loss: 16.3849 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.5165 global_avg_mtp_loss: 13.8684 +[titan] 2025-07-10 01:09:28,492 - root - INFO - lr: 1.5629e-04 gnorm: 0.81 [11:34:28<10:27:27] +[titan] 2025-07-10 01:09:32,401 - root - INFO - step: 52540 loss: 16.5206 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9806 +[titan] 2025-07-10 01:09:32,401 - root - INFO - lr: 1.5626e-04 gnorm: 0.86 [11:34:32<10:27:23] +[titan] 2025-07-10 01:09:36,354 - root - INFO - step: 52545 loss: 16.4045 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8876 +[titan] 2025-07-10 01:09:36,354 - root - INFO - lr: 1.5624e-04 gnorm: 0.87 [11:34:36<10:27:19] +[titan] 2025-07-10 01:09:39,472 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:09:40,267 - root - INFO - step: 52550 loss: 16.5511 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 13.9899 +[titan] 2025-07-10 01:09:40,267 - root - INFO - lr: 1.5622e-04 gnorm: 0.84 [11:34:40<10:27:15] +[titan] 2025-07-10 01:09:44,241 - root - INFO - step: 52555 loss: 16.6690 memory: 44.58GiB(31.99%) tps: 82,475 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.5821 global_avg_mtp_loss: 14.0869 +[titan] 2025-07-10 01:09:44,241 - root - INFO - lr: 1.5620e-04 gnorm: 0.85 [11:34:44<10:27:11] +[titan] 2025-07-10 01:09:48,173 - root - INFO - step: 52560 loss: 16.4931 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9724 +[titan] 2025-07-10 01:09:48,173 - root - INFO - lr: 1.5618e-04 gnorm: 0.82 [11:34:47<10:27:07] +[titan] 2025-07-10 01:09:52,071 - root - INFO - step: 52565 loss: 16.6777 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.1137 +[titan] 2025-07-10 01:09:52,071 - root - INFO - lr: 1.5616e-04 gnorm: 0.79 [11:34:51<10:27:03] +[titan] 2025-07-10 01:09:55,977 - root - INFO - step: 52570 loss: 15.9400 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.4973 +[titan] 2025-07-10 01:09:55,977 - root - INFO - lr: 1.5614e-04 gnorm: 0.84 [11:34:55<10:26:59] +[titan] 2025-07-10 01:09:59,909 - root - INFO - step: 52575 loss: 16.3486 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 01:09:59,909 - root - INFO - lr: 1.5612e-04 gnorm: 0.89 [11:34:59<10:26:55] +[titan] 2025-07-10 01:10:03,807 - root - INFO - step: 52580 loss: 16.2823 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5092 global_avg_mtp_loss: 13.7731 +[titan] 2025-07-10 01:10:03,807 - root - INFO - lr: 1.5609e-04 gnorm: 0.84 [11:35:03<10:26:50] +[titan] 2025-07-10 01:10:07,722 - root - INFO - step: 52585 loss: 16.4350 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9112 +[titan] 2025-07-10 01:10:07,723 - root - INFO - lr: 1.5607e-04 gnorm: 0.80 [11:35:07<10:26:46] +[titan] 2025-07-10 01:10:11,634 - root - INFO - step: 52590 loss: 16.2059 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7243 +[titan] 2025-07-10 01:10:11,635 - root - INFO - lr: 1.5605e-04 gnorm: 0.83 [11:35:11<10:26:42] +[titan] 2025-07-10 01:10:15,583 - root - INFO - step: 52595 loss: 16.5227 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 13.9703 +[titan] 2025-07-10 01:10:15,583 - root - INFO - lr: 1.5603e-04 gnorm: 0.92 [11:35:15<10:26:38] +[titan] 2025-07-10 01:10:18,716 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:10:19,506 - root - INFO - step: 52600 loss: 16.4197 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.8943 +[titan] 2025-07-10 01:10:19,506 - root - INFO - lr: 1.5601e-04 gnorm: 0.84 [11:35:19<10:26:34] +[titan] 2025-07-10 01:10:23,457 - root - INFO - step: 52605 loss: 16.4744 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9304 +[titan] 2025-07-10 01:10:23,457 - root - INFO - lr: 1.5599e-04 gnorm: 0.87 [11:35:23<10:26:30] +[titan] 2025-07-10 01:10:27,429 - root - INFO - step: 52610 loss: 16.5698 memory: 44.58GiB(31.99%) tps: 82,495 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0262 +[titan] 2025-07-10 01:10:27,430 - root - INFO - lr: 1.5597e-04 gnorm: 0.80 [11:35:27<10:26:27] +[titan] 2025-07-10 01:10:31,353 - root - INFO - step: 52615 loss: 16.5577 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-10 01:10:31,353 - root - INFO - lr: 1.5594e-04 gnorm: 0.77 [11:35:31<10:26:23] +[titan] 2025-07-10 01:10:35,264 - root - INFO - step: 52620 loss: 16.1959 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.6953 +[titan] 2025-07-10 01:10:35,264 - root - INFO - lr: 1.5592e-04 gnorm: 0.83 [11:35:35<10:26:19] +[titan] 2025-07-10 01:10:39,188 - root - INFO - step: 52625 loss: 16.7511 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5875 global_avg_mtp_loss: 14.1635 +[titan] 2025-07-10 01:10:39,189 - root - INFO - lr: 1.5590e-04 gnorm: 0.81 [11:35:38<10:26:15] +[titan] 2025-07-10 01:10:43,107 - root - INFO - step: 52630 loss: 16.6043 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0533 +[titan] 2025-07-10 01:10:43,107 - root - INFO - lr: 1.5588e-04 gnorm: 0.83 [11:35:42<10:26:11] +[titan] 2025-07-10 01:10:47,020 - root - INFO - step: 52635 loss: 16.6852 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1072 +[titan] 2025-07-10 01:10:47,020 - root - INFO - lr: 1.5586e-04 gnorm: 0.87 [11:35:46<10:26:06] +[titan] 2025-07-10 01:10:50,922 - root - INFO - step: 52640 loss: 16.2148 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7327 +[titan] 2025-07-10 01:10:50,922 - root - INFO - lr: 1.5584e-04 gnorm: 0.84 [11:35:50<10:26:02] +[titan] 2025-07-10 01:10:54,832 - root - INFO - step: 52645 loss: 16.6614 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0944 +[titan] 2025-07-10 01:10:54,832 - root - INFO - lr: 1.5582e-04 gnorm: 1.02 [11:35:54<10:25:58] +[titan] 2025-07-10 01:10:57,948 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:10:58,737 - root - INFO - step: 52650 loss: 16.4228 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.8847 +[titan] 2025-07-10 01:10:58,737 - root - INFO - lr: 1.5579e-04 gnorm: 0.85 [11:35:58<10:25:54] +[titan] 2025-07-10 01:11:02,654 - root - INFO - step: 52655 loss: 16.4804 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9489 +[titan] 2025-07-10 01:11:02,655 - root - INFO - lr: 1.5577e-04 gnorm: 0.87 [11:36:02<10:25:50] +[titan] 2025-07-10 01:11:06,583 - root - INFO - step: 52660 loss: 16.3003 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5038 global_avg_mtp_loss: 13.7964 +[titan] 2025-07-10 01:11:06,583 - root - INFO - lr: 1.5575e-04 gnorm: 0.85 [11:36:06<10:25:46] +[titan] 2025-07-10 01:11:10,501 - root - INFO - step: 52665 loss: 16.9265 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.6284 global_avg_mtp_loss: 14.2981 +[titan] 2025-07-10 01:11:10,501 - root - INFO - lr: 1.5573e-04 gnorm: 0.90 [11:36:10<10:25:42] +[titan] 2025-07-10 01:11:14,429 - root - INFO - step: 52670 loss: 16.5864 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 14.0279 +[titan] 2025-07-10 01:11:14,429 - root - INFO - lr: 1.5571e-04 gnorm: 0.85 [11:36:14<10:25:38] +[titan] 2025-07-10 01:11:18,339 - root - INFO - step: 52675 loss: 16.4902 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9476 +[titan] 2025-07-10 01:11:18,339 - root - INFO - lr: 1.5569e-04 gnorm: 0.79 [11:36:18<10:25:34] +[titan] 2025-07-10 01:11:22,235 - root - INFO - step: 52680 loss: 16.4626 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9336 +[titan] 2025-07-10 01:11:22,236 - root - INFO - lr: 1.5567e-04 gnorm: 0.80 [11:36:22<10:25:30] +[titan] 2025-07-10 01:11:26,144 - root - INFO - step: 52685 loss: 16.2679 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7718 +[titan] 2025-07-10 01:11:26,144 - root - INFO - lr: 1.5564e-04 gnorm: 0.84 [11:36:25<10:25:26] +[titan] 2025-07-10 01:11:30,108 - root - INFO - step: 52690 loss: 16.5620 memory: 44.58GiB(31.99%) tps: 82,659 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0007 +[titan] 2025-07-10 01:11:30,109 - root - INFO - lr: 1.5562e-04 gnorm: 0.83 [11:36:29<10:25:22] +[titan] 2025-07-10 01:11:34,032 - root - INFO - step: 52695 loss: 16.6680 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5638 global_avg_mtp_loss: 14.1042 +[titan] 2025-07-10 01:11:34,032 - root - INFO - lr: 1.5560e-04 gnorm: 0.83 [11:36:33<10:25:18] +[titan] 2025-07-10 01:11:37,173 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:11:37,969 - root - INFO - step: 52700 loss: 16.5754 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 14.0330 +[titan] 2025-07-10 01:11:37,969 - root - INFO - lr: 1.5558e-04 gnorm: 0.84 [11:36:37<10:25:14] +[titan] 2025-07-10 01:11:41,960 - root - INFO - step: 52705 loss: 16.5765 memory: 44.58GiB(31.99%) tps: 82,107 tflops: 283.36 mfu: 28.65% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0215 +[titan] 2025-07-10 01:11:41,960 - root - INFO - lr: 1.5556e-04 gnorm: 0.78 [11:36:41<10:25:10] +[titan] 2025-07-10 01:11:45,865 - root - INFO - step: 52710 loss: 16.5613 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 14.0199 +[titan] 2025-07-10 01:11:45,866 - root - INFO - lr: 1.5554e-04 gnorm: 0.81 [11:36:45<10:25:06] +[titan] 2025-07-10 01:11:49,769 - root - INFO - step: 52715 loss: 16.5246 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9793 +[titan] 2025-07-10 01:11:49,769 - root - INFO - lr: 1.5552e-04 gnorm: 0.83 [11:36:49<10:25:02] +[titan] 2025-07-10 01:11:53,680 - root - INFO - step: 52720 loss: 16.6712 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5764 global_avg_mtp_loss: 14.0948 +[titan] 2025-07-10 01:11:53,680 - root - INFO - lr: 1.5550e-04 gnorm: 0.80 [11:36:53<10:24:58] +[titan] 2025-07-10 01:11:57,635 - root - INFO - step: 52725 loss: 16.4803 memory: 44.58GiB(31.99%) tps: 82,852 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.5253 global_avg_mtp_loss: 13.9550 +[titan] 2025-07-10 01:11:57,635 - root - INFO - lr: 1.5547e-04 gnorm: 0.86 [11:36:57<10:24:54] +[titan] 2025-07-10 01:12:01,555 - root - INFO - step: 52730 loss: 16.5006 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 13.9529 +[titan] 2025-07-10 01:12:01,555 - root - INFO - lr: 1.5545e-04 gnorm: 0.82 [11:37:01<10:24:50] +[titan] 2025-07-10 01:12:05,545 - root - INFO - step: 52735 loss: 16.5924 memory: 44.58GiB(31.99%) tps: 82,136 tflops: 283.46 mfu: 28.66% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0390 +[titan] 2025-07-10 01:12:05,545 - root - INFO - lr: 1.5543e-04 gnorm: 0.82 [11:37:05<10:24:46] +[titan] 2025-07-10 01:12:06,483 - root - INFO - Dumping profiler traces at step 52736 +[titan] 2025-07-10 01:12:06,516 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:12:09,662 - root - INFO - step: 52740 loss: 16.6489 memory: 44.58GiB(31.99%) tps: 79,588 tflops: 274.67 mfu: 27.77% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0901 +[titan] 2025-07-10 01:12:09,663 - root - INFO - lr: 1.5541e-04 gnorm: 0.77 [11:37:09<10:24:43] +[titan] 2025-07-10 01:12:13,572 - root - INFO - step: 52745 loss: 16.5405 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 13.9981 +[titan] 2025-07-10 01:12:13,572 - root - INFO - lr: 1.5539e-04 gnorm: 0.83 [11:37:13<10:24:39] +[titan] 2025-07-10 01:12:16,687 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:12:17,476 - root - INFO - step: 52750 loss: 16.7320 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5815 global_avg_mtp_loss: 14.1505 +[titan] 2025-07-10 01:12:17,476 - root - INFO - lr: 1.5537e-04 gnorm: 0.79 [11:37:17<10:24:35] +[titan] 2025-07-10 01:12:21,386 - root - INFO - step: 52755 loss: 16.4685 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-10 01:12:21,386 - root - INFO - lr: 1.5535e-04 gnorm: 0.83 [11:37:21<10:24:31] +[titan] 2025-07-10 01:12:25,334 - root - INFO - step: 52760 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0579 +[titan] 2025-07-10 01:12:25,334 - root - INFO - lr: 1.5532e-04 gnorm: 0.83 [11:37:25<10:24:27] +[titan] 2025-07-10 01:12:29,227 - root - INFO - step: 52765 loss: 16.5980 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-10 01:12:29,228 - root - INFO - lr: 1.5530e-04 gnorm: 0.88 [11:37:29<10:24:23] +[titan] 2025-07-10 01:12:33,159 - root - INFO - step: 52770 loss: 16.2466 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.7633 +[titan] 2025-07-10 01:12:33,159 - root - INFO - lr: 1.5528e-04 gnorm: 0.86 [11:37:32<10:24:19] +[titan] 2025-07-10 01:12:37,072 - root - INFO - step: 52775 loss: 16.7496 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1675 +[titan] 2025-07-10 01:12:37,073 - root - INFO - lr: 1.5526e-04 gnorm: 0.81 [11:37:36<10:24:15] +[titan] 2025-07-10 01:12:40,976 - root - INFO - step: 52780 loss: 16.5538 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 14.0109 +[titan] 2025-07-10 01:12:40,977 - root - INFO - lr: 1.5524e-04 gnorm: 0.80 [11:37:40<10:24:11] +[titan] 2025-07-10 01:12:44,889 - root - INFO - step: 52785 loss: 16.4675 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9197 +[titan] 2025-07-10 01:12:44,889 - root - INFO - lr: 1.5522e-04 gnorm: 0.84 [11:37:44<10:24:07] +[titan] 2025-07-10 01:12:48,806 - root - INFO - step: 52790 loss: 16.5434 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 13.9981 +[titan] 2025-07-10 01:12:48,806 - root - INFO - lr: 1.5520e-04 gnorm: 0.84 [11:37:48<10:24:03] +[titan] 2025-07-10 01:12:52,722 - root - INFO - step: 52795 loss: 16.6882 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.1093 +[titan] 2025-07-10 01:12:52,723 - root - INFO - lr: 1.5517e-04 gnorm: 0.81 [11:37:52<10:23:59] +[titan] 2025-07-10 01:12:55,851 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:12:56,639 - root - INFO - step: 52800 loss: 16.5158 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9799 +[titan] 2025-07-10 01:12:56,640 - root - INFO - lr: 1.5515e-04 gnorm: 0.85 [11:37:56<10:23:54] +[titan] 2025-07-10 01:13:00,565 - root - INFO - step: 52805 loss: 16.5112 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 13.9585 +[titan] 2025-07-10 01:13:00,566 - root - INFO - lr: 1.5513e-04 gnorm: 0.85 [11:38:00<10:23:50] +[titan] 2025-07-10 01:13:04,527 - root - INFO - step: 52810 loss: 16.4116 memory: 44.58GiB(31.99%) tps: 82,723 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.5331 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-10 01:13:04,527 - root - INFO - lr: 1.5511e-04 gnorm: 0.83 [11:38:04<10:23:47] +[titan] 2025-07-10 01:13:08,426 - root - INFO - step: 52815 loss: 16.5830 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5580 global_avg_mtp_loss: 14.0250 +[titan] 2025-07-10 01:13:08,426 - root - INFO - lr: 1.5509e-04 gnorm: 0.79 [11:38:08<10:23:42] +[titan] 2025-07-10 01:13:12,351 - root - INFO - step: 52820 loss: 16.5010 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 13.9633 +[titan] 2025-07-10 01:13:12,351 - root - INFO - lr: 1.5507e-04 gnorm: 0.83 [11:38:12<10:23:38] +[titan] 2025-07-10 01:13:16,298 - root - INFO - step: 52825 loss: 16.1947 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7178 +[titan] 2025-07-10 01:13:16,298 - root - INFO - lr: 1.5505e-04 gnorm: 0.81 [11:38:16<10:23:35] +[titan] 2025-07-10 01:13:20,205 - root - INFO - step: 52830 loss: 16.2729 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 01:13:20,205 - root - INFO - lr: 1.5503e-04 gnorm: 0.86 [11:38:19<10:23:30] +[titan] 2025-07-10 01:13:24,115 - root - INFO - step: 52835 loss: 16.5355 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 13.9958 +[titan] 2025-07-10 01:13:24,115 - root - INFO - lr: 1.5500e-04 gnorm: 0.94 [11:38:23<10:23:26] +[titan] 2025-07-10 01:13:28,041 - root - INFO - step: 52840 loss: 16.6569 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5760 global_avg_mtp_loss: 14.0809 +[titan] 2025-07-10 01:13:28,042 - root - INFO - lr: 1.5498e-04 gnorm: 0.81 [11:38:27<10:23:22] +[titan] 2025-07-10 01:13:31,975 - root - INFO - step: 52845 loss: 16.5290 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9867 +[titan] 2025-07-10 01:13:31,975 - root - INFO - lr: 1.5496e-04 gnorm: 0.86 [11:38:31<10:23:18] +[titan] 2025-07-10 01:13:35,114 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:13:35,916 - root - INFO - step: 52850 loss: 16.7790 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5822 global_avg_mtp_loss: 14.1969 +[titan] 2025-07-10 01:13:35,916 - root - INFO - lr: 1.5494e-04 gnorm: 0.82 [11:38:35<10:23:14] +[titan] 2025-07-10 01:13:39,829 - root - INFO - step: 52855 loss: 16.6372 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5518 global_avg_mtp_loss: 14.0854 +[titan] 2025-07-10 01:13:39,829 - root - INFO - lr: 1.5492e-04 gnorm: 0.84 [11:38:39<10:23:10] +[titan] 2025-07-10 01:13:43,766 - root - INFO - step: 52860 loss: 16.5718 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 14.0192 +[titan] 2025-07-10 01:13:43,766 - root - INFO - lr: 1.5490e-04 gnorm: 0.79 [11:38:43<10:23:06] +[titan] 2025-07-10 01:13:47,720 - root - INFO - step: 52865 loss: 16.4216 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.9005 +[titan] 2025-07-10 01:13:47,720 - root - INFO - lr: 1.5488e-04 gnorm: 0.88 [11:38:47<10:23:03] +[titan] 2025-07-10 01:13:51,623 - root - INFO - step: 52870 loss: 16.4489 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9265 +[titan] 2025-07-10 01:13:51,623 - root - INFO - lr: 1.5485e-04 gnorm: 0.81 [11:38:51<10:22:58] +[titan] 2025-07-10 01:13:55,563 - root - INFO - step: 52875 loss: 16.4564 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.9385 +[titan] 2025-07-10 01:13:55,563 - root - INFO - lr: 1.5483e-04 gnorm: 0.80 [11:38:55<10:22:55] +[titan] 2025-07-10 01:13:59,468 - root - INFO - step: 52880 loss: 16.5464 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 13.9944 +[titan] 2025-07-10 01:13:59,468 - root - INFO - lr: 1.5481e-04 gnorm: 0.82 [11:38:59<10:22:50] +[titan] 2025-07-10 01:14:03,408 - root - INFO - step: 52885 loss: 16.1844 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7048 +[titan] 2025-07-10 01:14:03,409 - root - INFO - lr: 1.5479e-04 gnorm: 0.81 [11:39:03<10:22:46] +[titan] 2025-07-10 01:14:07,326 - root - INFO - step: 52890 loss: 16.2375 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.7462 +[titan] 2025-07-10 01:14:07,326 - root - INFO - lr: 1.5477e-04 gnorm: 0.88 [11:39:07<10:22:42] +[titan] 2025-07-10 01:14:11,248 - root - INFO - step: 52895 loss: 16.3681 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8512 +[titan] 2025-07-10 01:14:11,248 - root - INFO - lr: 1.5475e-04 gnorm: 0.81 [11:39:11<10:22:38] +[titan] 2025-07-10 01:14:14,370 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:14:15,156 - root - INFO - step: 52900 loss: 16.5270 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.9870 +[titan] 2025-07-10 01:14:15,156 - root - INFO - lr: 1.5473e-04 gnorm: 0.79 [11:39:14<10:22:34] +[titan] 2025-07-10 01:14:19,099 - root - INFO - step: 52905 loss: 16.4432 memory: 44.58GiB(31.99%) tps: 83,109 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.9220 +[titan] 2025-07-10 01:14:19,099 - root - INFO - lr: 1.5470e-04 gnorm: 0.81 [11:39:18<10:22:30] +[titan] 2025-07-10 01:14:22,993 - root - INFO - step: 52910 loss: 16.2833 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7917 +[titan] 2025-07-10 01:14:22,993 - root - INFO - lr: 1.5468e-04 gnorm: 0.80 [11:39:22<10:22:26] +[titan] 2025-07-10 01:14:26,894 - root - INFO - step: 52915 loss: 16.7488 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.1671 +[titan] 2025-07-10 01:14:26,894 - root - INFO - lr: 1.5466e-04 gnorm: 0.80 [11:39:26<10:22:22] +[titan] 2025-07-10 01:14:30,796 - root - INFO - step: 52920 loss: 16.7909 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5930 global_avg_mtp_loss: 14.1979 +[titan] 2025-07-10 01:14:30,796 - root - INFO - lr: 1.5464e-04 gnorm: 0.82 [11:39:30<10:22:18] +[titan] 2025-07-10 01:14:34,708 - root - INFO - step: 52925 loss: 16.4374 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9023 +[titan] 2025-07-10 01:14:34,708 - root - INFO - lr: 1.5462e-04 gnorm: 0.81 [11:39:34<10:22:14] +[titan] 2025-07-10 01:14:38,637 - root - INFO - step: 52930 loss: 16.5383 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 14.0128 +[titan] 2025-07-10 01:14:38,638 - root - INFO - lr: 1.5460e-04 gnorm: 0.90 [11:39:38<10:22:10] +[titan] 2025-07-10 01:14:42,583 - root - INFO - step: 52935 loss: 16.3929 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8722 +[titan] 2025-07-10 01:14:42,583 - root - INFO - lr: 1.5458e-04 gnorm: 0.83 [11:39:42<10:22:06] +[titan] 2025-07-10 01:14:46,507 - root - INFO - step: 52940 loss: 16.4523 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-10 01:14:46,508 - root - INFO - lr: 1.5456e-04 gnorm: 0.78 [11:39:46<10:22:02] +[titan] 2025-07-10 01:14:50,427 - root - INFO - step: 52945 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5065 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 01:14:50,427 - root - INFO - lr: 1.5453e-04 gnorm: 0.85 [11:39:50<10:21:58] +[titan] 2025-07-10 01:14:53,583 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:14:54,369 - root - INFO - step: 52950 loss: 16.8340 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.6057 global_avg_mtp_loss: 14.2282 +[titan] 2025-07-10 01:14:54,370 - root - INFO - lr: 1.5451e-04 gnorm: 0.89 [11:39:54<10:21:54] +[titan] 2025-07-10 01:14:58,316 - root - INFO - step: 52955 loss: 16.4670 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9313 +[titan] 2025-07-10 01:14:58,316 - root - INFO - lr: 1.5449e-04 gnorm: 0.90 [11:39:58<10:21:50] +[titan] 2025-07-10 01:15:02,260 - root - INFO - step: 52960 loss: 16.6232 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0691 +[titan] 2025-07-10 01:15:02,260 - root - INFO - lr: 1.5447e-04 gnorm: 0.77 [11:40:02<10:21:46] +[titan] 2025-07-10 01:15:06,163 - root - INFO - step: 52965 loss: 16.2672 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.7733 +[titan] 2025-07-10 01:15:06,163 - root - INFO - lr: 1.5445e-04 gnorm: 0.87 [11:40:05<10:21:42] +[titan] 2025-07-10 01:15:10,140 - root - INFO - step: 52970 loss: 16.4242 memory: 44.58GiB(31.99%) tps: 82,395 tflops: 284.36 mfu: 28.75% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.9090 +[titan] 2025-07-10 01:15:10,140 - root - INFO - lr: 1.5443e-04 gnorm: 0.84 [11:40:09<10:21:38] +[titan] 2025-07-10 01:15:14,057 - root - INFO - step: 52975 loss: 16.5675 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 14.0193 +[titan] 2025-07-10 01:15:14,057 - root - INFO - lr: 1.5441e-04 gnorm: 0.86 [11:40:13<10:21:34] +[titan] 2025-07-10 01:15:17,989 - root - INFO - step: 52980 loss: 16.5223 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9783 +[titan] 2025-07-10 01:15:17,989 - root - INFO - lr: 1.5438e-04 gnorm: 0.89 [11:40:17<10:21:30] +[titan] 2025-07-10 01:15:21,900 - root - INFO - step: 52985 loss: 16.5483 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 14.0156 +[titan] 2025-07-10 01:15:21,901 - root - INFO - lr: 1.5436e-04 gnorm: 0.85 [11:40:21<10:21:26] +[titan] 2025-07-10 01:15:25,807 - root - INFO - step: 52990 loss: 16.5797 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0341 +[titan] 2025-07-10 01:15:25,808 - root - INFO - lr: 1.5434e-04 gnorm: 0.88 [11:40:25<10:21:22] +[titan] 2025-07-10 01:15:29,808 - root - INFO - step: 52995 loss: 16.2762 memory: 44.58GiB(31.99%) tps: 81,919 tflops: 282.72 mfu: 28.59% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.7734 +[titan] 2025-07-10 01:15:29,808 - root - INFO - lr: 1.5432e-04 gnorm: 0.89 [11:40:29<10:21:18] +[titan] 2025-07-10 01:15:32,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:15:33,737 - root - INFO - step: 53000 loss: 16.5303 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9874 +[titan] 2025-07-10 01:15:33,738 - root - INFO - lr: 1.5430e-04 gnorm: 0.85 [11:40:33<10:21:15] +[titan] 2025-07-10 01:15:37,644 - root - INFO - step: 53005 loss: 16.7458 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5872 global_avg_mtp_loss: 14.1585 +[titan] 2025-07-10 01:15:37,645 - root - INFO - lr: 1.5428e-04 gnorm: 0.79 [11:40:37<10:21:10] +[titan] 2025-07-10 01:15:41,537 - root - INFO - step: 53010 loss: 16.3282 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8090 +[titan] 2025-07-10 01:15:41,538 - root - INFO - lr: 1.5426e-04 gnorm: 0.82 [11:40:41<10:21:06] +[titan] 2025-07-10 01:15:45,490 - root - INFO - step: 53015 loss: 16.4069 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 01:15:45,490 - root - INFO - lr: 1.5424e-04 gnorm: 0.90 [11:40:45<10:21:02] +[titan] 2025-07-10 01:15:49,385 - root - INFO - step: 53020 loss: 16.6497 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-10 01:15:49,385 - root - INFO - lr: 1.5421e-04 gnorm: 0.80 [11:40:49<10:20:58] +[titan] 2025-07-10 01:15:53,392 - root - INFO - step: 53025 loss: 16.6943 memory: 44.58GiB(31.99%) tps: 81,786 tflops: 282.26 mfu: 28.54% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.1207 +[titan] 2025-07-10 01:15:53,392 - root - INFO - lr: 1.5419e-04 gnorm: 0.79 [11:40:53<10:20:55] +[titan] 2025-07-10 01:15:57,323 - root - INFO - step: 53030 loss: 16.9121 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.6249 global_avg_mtp_loss: 14.2872 +[titan] 2025-07-10 01:15:57,323 - root - INFO - lr: 1.5417e-04 gnorm: 0.88 [11:40:57<10:20:51] +[titan] 2025-07-10 01:16:01,272 - root - INFO - step: 53035 loss: 16.1887 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7069 +[titan] 2025-07-10 01:16:01,273 - root - INFO - lr: 1.5415e-04 gnorm: 0.85 [11:41:01<10:20:47] +[titan] 2025-07-10 01:16:05,183 - root - INFO - step: 53040 loss: 16.4389 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9071 +[titan] 2025-07-10 01:16:05,183 - root - INFO - lr: 1.5413e-04 gnorm: 0.85 [11:41:04<10:20:43] +[titan] 2025-07-10 01:16:09,140 - root - INFO - step: 53045 loss: 16.5196 memory: 44.58GiB(31.99%) tps: 82,819 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 13.9661 +[titan] 2025-07-10 01:16:09,140 - root - INFO - lr: 1.5411e-04 gnorm: 0.86 [11:41:08<10:20:39] +[titan] 2025-07-10 01:16:12,249 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:16:13,044 - root - INFO - step: 53050 loss: 16.5387 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 14.0124 +[titan] 2025-07-10 01:16:13,045 - root - INFO - lr: 1.5409e-04 gnorm: 0.86 [11:41:12<10:20:35] +[titan] 2025-07-10 01:16:17,026 - root - INFO - step: 53055 loss: 16.4714 memory: 44.58GiB(31.99%) tps: 82,307 tflops: 284.06 mfu: 28.72% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9341 +[titan] 2025-07-10 01:16:17,026 - root - INFO - lr: 1.5406e-04 gnorm: 0.85 [11:41:16<10:20:31] +[titan] 2025-07-10 01:16:20,967 - root - INFO - step: 53060 loss: 16.4382 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 13.9028 +[titan] 2025-07-10 01:16:20,967 - root - INFO - lr: 1.5404e-04 gnorm: 0.80 [11:41:20<10:20:27] +[titan] 2025-07-10 01:16:24,876 - root - INFO - step: 53065 loss: 16.4943 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9550 +[titan] 2025-07-10 01:16:24,876 - root - INFO - lr: 1.5402e-04 gnorm: 0.81 [11:41:24<10:20:23] +[titan] 2025-07-10 01:16:28,808 - root - INFO - step: 53070 loss: 16.4286 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9037 +[titan] 2025-07-10 01:16:28,809 - root - INFO - lr: 1.5400e-04 gnorm: 0.85 [11:41:28<10:20:19] +[titan] 2025-07-10 01:16:32,744 - root - INFO - step: 53075 loss: 16.5769 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0216 +[titan] 2025-07-10 01:16:32,744 - root - INFO - lr: 1.5398e-04 gnorm: 0.88 [11:41:32<10:20:15] +[titan] 2025-07-10 01:16:36,637 - root - INFO - step: 53080 loss: 16.5598 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 14.0112 +[titan] 2025-07-10 01:16:36,637 - root - INFO - lr: 1.5396e-04 gnorm: 0.83 [11:41:36<10:20:11] +[titan] 2025-07-10 01:16:40,543 - root - INFO - step: 53085 loss: 16.3437 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.8196 +[titan] 2025-07-10 01:16:40,544 - root - INFO - lr: 1.5394e-04 gnorm: 0.81 [11:41:40<10:20:07] +[titan] 2025-07-10 01:16:44,450 - root - INFO - step: 53090 loss: 16.5084 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9629 +[titan] 2025-07-10 01:16:44,450 - root - INFO - lr: 1.5391e-04 gnorm: 0.84 [11:41:44<10:20:03] +[titan] 2025-07-10 01:16:48,348 - root - INFO - step: 53095 loss: 16.3545 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8391 +[titan] 2025-07-10 01:16:48,349 - root - INFO - lr: 1.5389e-04 gnorm: 0.82 [11:41:48<10:19:59] +[titan] 2025-07-10 01:16:51,462 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:16:52,267 - root - INFO - step: 53100 loss: 16.3669 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8466 +[titan] 2025-07-10 01:16:52,267 - root - INFO - lr: 1.5387e-04 gnorm: 0.76 [11:41:52<10:19:54] +[titan] 2025-07-10 01:16:56,164 - root - INFO - step: 53105 loss: 16.5965 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 14.0457 +[titan] 2025-07-10 01:16:56,164 - root - INFO - lr: 1.5385e-04 gnorm: 0.84 [11:41:55<10:19:50] +[titan] 2025-07-10 01:17:00,089 - root - INFO - step: 53110 loss: 16.5608 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 14.0288 +[titan] 2025-07-10 01:17:00,089 - root - INFO - lr: 1.5383e-04 gnorm: 0.79 [11:41:59<10:19:46] +[titan] 2025-07-10 01:17:04,022 - root - INFO - step: 53115 loss: 16.4769 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9473 +[titan] 2025-07-10 01:17:04,022 - root - INFO - lr: 1.5381e-04 gnorm: 0.79 [11:42:03<10:19:42] +[titan] 2025-07-10 01:17:07,948 - root - INFO - step: 53120 loss: 16.5004 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.9671 +[titan] 2025-07-10 01:17:07,948 - root - INFO - lr: 1.5379e-04 gnorm: 0.84 [11:42:07<10:19:38] +[titan] 2025-07-10 01:17:11,879 - root - INFO - step: 53125 loss: 16.4238 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 01:17:11,879 - root - INFO - lr: 1.5377e-04 gnorm: 0.85 [11:42:11<10:19:34] +[titan] 2025-07-10 01:17:15,817 - root - INFO - step: 53130 loss: 16.4975 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9606 +[titan] 2025-07-10 01:17:15,817 - root - INFO - lr: 1.5374e-04 gnorm: 0.85 [11:42:15<10:19:30] +[titan] 2025-07-10 01:17:19,735 - root - INFO - step: 53135 loss: 16.5611 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 14.0108 +[titan] 2025-07-10 01:17:19,736 - root - INFO - lr: 1.5372e-04 gnorm: 0.77 [11:42:19<10:19:26] +[titan] 2025-07-10 01:17:23,674 - root - INFO - step: 53140 loss: 16.4304 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9011 +[titan] 2025-07-10 01:17:23,674 - root - INFO - lr: 1.5370e-04 gnorm: 0.80 [11:42:23<10:19:22] +[titan] 2025-07-10 01:17:27,608 - root - INFO - step: 53145 loss: 16.4534 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9277 +[titan] 2025-07-10 01:17:27,609 - root - INFO - lr: 1.5368e-04 gnorm: 0.79 [11:42:27<10:19:18] +[titan] 2025-07-10 01:17:30,731 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:17:31,540 - root - INFO - step: 53150 loss: 16.5512 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 13.9949 +[titan] 2025-07-10 01:17:31,541 - root - INFO - lr: 1.5366e-04 gnorm: 0.82 [11:42:31<10:19:15] +[titan] 2025-07-10 01:17:35,488 - root - INFO - step: 53155 loss: 16.3441 memory: 44.58GiB(31.99%) tps: 83,017 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5033 global_avg_mtp_loss: 13.8408 +[titan] 2025-07-10 01:17:35,488 - root - INFO - lr: 1.5364e-04 gnorm: 0.81 [11:42:35<10:19:11] +[titan] 2025-07-10 01:17:39,453 - root - INFO - step: 53160 loss: 16.4931 memory: 44.58GiB(31.99%) tps: 82,658 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9523 +[titan] 2025-07-10 01:17:39,453 - root - INFO - lr: 1.5362e-04 gnorm: 0.79 [11:42:39<10:19:07] +[titan] 2025-07-10 01:17:43,399 - root - INFO - step: 53165 loss: 16.4156 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.5267 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 01:17:43,400 - root - INFO - lr: 1.5359e-04 gnorm: 0.80 [11:42:43<10:19:03] +[titan] 2025-07-10 01:17:47,335 - root - INFO - step: 53170 loss: 16.2401 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.7463 +[titan] 2025-07-10 01:17:47,335 - root - INFO - lr: 1.5357e-04 gnorm: 0.82 [11:42:47<10:18:59] +[titan] 2025-07-10 01:17:51,288 - root - INFO - step: 53175 loss: 16.5802 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 14.0321 +[titan] 2025-07-10 01:17:51,288 - root - INFO - lr: 1.5355e-04 gnorm: 0.80 [11:42:51<10:18:55] +[titan] 2025-07-10 01:17:55,216 - root - INFO - step: 53180 loss: 16.3815 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.8558 +[titan] 2025-07-10 01:17:55,216 - root - INFO - lr: 1.5353e-04 gnorm: 0.86 [11:42:54<10:18:51] +[titan] 2025-07-10 01:17:59,151 - root - INFO - step: 53185 loss: 16.4504 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9259 +[titan] 2025-07-10 01:17:59,151 - root - INFO - lr: 1.5351e-04 gnorm: 0.84 [11:42:58<10:18:47] +[titan] 2025-07-10 01:18:03,054 - root - INFO - step: 53190 loss: 16.6102 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 14.0642 +[titan] 2025-07-10 01:18:03,054 - root - INFO - lr: 1.5349e-04 gnorm: 0.83 [11:43:02<10:18:43] +[titan] 2025-07-10 01:18:06,985 - root - INFO - step: 53195 loss: 16.2308 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7356 +[titan] 2025-07-10 01:18:06,985 - root - INFO - lr: 1.5347e-04 gnorm: 0.86 [11:43:06<10:18:39] +[titan] 2025-07-10 01:18:10,108 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:18:10,902 - root - INFO - step: 53200 loss: 16.4825 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.9536 +[titan] 2025-07-10 01:18:10,902 - root - INFO - lr: 1.5345e-04 gnorm: 0.91 [11:43:10<10:18:35] +[titan] 2025-07-10 01:18:14,812 - root - INFO - step: 53205 loss: 16.5056 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.9694 +[titan] 2025-07-10 01:18:14,812 - root - INFO - lr: 1.5342e-04 gnorm: 0.84 [11:43:14<10:18:31] +[titan] 2025-07-10 01:18:18,726 - root - INFO - step: 53210 loss: 16.3904 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8891 +[titan] 2025-07-10 01:18:18,727 - root - INFO - lr: 1.5340e-04 gnorm: 0.85 [11:43:18<10:18:27] +[titan] 2025-07-10 01:18:22,634 - root - INFO - step: 53215 loss: 16.4833 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.9636 +[titan] 2025-07-10 01:18:22,635 - root - INFO - lr: 1.5338e-04 gnorm: 0.84 [11:43:22<10:18:23] +[titan] 2025-07-10 01:18:26,563 - root - INFO - step: 53220 loss: 16.5917 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5666 global_avg_mtp_loss: 14.0251 +[titan] 2025-07-10 01:18:26,563 - root - INFO - lr: 1.5336e-04 gnorm: 0.78 [11:43:26<10:18:19] +[titan] 2025-07-10 01:18:30,488 - root - INFO - step: 53225 loss: 16.5189 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9800 +[titan] 2025-07-10 01:18:30,488 - root - INFO - lr: 1.5334e-04 gnorm: 0.76 [11:43:30<10:18:15] +[titan] 2025-07-10 01:18:34,439 - root - INFO - step: 53230 loss: 16.3876 memory: 44.58GiB(31.99%) tps: 82,941 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8759 +[titan] 2025-07-10 01:18:34,439 - root - INFO - lr: 1.5332e-04 gnorm: 0.86 [11:43:34<10:18:11] +[titan] 2025-07-10 01:18:38,357 - root - INFO - step: 53235 loss: 16.5354 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 13.9828 +[titan] 2025-07-10 01:18:38,357 - root - INFO - lr: 1.5330e-04 gnorm: 0.80 [11:43:38<10:18:07] +[titan] 2025-07-10 01:18:42,301 - root - INFO - step: 53240 loss: 16.2524 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7572 +[titan] 2025-07-10 01:18:42,302 - root - INFO - lr: 1.5327e-04 gnorm: 0.79 [11:43:42<10:18:03] +[titan] 2025-07-10 01:18:46,292 - root - INFO - step: 53245 loss: 16.2214 memory: 44.58GiB(31.99%) tps: 82,125 tflops: 283.43 mfu: 28.66% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7305 +[titan] 2025-07-10 01:18:46,292 - root - INFO - lr: 1.5325e-04 gnorm: 0.87 [11:43:46<10:17:59] +[titan] 2025-07-10 01:18:48,801 - root - INFO - Dumping profiler traces at step 53248 +[titan] 2025-07-10 01:18:48,833 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:18:49,625 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:18:50,428 - root - INFO - step: 53250 loss: 16.6087 memory: 44.58GiB(31.99%) tps: 79,237 tflops: 273.46 mfu: 27.65% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0473 +[titan] 2025-07-10 01:18:50,428 - root - INFO - lr: 1.5323e-04 gnorm: 0.83 [11:43:50<10:17:55] +[titan] 2025-07-10 01:18:54,333 - root - INFO - step: 53255 loss: 16.6735 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.0999 +[titan] 2025-07-10 01:18:54,333 - root - INFO - lr: 1.5321e-04 gnorm: 0.86 [11:43:54<10:17:51] +[titan] 2025-07-10 01:18:58,251 - root - INFO - step: 53260 loss: 16.5399 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9975 +[titan] 2025-07-10 01:18:58,251 - root - INFO - lr: 1.5319e-04 gnorm: 0.90 [11:43:58<10:17:47] +[titan] 2025-07-10 01:19:02,199 - root - INFO - step: 53265 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.7805 +[titan] 2025-07-10 01:19:02,200 - root - INFO - lr: 1.5317e-04 gnorm: 0.84 [11:44:01<10:17:43] +[titan] 2025-07-10 01:19:06,163 - root - INFO - step: 53270 loss: 16.4291 memory: 44.58GiB(31.99%) tps: 82,684 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.9117 +[titan] 2025-07-10 01:19:06,163 - root - INFO - lr: 1.5315e-04 gnorm: 0.86 [11:44:05<10:17:39] +[titan] 2025-07-10 01:19:10,123 - root - INFO - step: 53275 loss: 16.6634 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.0929 +[titan] 2025-07-10 01:19:10,123 - root - INFO - lr: 1.5313e-04 gnorm: 0.80 [11:44:09<10:17:35] +[titan] 2025-07-10 01:19:14,052 - root - INFO - step: 53280 loss: 16.4819 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 13.9303 +[titan] 2025-07-10 01:19:14,053 - root - INFO - lr: 1.5310e-04 gnorm: 0.88 [11:44:13<10:17:31] +[titan] 2025-07-10 01:19:18,011 - root - INFO - step: 53285 loss: 16.5973 memory: 44.58GiB(31.99%) tps: 82,784 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0343 +[titan] 2025-07-10 01:19:18,011 - root - INFO - lr: 1.5308e-04 gnorm: 0.83 [11:44:17<10:17:27] +[titan] 2025-07-10 01:19:21,912 - root - INFO - step: 53290 loss: 16.6619 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0978 +[titan] 2025-07-10 01:19:21,913 - root - INFO - lr: 1.5306e-04 gnorm: 0.83 [11:44:21<10:17:23] +[titan] 2025-07-10 01:19:25,845 - root - INFO - step: 53295 loss: 16.5954 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-10 01:19:25,845 - root - INFO - lr: 1.5304e-04 gnorm: 0.82 [11:44:25<10:17:19] +[titan] 2025-07-10 01:19:28,975 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:19:29,781 - root - INFO - step: 53300 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.9457 +[titan] 2025-07-10 01:19:29,781 - root - INFO - lr: 1.5302e-04 gnorm: 0.89 [11:44:29<10:17:15] +[titan] 2025-07-10 01:19:33,717 - root - INFO - step: 53305 loss: 16.3314 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.8144 +[titan] 2025-07-10 01:19:33,717 - root - INFO - lr: 1.5300e-04 gnorm: 0.82 [11:44:33<10:17:11] +[titan] 2025-07-10 01:19:37,624 - root - INFO - step: 53310 loss: 16.4169 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5360 global_avg_mtp_loss: 13.8810 +[titan] 2025-07-10 01:19:37,625 - root - INFO - lr: 1.5298e-04 gnorm: 0.90 [11:44:37<10:17:07] +[titan] 2025-07-10 01:19:41,530 - root - INFO - step: 53315 loss: 16.5928 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 14.0283 +[titan] 2025-07-10 01:19:41,531 - root - INFO - lr: 1.5295e-04 gnorm: 0.85 [11:44:41<10:17:03] +[titan] 2025-07-10 01:19:45,452 - root - INFO - step: 53320 loss: 16.6744 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5712 global_avg_mtp_loss: 14.1032 +[titan] 2025-07-10 01:19:45,452 - root - INFO - lr: 1.5293e-04 gnorm: 0.82 [11:44:45<10:16:59] +[titan] 2025-07-10 01:19:49,368 - root - INFO - step: 53325 loss: 16.7159 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.1386 +[titan] 2025-07-10 01:19:49,368 - root - INFO - lr: 1.5291e-04 gnorm: 0.83 [11:44:49<10:16:55] +[titan] 2025-07-10 01:19:53,282 - root - INFO - step: 53330 loss: 16.2783 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.7792 +[titan] 2025-07-10 01:19:53,283 - root - INFO - lr: 1.5289e-04 gnorm: 0.94 [11:44:53<10:16:51] +[titan] 2025-07-10 01:19:57,194 - root - INFO - step: 53335 loss: 16.6685 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.1026 +[titan] 2025-07-10 01:19:57,194 - root - INFO - lr: 1.5287e-04 gnorm: 0.79 [11:44:56<10:16:47] +[titan] 2025-07-10 01:20:01,110 - root - INFO - step: 53340 loss: 16.5951 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0430 +[titan] 2025-07-10 01:20:01,110 - root - INFO - lr: 1.5285e-04 gnorm: 0.82 [11:45:00<10:16:43] +[titan] 2025-07-10 01:20:05,072 - root - INFO - step: 53345 loss: 16.6307 memory: 44.58GiB(31.99%) tps: 82,718 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0753 +[titan] 2025-07-10 01:20:05,072 - root - INFO - lr: 1.5283e-04 gnorm: 0.85 [11:45:04<10:16:39] +[titan] 2025-07-10 01:20:08,205 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:20:09,003 - root - INFO - step: 53350 loss: 16.3450 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.8264 +[titan] 2025-07-10 01:20:09,003 - root - INFO - lr: 1.5281e-04 gnorm: 0.82 [11:45:08<10:16:35] +[titan] 2025-07-10 01:20:12,973 - root - INFO - step: 53355 loss: 16.4177 memory: 44.58GiB(31.99%) tps: 82,541 tflops: 284.86 mfu: 28.80% global_avg_ntp_loss: 2.5361 global_avg_mtp_loss: 13.8815 +[titan] 2025-07-10 01:20:12,973 - root - INFO - lr: 1.5278e-04 gnorm: 0.84 [11:45:12<10:16:31] +[titan] 2025-07-10 01:20:16,884 - root - INFO - step: 53360 loss: 16.4898 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9554 +[titan] 2025-07-10 01:20:16,884 - root - INFO - lr: 1.5276e-04 gnorm: 0.81 [11:45:16<10:16:27] +[titan] 2025-07-10 01:20:20,803 - root - INFO - step: 53365 loss: 16.6090 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5767 global_avg_mtp_loss: 14.0323 +[titan] 2025-07-10 01:20:20,804 - root - INFO - lr: 1.5274e-04 gnorm: 0.80 [11:45:20<10:16:23] +[titan] 2025-07-10 01:20:24,706 - root - INFO - step: 53370 loss: 16.7129 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5721 global_avg_mtp_loss: 14.1408 +[titan] 2025-07-10 01:20:24,706 - root - INFO - lr: 1.5272e-04 gnorm: 0.83 [11:45:24<10:16:19] +[titan] 2025-07-10 01:20:28,620 - root - INFO - step: 53375 loss: 16.2703 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.7723 +[titan] 2025-07-10 01:20:28,621 - root - INFO - lr: 1.5270e-04 gnorm: 0.79 [11:45:28<10:16:15] +[titan] 2025-07-10 01:20:32,521 - root - INFO - step: 53380 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 01:20:32,521 - root - INFO - lr: 1.5268e-04 gnorm: 0.83 [11:45:32<10:16:11] +[titan] 2025-07-10 01:20:36,416 - root - INFO - step: 53385 loss: 16.4666 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9332 +[titan] 2025-07-10 01:20:36,417 - root - INFO - lr: 1.5266e-04 gnorm: 0.80 [11:45:36<10:16:07] +[titan] 2025-07-10 01:20:40,338 - root - INFO - step: 53390 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9585 +[titan] 2025-07-10 01:20:40,339 - root - INFO - lr: 1.5263e-04 gnorm: 0.84 [11:45:40<10:16:03] +[titan] 2025-07-10 01:20:44,258 - root - INFO - step: 53395 loss: 16.4109 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8838 +[titan] 2025-07-10 01:20:44,258 - root - INFO - lr: 1.5261e-04 gnorm: 0.81 [11:45:43<10:15:59] +[titan] 2025-07-10 01:20:47,393 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:20:48,184 - root - INFO - step: 53400 loss: 16.5651 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5465 global_avg_mtp_loss: 14.0186 +[titan] 2025-07-10 01:20:48,184 - root - INFO - lr: 1.5259e-04 gnorm: 0.86 [11:45:47<10:15:55] +[titan] 2025-07-10 01:20:52,111 - root - INFO - step: 53405 loss: 16.4582 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9375 +[titan] 2025-07-10 01:20:52,111 - root - INFO - lr: 1.5257e-04 gnorm: 0.83 [11:45:51<10:15:51] +[titan] 2025-07-10 01:20:56,025 - root - INFO - step: 53410 loss: 16.3214 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8041 +[titan] 2025-07-10 01:20:56,026 - root - INFO - lr: 1.5255e-04 gnorm: 0.84 [11:45:55<10:15:47] +[titan] 2025-07-10 01:20:59,953 - root - INFO - step: 53415 loss: 16.2836 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.7763 +[titan] 2025-07-10 01:20:59,953 - root - INFO - lr: 1.5253e-04 gnorm: 0.83 [11:45:59<10:15:43] +[titan] 2025-07-10 01:21:03,848 - root - INFO - step: 53420 loss: 16.6474 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0814 +[titan] 2025-07-10 01:21:03,849 - root - INFO - lr: 1.5251e-04 gnorm: 0.83 [11:46:03<10:15:39] +[titan] 2025-07-10 01:21:07,778 - root - INFO - step: 53425 loss: 16.5655 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-10 01:21:07,778 - root - INFO - lr: 1.5249e-04 gnorm: 0.88 [11:46:07<10:15:35] +[titan] 2025-07-10 01:21:11,701 - root - INFO - step: 53430 loss: 16.6068 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0420 +[titan] 2025-07-10 01:21:11,701 - root - INFO - lr: 1.5246e-04 gnorm: 0.86 [11:46:11<10:15:31] +[titan] 2025-07-10 01:21:15,632 - root - INFO - step: 53435 loss: 16.5306 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-10 01:21:15,632 - root - INFO - lr: 1.5244e-04 gnorm: 0.83 [11:46:15<10:15:27] +[titan] 2025-07-10 01:21:19,552 - root - INFO - step: 53440 loss: 16.3442 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8360 +[titan] 2025-07-10 01:21:19,552 - root - INFO - lr: 1.5242e-04 gnorm: 0.88 [11:46:19<10:15:23] +[titan] 2025-07-10 01:21:23,448 - root - INFO - step: 53445 loss: 16.7864 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.1869 +[titan] 2025-07-10 01:21:23,448 - root - INFO - lr: 1.5240e-04 gnorm: 0.86 [11:46:23<10:15:19] +[titan] 2025-07-10 01:21:26,556 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:21:27,349 - root - INFO - step: 53450 loss: 16.4104 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8913 +[titan] 2025-07-10 01:21:27,350 - root - INFO - lr: 1.5238e-04 gnorm: 0.86 [11:46:27<10:15:15] +[titan] 2025-07-10 01:21:31,298 - root - INFO - step: 53455 loss: 16.5559 memory: 44.58GiB(31.99%) tps: 82,985 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 14.0154 +[titan] 2025-07-10 01:21:31,299 - root - INFO - lr: 1.5236e-04 gnorm: 0.84 [11:46:31<10:15:11] +[titan] 2025-07-10 01:21:35,232 - root - INFO - step: 53460 loss: 16.4716 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9348 +[titan] 2025-07-10 01:21:35,232 - root - INFO - lr: 1.5234e-04 gnorm: 0.80 [11:46:34<10:15:07] +[titan] 2025-07-10 01:21:39,134 - root - INFO - step: 53465 loss: 16.1810 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7004 +[titan] 2025-07-10 01:21:39,134 - root - INFO - lr: 1.5231e-04 gnorm: 0.91 [11:46:38<10:15:03] +[titan] 2025-07-10 01:21:43,029 - root - INFO - step: 53470 loss: 16.1138 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.6241 +[titan] 2025-07-10 01:21:43,029 - root - INFO - lr: 1.5229e-04 gnorm: 0.87 [11:46:42<10:14:59] +[titan] 2025-07-10 01:21:46,933 - root - INFO - step: 53475 loss: 16.3839 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-10 01:21:46,933 - root - INFO - lr: 1.5227e-04 gnorm: 0.80 [11:46:46<10:14:55] +[titan] 2025-07-10 01:21:50,844 - root - INFO - step: 53480 loss: 16.4724 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 13.9347 +[titan] 2025-07-10 01:21:50,844 - root - INFO - lr: 1.5225e-04 gnorm: 0.78 [11:46:50<10:14:51] +[titan] 2025-07-10 01:21:54,764 - root - INFO - step: 53485 loss: 16.4916 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 13.9380 +[titan] 2025-07-10 01:21:54,765 - root - INFO - lr: 1.5223e-04 gnorm: 0.87 [11:46:54<10:14:47] +[titan] 2025-07-10 01:21:58,689 - root - INFO - step: 53490 loss: 16.4828 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-10 01:21:58,689 - root - INFO - lr: 1.5221e-04 gnorm: 0.81 [11:46:58<10:14:43] +[titan] 2025-07-10 01:22:02,607 - root - INFO - step: 53495 loss: 16.5468 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5378 global_avg_mtp_loss: 14.0090 +[titan] 2025-07-10 01:22:02,607 - root - INFO - lr: 1.5219e-04 gnorm: 0.83 [11:47:02<10:14:39] +[titan] 2025-07-10 01:22:05,730 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:22:06,518 - root - INFO - step: 53500 loss: 16.3951 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8860 +[titan] 2025-07-10 01:22:06,518 - root - INFO - lr: 1.5217e-04 gnorm: 0.84 [11:47:06<10:14:35] +[titan] 2025-07-10 01:22:10,434 - root - INFO - step: 53505 loss: 16.6266 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0784 +[titan] 2025-07-10 01:22:10,434 - root - INFO - lr: 1.5214e-04 gnorm: 0.78 [11:47:10<10:14:31] +[titan] 2025-07-10 01:22:14,332 - root - INFO - step: 53510 loss: 16.3872 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8782 +[titan] 2025-07-10 01:22:14,333 - root - INFO - lr: 1.5212e-04 gnorm: 0.77 [11:47:14<10:14:27] +[titan] 2025-07-10 01:22:18,234 - root - INFO - step: 53515 loss: 16.5674 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 14.0168 +[titan] 2025-07-10 01:22:18,234 - root - INFO - lr: 1.5210e-04 gnorm: 0.78 [11:47:17<10:14:23] +[titan] 2025-07-10 01:22:22,177 - root - INFO - step: 53520 loss: 17.0759 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.6640 global_avg_mtp_loss: 14.4119 +[titan] 2025-07-10 01:22:22,177 - root - INFO - lr: 1.5208e-04 gnorm: 0.89 [11:47:21<10:14:19] +[titan] 2025-07-10 01:22:26,144 - root - INFO - step: 53525 loss: 16.4944 memory: 44.58GiB(31.99%) tps: 82,600 tflops: 285.07 mfu: 28.82% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9526 +[titan] 2025-07-10 01:22:26,145 - root - INFO - lr: 1.5206e-04 gnorm: 0.79 [11:47:25<10:14:15] +[titan] 2025-07-10 01:22:30,087 - root - INFO - step: 53530 loss: 16.7413 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5803 global_avg_mtp_loss: 14.1610 +[titan] 2025-07-10 01:22:30,087 - root - INFO - lr: 1.5204e-04 gnorm: 0.80 [11:47:29<10:14:11] +[titan] 2025-07-10 01:22:34,018 - root - INFO - step: 53535 loss: 16.5234 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9847 +[titan] 2025-07-10 01:22:34,018 - root - INFO - lr: 1.5202e-04 gnorm: 0.89 [11:47:33<10:14:07] +[titan] 2025-07-10 01:22:37,945 - root - INFO - step: 53540 loss: 16.3983 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.8666 +[titan] 2025-07-10 01:22:37,946 - root - INFO - lr: 1.5199e-04 gnorm: 0.91 [11:47:37<10:14:03] +[titan] 2025-07-10 01:22:41,905 - root - INFO - step: 53545 loss: 16.5291 memory: 44.58GiB(31.99%) tps: 82,755 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5431 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-10 01:22:41,906 - root - INFO - lr: 1.5197e-04 gnorm: 0.82 [11:47:41<10:13:59] +[titan] 2025-07-10 01:22:45,035 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:22:45,821 - root - INFO - step: 53550 loss: 16.6561 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0939 +[titan] 2025-07-10 01:22:45,821 - root - INFO - lr: 1.5195e-04 gnorm: 0.85 [11:47:45<10:13:55] +[titan] 2025-07-10 01:22:49,759 - root - INFO - step: 53555 loss: 16.4840 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9427 +[titan] 2025-07-10 01:22:49,760 - root - INFO - lr: 1.5193e-04 gnorm: 0.82 [11:47:49<10:13:51] +[titan] 2025-07-10 01:22:53,667 - root - INFO - step: 53560 loss: 16.3495 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.8359 +[titan] 2025-07-10 01:22:53,668 - root - INFO - lr: 1.5191e-04 gnorm: 0.80 [11:47:53<10:13:47] +[titan] 2025-07-10 01:22:57,589 - root - INFO - step: 53565 loss: 16.8334 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6013 global_avg_mtp_loss: 14.2321 +[titan] 2025-07-10 01:22:57,589 - root - INFO - lr: 1.5189e-04 gnorm: 0.82 [11:47:57<10:13:43] +[titan] 2025-07-10 01:23:01,511 - root - INFO - step: 53570 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.9211 +[titan] 2025-07-10 01:23:01,511 - root - INFO - lr: 1.5187e-04 gnorm: 0.83 [11:48:01<10:13:39] +[titan] 2025-07-10 01:23:05,426 - root - INFO - step: 53575 loss: 16.2480 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7580 +[titan] 2025-07-10 01:23:05,426 - root - INFO - lr: 1.5185e-04 gnorm: 0.85 [11:48:05<10:13:35] +[titan] 2025-07-10 01:23:09,335 - root - INFO - step: 53580 loss: 16.5052 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9678 +[titan] 2025-07-10 01:23:09,335 - root - INFO - lr: 1.5182e-04 gnorm: 0.79 [11:48:09<10:13:31] +[titan] 2025-07-10 01:23:13,266 - root - INFO - step: 53585 loss: 16.2810 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.7806 +[titan] 2025-07-10 01:23:13,266 - root - INFO - lr: 1.5180e-04 gnorm: 0.84 [11:48:12<10:13:27] +[titan] 2025-07-10 01:23:17,212 - root - INFO - step: 53590 loss: 16.5898 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 14.0374 +[titan] 2025-07-10 01:23:17,212 - root - INFO - lr: 1.5178e-04 gnorm: 0.83 [11:48:16<10:13:23] +[titan] 2025-07-10 01:23:21,147 - root - INFO - step: 53595 loss: 16.4819 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5449 global_avg_mtp_loss: 13.9370 +[titan] 2025-07-10 01:23:21,147 - root - INFO - lr: 1.5176e-04 gnorm: 0.83 [11:48:20<10:13:19] +[titan] 2025-07-10 01:23:24,252 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:23:25,053 - root - INFO - step: 53600 loss: 16.2949 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.7779 +[titan] 2025-07-10 01:23:25,054 - root - INFO - lr: 1.5174e-04 gnorm: 0.86 [11:48:24<10:13:15] +[titan] 2025-07-10 01:23:29,004 - root - INFO - step: 53605 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 82,952 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.8746 +[titan] 2025-07-10 01:23:29,004 - root - INFO - lr: 1.5172e-04 gnorm: 0.87 [11:48:28<10:13:11] +[titan] 2025-07-10 01:23:32,936 - root - INFO - step: 53610 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.0797 +[titan] 2025-07-10 01:23:32,937 - root - INFO - lr: 1.5170e-04 gnorm: 0.88 [11:48:32<10:13:07] +[titan] 2025-07-10 01:23:36,827 - root - INFO - step: 53615 loss: 16.8285 memory: 44.58GiB(31.99%) tps: 84,234 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.6020 global_avg_mtp_loss: 14.2265 +[titan] 2025-07-10 01:23:36,827 - root - INFO - lr: 1.5168e-04 gnorm: 0.93 [11:48:36<10:13:03] +[titan] 2025-07-10 01:23:40,756 - root - INFO - step: 53620 loss: 16.6003 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0417 +[titan] 2025-07-10 01:23:40,756 - root - INFO - lr: 1.5165e-04 gnorm: 0.86 [11:48:40<10:12:59] +[titan] 2025-07-10 01:23:44,686 - root - INFO - step: 53625 loss: 16.7102 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5706 global_avg_mtp_loss: 14.1396 +[titan] 2025-07-10 01:23:44,686 - root - INFO - lr: 1.5163e-04 gnorm: 0.83 [11:48:44<10:12:55] +[titan] 2025-07-10 01:23:48,594 - root - INFO - step: 53630 loss: 16.4178 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.8776 +[titan] 2025-07-10 01:23:48,594 - root - INFO - lr: 1.5161e-04 gnorm: 0.90 [11:48:48<10:12:51] +[titan] 2025-07-10 01:23:52,541 - root - INFO - step: 53635 loss: 16.4378 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9135 +[titan] 2025-07-10 01:23:52,542 - root - INFO - lr: 1.5159e-04 gnorm: 0.81 [11:48:52<10:12:47] +[titan] 2025-07-10 01:23:56,452 - root - INFO - step: 53640 loss: 16.3552 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8375 +[titan] 2025-07-10 01:23:56,452 - root - INFO - lr: 1.5157e-04 gnorm: 0.83 [11:48:56<10:12:43] +[titan] 2025-07-10 01:24:00,354 - root - INFO - step: 53645 loss: 16.4981 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9530 +[titan] 2025-07-10 01:24:00,354 - root - INFO - lr: 1.5155e-04 gnorm: 0.80 [11:49:00<10:12:39] +[titan] 2025-07-10 01:24:03,469 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:24:04,261 - root - INFO - step: 53650 loss: 16.6897 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.1214 +[titan] 2025-07-10 01:24:04,261 - root - INFO - lr: 1.5153e-04 gnorm: 0.87 [11:49:03<10:12:35] +[titan] 2025-07-10 01:24:08,160 - root - INFO - step: 53655 loss: 16.8649 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5959 global_avg_mtp_loss: 14.2690 +[titan] 2025-07-10 01:24:08,161 - root - INFO - lr: 1.5150e-04 gnorm: 0.90 [11:49:07<10:12:31] +[titan] 2025-07-10 01:24:12,070 - root - INFO - step: 53660 loss: 16.6032 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0511 +[titan] 2025-07-10 01:24:12,070 - root - INFO - lr: 1.5148e-04 gnorm: 0.91 [11:49:11<10:12:27] +[titan] 2025-07-10 01:24:15,994 - root - INFO - step: 53665 loss: 16.3240 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8245 +[titan] 2025-07-10 01:24:15,994 - root - INFO - lr: 1.5146e-04 gnorm: 0.79 [11:49:15<10:12:23] +[titan] 2025-07-10 01:24:19,942 - root - INFO - step: 53670 loss: 16.5694 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0140 +[titan] 2025-07-10 01:24:19,943 - root - INFO - lr: 1.5144e-04 gnorm: 0.85 [11:49:19<10:12:19] +[titan] 2025-07-10 01:24:23,859 - root - INFO - step: 53675 loss: 16.7246 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5646 global_avg_mtp_loss: 14.1600 +[titan] 2025-07-10 01:24:23,859 - root - INFO - lr: 1.5142e-04 gnorm: 0.84 [11:49:23<10:12:15] +[titan] 2025-07-10 01:24:27,794 - root - INFO - step: 53680 loss: 16.2738 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7769 +[titan] 2025-07-10 01:24:27,794 - root - INFO - lr: 1.5140e-04 gnorm: 0.80 [11:49:27<10:12:11] +[titan] 2025-07-10 01:24:31,697 - root - INFO - step: 53685 loss: 16.6845 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.1116 +[titan] 2025-07-10 01:24:31,698 - root - INFO - lr: 1.5138e-04 gnorm: 0.79 [11:49:31<10:12:07] +[titan] 2025-07-10 01:24:35,599 - root - INFO - step: 53690 loss: 16.3843 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.8665 +[titan] 2025-07-10 01:24:35,599 - root - INFO - lr: 1.5136e-04 gnorm: 0.86 [11:49:35<10:12:03] +[titan] 2025-07-10 01:24:39,518 - root - INFO - step: 53695 loss: 16.5374 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 14.0035 +[titan] 2025-07-10 01:24:39,518 - root - INFO - lr: 1.5133e-04 gnorm: 0.80 [11:49:39<10:11:59] +[titan] 2025-07-10 01:24:42,639 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:24:43,425 - root - INFO - step: 53700 loss: 16.4347 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9123 +[titan] 2025-07-10 01:24:43,425 - root - INFO - lr: 1.5131e-04 gnorm: 0.79 [11:49:43<10:11:55] +[titan] 2025-07-10 01:24:47,365 - root - INFO - step: 53705 loss: 16.5168 memory: 44.58GiB(31.99%) tps: 83,174 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9743 +[titan] 2025-07-10 01:24:47,365 - root - INFO - lr: 1.5129e-04 gnorm: 0.79 [11:49:47<10:11:51] +[titan] 2025-07-10 01:24:51,308 - root - INFO - step: 53710 loss: 16.4252 memory: 44.58GiB(31.99%) tps: 83,113 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.9075 +[titan] 2025-07-10 01:24:51,308 - root - INFO - lr: 1.5127e-04 gnorm: 0.81 [11:49:51<10:11:47] +[titan] 2025-07-10 01:24:55,224 - root - INFO - step: 53715 loss: 16.3449 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-10 01:24:55,225 - root - INFO - lr: 1.5125e-04 gnorm: 0.86 [11:49:54<10:11:43] +[titan] 2025-07-10 01:24:59,121 - root - INFO - step: 53720 loss: 16.4648 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9365 +[titan] 2025-07-10 01:24:59,121 - root - INFO - lr: 1.5123e-04 gnorm: 0.84 [11:49:58<10:11:39] +[titan] 2025-07-10 01:25:03,050 - root - INFO - step: 53725 loss: 16.4286 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9020 +[titan] 2025-07-10 01:25:03,051 - root - INFO - lr: 1.5121e-04 gnorm: 0.83 [11:50:02<10:11:35] +[titan] 2025-07-10 01:25:06,952 - root - INFO - step: 53730 loss: 16.6611 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-10 01:25:06,953 - root - INFO - lr: 1.5118e-04 gnorm: 0.82 [11:50:06<10:11:31] +[titan] 2025-07-10 01:25:10,853 - root - INFO - step: 53735 loss: 16.3705 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.8554 +[titan] 2025-07-10 01:25:10,854 - root - INFO - lr: 1.5116e-04 gnorm: 0.83 [11:50:10<10:11:27] +[titan] 2025-07-10 01:25:14,807 - root - INFO - step: 53740 loss: 16.3010 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.7980 +[titan] 2025-07-10 01:25:14,808 - root - INFO - lr: 1.5114e-04 gnorm: 0.80 [11:50:14<10:11:23] +[titan] 2025-07-10 01:25:18,853 - root - INFO - step: 53745 loss: 16.2840 memory: 44.58GiB(31.99%) tps: 81,006 tflops: 279.57 mfu: 28.27% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7913 +[titan] 2025-07-10 01:25:18,853 - root - INFO - lr: 1.5112e-04 gnorm: 0.83 [11:50:18<10:11:19] +[titan] 2025-07-10 01:25:22,018 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:25:22,810 - root - INFO - step: 53750 loss: 16.6727 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.1058 +[titan] 2025-07-10 01:25:22,810 - root - INFO - lr: 1.5110e-04 gnorm: 0.92 [11:50:22<10:11:15] +[titan] 2025-07-10 01:25:26,719 - root - INFO - step: 53755 loss: 16.4662 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9397 +[titan] 2025-07-10 01:25:26,720 - root - INFO - lr: 1.5108e-04 gnorm: 0.82 [11:50:26<10:11:11] +[titan] 2025-07-10 01:25:30,714 - root - INFO - step: 53760 loss: 16.4497 memory: 44.58GiB(31.99%) tps: 82,045 tflops: 283.15 mfu: 28.63% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9121 +[titan] 2025-07-10 01:25:30,714 - root - INFO - lr: 1.5106e-04 gnorm: 0.84 [11:50:30<10:11:07] +[titan] 2025-07-10 01:25:30,870 - root - INFO - Dumping profiler traces at step 53760 +[titan] 2025-07-10 01:25:30,902 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:25:34,830 - root - INFO - step: 53765 loss: 16.6795 memory: 44.58GiB(31.99%) tps: 79,623 tflops: 274.79 mfu: 27.78% global_avg_ntp_loss: 2.5838 global_avg_mtp_loss: 14.0957 +[titan] 2025-07-10 01:25:34,830 - root - INFO - lr: 1.5104e-04 gnorm: 0.82 [11:50:34<10:11:03] +[titan] 2025-07-10 01:25:38,807 - root - INFO - step: 53770 loss: 16.5356 memory: 44.58GiB(31.99%) tps: 82,385 tflops: 284.32 mfu: 28.75% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9927 +[titan] 2025-07-10 01:25:38,808 - root - INFO - lr: 1.5101e-04 gnorm: 0.81 [11:50:38<10:10:59] +[titan] 2025-07-10 01:25:42,722 - root - INFO - step: 53775 loss: 16.3680 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8666 +[titan] 2025-07-10 01:25:42,723 - root - INFO - lr: 1.5099e-04 gnorm: 0.81 [11:50:42<10:10:55] +[titan] 2025-07-10 01:25:46,671 - root - INFO - step: 53780 loss: 16.5263 memory: 44.58GiB(31.99%) tps: 82,985 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9804 +[titan] 2025-07-10 01:25:46,672 - root - INFO - lr: 1.5097e-04 gnorm: 0.85 [11:50:46<10:10:51] +[titan] 2025-07-10 01:25:50,611 - root - INFO - step: 53785 loss: 16.7713 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5839 global_avg_mtp_loss: 14.1874 +[titan] 2025-07-10 01:25:50,611 - root - INFO - lr: 1.5095e-04 gnorm: 0.83 [11:50:50<10:10:47] +[titan] 2025-07-10 01:25:54,521 - root - INFO - step: 53790 loss: 16.5232 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 13.9796 +[titan] 2025-07-10 01:25:54,522 - root - INFO - lr: 1.5093e-04 gnorm: 0.81 [11:50:54<10:10:43] +[titan] 2025-07-10 01:25:58,469 - root - INFO - step: 53795 loss: 16.4273 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9073 +[titan] 2025-07-10 01:25:58,470 - root - INFO - lr: 1.5091e-04 gnorm: 0.88 [11:50:58<10:10:39] +[titan] 2025-07-10 01:26:01,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:26:02,363 - root - INFO - step: 53800 loss: 16.3629 memory: 44.58GiB(31.99%) tps: 84,172 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-10 01:26:02,363 - root - INFO - lr: 1.5089e-04 gnorm: 0.92 [11:51:02<10:10:35] +[titan] 2025-07-10 01:26:06,272 - root - INFO - step: 53805 loss: 16.4477 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9238 +[titan] 2025-07-10 01:26:06,272 - root - INFO - lr: 1.5087e-04 gnorm: 0.80 [11:51:05<10:10:31] +[titan] 2025-07-10 01:26:10,265 - root - INFO - step: 53810 loss: 16.3481 memory: 44.58GiB(31.99%) tps: 82,059 tflops: 283.20 mfu: 28.63% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8341 +[titan] 2025-07-10 01:26:10,266 - root - INFO - lr: 1.5084e-04 gnorm: 0.83 [11:51:09<10:10:27] +[titan] 2025-07-10 01:26:14,175 - root - INFO - step: 53815 loss: 16.6011 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 14.0454 +[titan] 2025-07-10 01:26:14,175 - root - INFO - lr: 1.5082e-04 gnorm: 0.83 [11:51:13<10:10:23] +[titan] 2025-07-10 01:26:18,102 - root - INFO - step: 53820 loss: 16.2837 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.7848 +[titan] 2025-07-10 01:26:18,102 - root - INFO - lr: 1.5080e-04 gnorm: 0.81 [11:51:17<10:10:19] +[titan] 2025-07-10 01:26:21,997 - root - INFO - step: 53825 loss: 16.2995 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.7981 +[titan] 2025-07-10 01:26:21,998 - root - INFO - lr: 1.5078e-04 gnorm: 0.84 [11:51:21<10:10:15] +[titan] 2025-07-10 01:26:25,928 - root - INFO - step: 53830 loss: 16.2176 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7320 +[titan] 2025-07-10 01:26:25,928 - root - INFO - lr: 1.5076e-04 gnorm: 0.87 [11:51:25<10:10:11] +[titan] 2025-07-10 01:26:29,862 - root - INFO - step: 53835 loss: 16.6949 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5615 global_avg_mtp_loss: 14.1333 +[titan] 2025-07-10 01:26:29,863 - root - INFO - lr: 1.5074e-04 gnorm: 0.84 [11:51:29<10:10:07] +[titan] 2025-07-10 01:26:33,770 - root - INFO - step: 53840 loss: 16.4205 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.8970 +[titan] 2025-07-10 01:26:33,770 - root - INFO - lr: 1.5072e-04 gnorm: 0.84 [11:51:33<10:10:03] +[titan] 2025-07-10 01:26:37,678 - root - INFO - step: 53845 loss: 16.6315 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.0578 +[titan] 2025-07-10 01:26:37,679 - root - INFO - lr: 1.5069e-04 gnorm: 0.85 [11:51:37<10:09:59] +[titan] 2025-07-10 01:26:40,806 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:26:41,594 - root - INFO - step: 53850 loss: 16.5863 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0381 +[titan] 2025-07-10 01:26:41,594 - root - INFO - lr: 1.5067e-04 gnorm: 0.83 [11:51:41<10:09:55] +[titan] 2025-07-10 01:26:45,507 - root - INFO - step: 53855 loss: 16.5951 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5611 global_avg_mtp_loss: 14.0340 +[titan] 2025-07-10 01:26:45,507 - root - INFO - lr: 1.5065e-04 gnorm: 0.81 [11:51:45<10:09:51] +[titan] 2025-07-10 01:26:49,429 - root - INFO - step: 53860 loss: 16.6147 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 14.0522 +[titan] 2025-07-10 01:26:49,430 - root - INFO - lr: 1.5063e-04 gnorm: 0.83 [11:51:49<10:09:47] +[titan] 2025-07-10 01:26:53,347 - root - INFO - step: 53865 loss: 16.6230 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0589 +[titan] 2025-07-10 01:26:53,347 - root - INFO - lr: 1.5061e-04 gnorm: 0.88 [11:51:53<10:09:43] +[titan] 2025-07-10 01:26:57,259 - root - INFO - step: 53870 loss: 16.1848 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7110 +[titan] 2025-07-10 01:26:57,259 - root - INFO - lr: 1.5059e-04 gnorm: 0.85 [11:51:56<10:09:39] +[titan] 2025-07-10 01:27:01,194 - root - INFO - step: 53875 loss: 16.5152 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.9797 +[titan] 2025-07-10 01:27:01,195 - root - INFO - lr: 1.5057e-04 gnorm: 0.82 [11:52:00<10:09:35] +[titan] 2025-07-10 01:27:05,113 - root - INFO - step: 53880 loss: 16.4582 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9320 +[titan] 2025-07-10 01:27:05,113 - root - INFO - lr: 1.5055e-04 gnorm: 0.79 [11:52:04<10:09:31] +[titan] 2025-07-10 01:27:09,059 - root - INFO - step: 53885 loss: 16.6120 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.0411 +[titan] 2025-07-10 01:27:09,059 - root - INFO - lr: 1.5052e-04 gnorm: 0.85 [11:52:08<10:09:27] +[titan] 2025-07-10 01:27:13,006 - root - INFO - step: 53890 loss: 16.2798 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.7664 +[titan] 2025-07-10 01:27:13,006 - root - INFO - lr: 1.5050e-04 gnorm: 0.96 [11:52:12<10:09:23] +[titan] 2025-07-10 01:27:16,936 - root - INFO - step: 53895 loss: 16.3236 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.8337 +[titan] 2025-07-10 01:27:16,936 - root - INFO - lr: 1.5048e-04 gnorm: 0.82 [11:52:16<10:09:19] +[titan] 2025-07-10 01:27:20,080 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:27:20,873 - root - INFO - step: 53900 loss: 16.3130 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8116 +[titan] 2025-07-10 01:27:20,873 - root - INFO - lr: 1.5046e-04 gnorm: 0.88 [11:52:20<10:09:15] +[titan] 2025-07-10 01:27:24,806 - root - INFO - step: 53905 loss: 16.3134 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8048 +[titan] 2025-07-10 01:27:24,806 - root - INFO - lr: 1.5044e-04 gnorm: 0.84 [11:52:24<10:09:11] +[titan] 2025-07-10 01:27:28,722 - root - INFO - step: 53910 loss: 16.6997 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5820 global_avg_mtp_loss: 14.1178 +[titan] 2025-07-10 01:27:28,722 - root - INFO - lr: 1.5042e-04 gnorm: 0.90 [11:52:28<10:09:07] +[titan] 2025-07-10 01:27:32,805 - root - INFO - step: 53915 loss: 16.3794 memory: 44.58GiB(31.99%) tps: 80,257 tflops: 276.98 mfu: 28.01% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-10 01:27:32,806 - root - INFO - lr: 1.5040e-04 gnorm: 0.82 [11:52:32<10:09:03] +[titan] 2025-07-10 01:27:36,742 - root - INFO - step: 53920 loss: 16.1816 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.6904 +[titan] 2025-07-10 01:27:36,742 - root - INFO - lr: 1.5038e-04 gnorm: 0.81 [11:52:36<10:08:59] +[titan] 2025-07-10 01:27:40,663 - root - INFO - step: 53925 loss: 16.5331 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9959 +[titan] 2025-07-10 01:27:40,663 - root - INFO - lr: 1.5035e-04 gnorm: 0.83 [11:52:40<10:08:55] +[titan] 2025-07-10 01:27:44,568 - root - INFO - step: 53930 loss: 16.1766 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 01:27:44,568 - root - INFO - lr: 1.5033e-04 gnorm: 0.85 [11:52:44<10:08:51] +[titan] 2025-07-10 01:27:48,505 - root - INFO - step: 53935 loss: 16.4823 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9560 +[titan] 2025-07-10 01:27:48,505 - root - INFO - lr: 1.5031e-04 gnorm: 0.83 [11:52:48<10:08:47] +[titan] 2025-07-10 01:27:52,424 - root - INFO - step: 53940 loss: 16.3738 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8698 +[titan] 2025-07-10 01:27:52,424 - root - INFO - lr: 1.5029e-04 gnorm: 0.84 [11:52:52<10:08:43] +[titan] 2025-07-10 01:27:56,346 - root - INFO - step: 53945 loss: 16.7553 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5836 global_avg_mtp_loss: 14.1717 +[titan] 2025-07-10 01:27:56,346 - root - INFO - lr: 1.5027e-04 gnorm: 0.85 [11:52:56<10:08:39] +[titan] 2025-07-10 01:27:59,644 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:28:00,433 - root - INFO - step: 53950 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 80,177 tflops: 276.70 mfu: 27.98% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0614 +[titan] 2025-07-10 01:28:00,433 - root - INFO - lr: 1.5025e-04 gnorm: 0.89 [11:53:00<10:08:35] +[titan] 2025-07-10 01:28:04,387 - root - INFO - step: 53955 loss: 16.4105 memory: 44.58GiB(31.99%) tps: 82,886 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8915 +[titan] 2025-07-10 01:28:04,387 - root - INFO - lr: 1.5023e-04 gnorm: 0.86 [11:53:04<10:08:31] +[titan] 2025-07-10 01:28:08,310 - root - INFO - step: 53960 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 01:28:08,310 - root - INFO - lr: 1.5021e-04 gnorm: 0.84 [11:53:07<10:08:27] +[titan] 2025-07-10 01:28:12,228 - root - INFO - step: 53965 loss: 16.2975 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 01:28:12,228 - root - INFO - lr: 1.5018e-04 gnorm: 0.93 [11:53:11<10:08:23] +[titan] 2025-07-10 01:28:16,167 - root - INFO - step: 53970 loss: 16.5253 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5564 global_avg_mtp_loss: 13.9689 +[titan] 2025-07-10 01:28:16,168 - root - INFO - lr: 1.5016e-04 gnorm: 0.82 [11:53:15<10:08:19] +[titan] 2025-07-10 01:28:20,087 - root - INFO - step: 53975 loss: 16.6389 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0724 +[titan] 2025-07-10 01:28:20,087 - root - INFO - lr: 1.5014e-04 gnorm: 0.87 [11:53:19<10:08:15] +[titan] 2025-07-10 01:28:24,009 - root - INFO - step: 53980 loss: 16.8009 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5987 global_avg_mtp_loss: 14.2023 +[titan] 2025-07-10 01:28:24,009 - root - INFO - lr: 1.5012e-04 gnorm: 0.82 [11:53:23<10:08:11] +[titan] 2025-07-10 01:28:27,938 - root - INFO - step: 53985 loss: 16.6907 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.1127 +[titan] 2025-07-10 01:28:27,939 - root - INFO - lr: 1.5010e-04 gnorm: 0.79 [11:53:27<10:08:07] +[titan] 2025-07-10 01:28:31,862 - root - INFO - step: 53990 loss: 16.2952 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.8088 +[titan] 2025-07-10 01:28:31,862 - root - INFO - lr: 1.5008e-04 gnorm: 0.84 [11:53:31<10:08:03] +[titan] 2025-07-10 01:28:35,781 - root - INFO - step: 53995 loss: 16.3258 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8126 +[titan] 2025-07-10 01:28:35,781 - root - INFO - lr: 1.5006e-04 gnorm: 0.82 [11:53:35<10:07:59] +[titan] 2025-07-10 01:28:38,904 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:28:39,690 - root - INFO - step: 54000 loss: 16.4600 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9358 +[titan] 2025-07-10 01:28:39,690 - root - INFO - lr: 1.5003e-04 gnorm: 0.84 [11:53:39<10:07:55] +[titan] 2025-07-10 01:28:43,602 - root - INFO - step: 54005 loss: 16.5352 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-10 01:28:43,602 - root - INFO - lr: 1.5001e-04 gnorm: 0.84 [11:53:43<10:07:51] +[titan] 2025-07-10 01:28:47,539 - root - INFO - step: 54010 loss: 16.5580 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 14.0102 +[titan] 2025-07-10 01:28:47,539 - root - INFO - lr: 1.4999e-04 gnorm: 0.87 [11:53:47<10:07:47] +[titan] 2025-07-10 01:28:51,442 - root - INFO - step: 54015 loss: 16.4055 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5355 global_avg_mtp_loss: 13.8699 +[titan] 2025-07-10 01:28:51,442 - root - INFO - lr: 1.4997e-04 gnorm: 0.86 [11:53:51<10:07:43] +[titan] 2025-07-10 01:28:55,353 - root - INFO - step: 54020 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.1383 +[titan] 2025-07-10 01:28:55,353 - root - INFO - lr: 1.4995e-04 gnorm: 0.87 [11:53:55<10:07:39] +[titan] 2025-07-10 01:28:59,310 - root - INFO - step: 54025 loss: 16.2797 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7870 +[titan] 2025-07-10 01:28:59,310 - root - INFO - lr: 1.4993e-04 gnorm: 1.11 [11:53:58<10:07:35] +[titan] 2025-07-10 01:29:03,228 - root - INFO - step: 54030 loss: 16.4196 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-10 01:29:03,228 - root - INFO - lr: 1.4991e-04 gnorm: 0.84 [11:54:02<10:07:31] +[titan] 2025-07-10 01:29:07,143 - root - INFO - step: 54035 loss: 16.5734 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 14.0291 +[titan] 2025-07-10 01:29:07,143 - root - INFO - lr: 1.4989e-04 gnorm: 0.84 [11:54:06<10:07:27] +[titan] 2025-07-10 01:29:11,076 - root - INFO - step: 54040 loss: 16.7077 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5733 global_avg_mtp_loss: 14.1344 +[titan] 2025-07-10 01:29:11,077 - root - INFO - lr: 1.4986e-04 gnorm: 0.82 [11:54:10<10:07:23] +[titan] 2025-07-10 01:29:15,002 - root - INFO - step: 54045 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 01:29:15,003 - root - INFO - lr: 1.4984e-04 gnorm: 0.83 [11:54:14<10:07:19] +[titan] 2025-07-10 01:29:18,114 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:29:18,903 - root - INFO - step: 54050 loss: 16.5179 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.9826 +[titan] 2025-07-10 01:29:18,903 - root - INFO - lr: 1.4982e-04 gnorm: 0.84 [11:54:18<10:07:15] +[titan] 2025-07-10 01:29:22,809 - root - INFO - step: 54055 loss: 16.4070 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.8795 +[titan] 2025-07-10 01:29:22,810 - root - INFO - lr: 1.4980e-04 gnorm: 0.88 [11:54:22<10:07:11] +[titan] 2025-07-10 01:29:26,778 - root - INFO - step: 54060 loss: 16.5383 memory: 44.58GiB(31.99%) tps: 82,582 tflops: 285.01 mfu: 28.82% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-10 01:29:26,778 - root - INFO - lr: 1.4978e-04 gnorm: 0.82 [11:54:26<10:07:07] +[titan] 2025-07-10 01:29:30,700 - root - INFO - step: 54065 loss: 16.5879 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0369 +[titan] 2025-07-10 01:29:30,700 - root - INFO - lr: 1.4976e-04 gnorm: 0.85 [11:54:30<10:07:03] +[titan] 2025-07-10 01:29:34,619 - root - INFO - step: 54070 loss: 16.5318 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9875 +[titan] 2025-07-10 01:29:34,619 - root - INFO - lr: 1.4974e-04 gnorm: 0.87 [11:54:34<10:06:59] +[titan] 2025-07-10 01:29:38,554 - root - INFO - step: 54075 loss: 16.3384 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8300 +[titan] 2025-07-10 01:29:38,554 - root - INFO - lr: 1.4972e-04 gnorm: 0.86 [11:54:38<10:06:55] +[titan] 2025-07-10 01:29:42,444 - root - INFO - step: 54080 loss: 16.5324 memory: 44.58GiB(31.99%) tps: 84,235 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 13.9853 +[titan] 2025-07-10 01:29:42,445 - root - INFO - lr: 1.4969e-04 gnorm: 0.82 [11:54:42<10:06:51] +[titan] 2025-07-10 01:29:46,353 - root - INFO - step: 54085 loss: 16.3093 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8002 +[titan] 2025-07-10 01:29:46,353 - root - INFO - lr: 1.4967e-04 gnorm: 0.93 [11:54:46<10:06:47] +[titan] 2025-07-10 01:29:50,266 - root - INFO - step: 54090 loss: 16.5693 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 14.0230 +[titan] 2025-07-10 01:29:50,266 - root - INFO - lr: 1.4965e-04 gnorm: 0.83 [11:54:49<10:06:43] +[titan] 2025-07-10 01:29:54,180 - root - INFO - step: 54095 loss: 16.6031 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0485 +[titan] 2025-07-10 01:29:54,180 - root - INFO - lr: 1.4963e-04 gnorm: 0.86 [11:54:53<10:06:39] +[titan] 2025-07-10 01:29:57,316 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:29:58,121 - root - INFO - step: 54100 loss: 16.3869 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-10 01:29:58,121 - root - INFO - lr: 1.4961e-04 gnorm: 0.81 [11:54:57<10:06:35] +[titan] 2025-07-10 01:30:02,024 - root - INFO - step: 54105 loss: 16.5039 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5355 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-10 01:30:02,024 - root - INFO - lr: 1.4959e-04 gnorm: 0.83 [11:55:01<10:06:31] +[titan] 2025-07-10 01:30:05,940 - root - INFO - step: 54110 loss: 16.4492 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9184 +[titan] 2025-07-10 01:30:05,940 - root - INFO - lr: 1.4957e-04 gnorm: 0.86 [11:55:05<10:06:27] +[titan] 2025-07-10 01:30:09,862 - root - INFO - step: 54115 loss: 16.6065 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5595 global_avg_mtp_loss: 14.0471 +[titan] 2025-07-10 01:30:09,862 - root - INFO - lr: 1.4955e-04 gnorm: 0.85 [11:55:09<10:06:23] +[titan] 2025-07-10 01:30:13,779 - root - INFO - step: 54120 loss: 16.5418 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9927 +[titan] 2025-07-10 01:30:13,779 - root - INFO - lr: 1.4952e-04 gnorm: 0.93 [11:55:13<10:06:19] +[titan] 2025-07-10 01:30:17,687 - root - INFO - step: 54125 loss: 15.9372 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4657 global_avg_mtp_loss: 13.4715 +[titan] 2025-07-10 01:30:17,687 - root - INFO - lr: 1.4950e-04 gnorm: 0.84 [11:55:17<10:06:15] +[titan] 2025-07-10 01:30:21,623 - root - INFO - step: 54130 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.8385 +[titan] 2025-07-10 01:30:21,623 - root - INFO - lr: 1.4948e-04 gnorm: 0.90 [11:55:21<10:06:11] +[titan] 2025-07-10 01:30:25,732 - root - INFO - step: 54135 loss: 16.2456 memory: 44.58GiB(31.99%) tps: 79,747 tflops: 275.22 mfu: 27.83% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7606 +[titan] 2025-07-10 01:30:25,732 - root - INFO - lr: 1.4946e-04 gnorm: 0.84 [11:55:25<10:06:07] +[titan] 2025-07-10 01:30:29,669 - root - INFO - step: 54140 loss: 16.5436 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9962 +[titan] 2025-07-10 01:30:29,669 - root - INFO - lr: 1.4944e-04 gnorm: 0.84 [11:55:29<10:06:03] +[titan] 2025-07-10 01:30:33,598 - root - INFO - step: 54145 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.7900 +[titan] 2025-07-10 01:30:33,598 - root - INFO - lr: 1.4942e-04 gnorm: 0.80 [11:55:33<10:05:59] +[titan] 2025-07-10 01:30:36,733 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:30:37,551 - root - INFO - step: 54150 loss: 16.6416 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5601 global_avg_mtp_loss: 14.0815 +[titan] 2025-07-10 01:30:37,552 - root - INFO - lr: 1.4940e-04 gnorm: 0.83 [11:55:37<10:05:55] +[titan] 2025-07-10 01:30:41,509 - root - INFO - step: 54155 loss: 16.3319 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8153 +[titan] 2025-07-10 01:30:41,509 - root - INFO - lr: 1.4938e-04 gnorm: 0.89 [11:55:41<10:05:51] +[titan] 2025-07-10 01:30:45,448 - root - INFO - step: 54160 loss: 16.3271 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8258 +[titan] 2025-07-10 01:30:45,449 - root - INFO - lr: 1.4935e-04 gnorm: 0.85 [11:55:45<10:05:47] +[titan] 2025-07-10 01:30:49,343 - root - INFO - step: 54165 loss: 16.4436 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.9236 +[titan] 2025-07-10 01:30:49,343 - root - INFO - lr: 1.4933e-04 gnorm: 0.78 [11:55:49<10:05:43] +[titan] 2025-07-10 01:30:53,283 - root - INFO - step: 54170 loss: 16.4700 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-10 01:30:53,284 - root - INFO - lr: 1.4931e-04 gnorm: 0.93 [11:55:52<10:05:39] +[titan] 2025-07-10 01:30:57,218 - root - INFO - step: 54175 loss: 16.5897 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 14.0393 +[titan] 2025-07-10 01:30:57,219 - root - INFO - lr: 1.4929e-04 gnorm: 0.87 [11:55:56<10:05:35] +[titan] 2025-07-10 01:31:01,141 - root - INFO - step: 54180 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.8680 +[titan] 2025-07-10 01:31:01,141 - root - INFO - lr: 1.4927e-04 gnorm: 0.89 [11:56:00<10:05:31] +[titan] 2025-07-10 01:31:05,051 - root - INFO - step: 54185 loss: 16.5508 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 14.0157 +[titan] 2025-07-10 01:31:05,052 - root - INFO - lr: 1.4925e-04 gnorm: 0.90 [11:56:04<10:05:27] +[titan] 2025-07-10 01:31:08,978 - root - INFO - step: 54190 loss: 16.6168 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0648 +[titan] 2025-07-10 01:31:08,979 - root - INFO - lr: 1.4923e-04 gnorm: 0.86 [11:56:08<10:05:23] +[titan] 2025-07-10 01:31:12,887 - root - INFO - step: 54195 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.8791 +[titan] 2025-07-10 01:31:12,888 - root - INFO - lr: 1.4920e-04 gnorm: 0.78 [11:56:12<10:05:19] +[titan] 2025-07-10 01:31:16,008 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:31:16,800 - root - INFO - step: 54200 loss: 16.5885 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.0279 +[titan] 2025-07-10 01:31:16,800 - root - INFO - lr: 1.4918e-04 gnorm: 0.79 [11:56:16<10:05:15] +[titan] 2025-07-10 01:31:20,715 - root - INFO - step: 54205 loss: 16.5461 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 13.9928 +[titan] 2025-07-10 01:31:20,715 - root - INFO - lr: 1.4916e-04 gnorm: 0.84 [11:56:20<10:05:11] +[titan] 2025-07-10 01:31:24,625 - root - INFO - step: 54210 loss: 16.6439 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5765 global_avg_mtp_loss: 14.0674 +[titan] 2025-07-10 01:31:24,626 - root - INFO - lr: 1.4914e-04 gnorm: 0.90 [11:56:24<10:05:07] +[titan] 2025-07-10 01:31:28,536 - root - INFO - step: 54215 loss: 16.7357 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5851 global_avg_mtp_loss: 14.1505 +[titan] 2025-07-10 01:31:28,536 - root - INFO - lr: 1.4912e-04 gnorm: 0.82 [11:56:28<10:05:03] +[titan] 2025-07-10 01:31:32,470 - root - INFO - step: 54220 loss: 16.2070 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7219 +[titan] 2025-07-10 01:31:32,471 - root - INFO - lr: 1.4910e-04 gnorm: 0.85 [11:56:32<10:04:59] +[titan] 2025-07-10 01:31:36,417 - root - INFO - step: 54225 loss: 16.5407 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9994 +[titan] 2025-07-10 01:31:36,417 - root - INFO - lr: 1.4908e-04 gnorm: 0.90 [11:56:36<10:04:55] +[titan] 2025-07-10 01:31:40,328 - root - INFO - step: 54230 loss: 16.5772 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0186 +[titan] 2025-07-10 01:31:40,328 - root - INFO - lr: 1.4906e-04 gnorm: 0.86 [11:56:39<10:04:51] +[titan] 2025-07-10 01:31:44,272 - root - INFO - step: 54235 loss: 16.5119 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5306 global_avg_mtp_loss: 13.9812 +[titan] 2025-07-10 01:31:44,272 - root - INFO - lr: 1.4903e-04 gnorm: 0.83 [11:56:43<10:04:47] +[titan] 2025-07-10 01:31:48,192 - root - INFO - step: 54240 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5348 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 01:31:48,192 - root - INFO - lr: 1.4901e-04 gnorm: 0.81 [11:56:47<10:04:43] +[titan] 2025-07-10 01:31:52,120 - root - INFO - step: 54245 loss: 16.5348 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 13.9852 +[titan] 2025-07-10 01:31:52,120 - root - INFO - lr: 1.4899e-04 gnorm: 0.82 [11:56:51<10:04:39] +[titan] 2025-07-10 01:31:55,260 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:31:56,048 - root - INFO - step: 54250 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8693 +[titan] 2025-07-10 01:31:56,048 - root - INFO - lr: 1.4897e-04 gnorm: 0.76 [11:56:55<10:04:35] +[titan] 2025-07-10 01:31:59,950 - root - INFO - step: 54255 loss: 16.5232 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 13.9694 +[titan] 2025-07-10 01:31:59,950 - root - INFO - lr: 1.4895e-04 gnorm: 1.26 [11:56:59<10:04:31] +[titan] 2025-07-10 01:32:03,854 - root - INFO - step: 54260 loss: 16.5228 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5381 global_avg_mtp_loss: 13.9847 +[titan] 2025-07-10 01:32:03,854 - root - INFO - lr: 1.4893e-04 gnorm: 0.87 [11:57:03<10:04:27] +[titan] 2025-07-10 01:32:07,789 - root - INFO - step: 54265 loss: 16.6056 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0460 +[titan] 2025-07-10 01:32:07,789 - root - INFO - lr: 1.4891e-04 gnorm: 0.87 [11:57:07<10:04:23] +[titan] 2025-07-10 01:32:11,769 - root - INFO - step: 54270 loss: 16.4538 memory: 44.58GiB(31.99%) tps: 82,342 tflops: 284.18 mfu: 28.73% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9139 +[titan] 2025-07-10 01:32:11,769 - root - INFO - lr: 1.4889e-04 gnorm: 0.81 [11:57:11<10:04:19] +[titan] 2025-07-10 01:32:13,509 - root - INFO - Dumping profiler traces at step 54272 +[titan] 2025-07-10 01:32:13,542 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:32:15,937 - root - INFO - step: 54275 loss: 16.7566 memory: 44.58GiB(31.99%) tps: 78,631 tflops: 271.37 mfu: 27.44% global_avg_ntp_loss: 2.5925 global_avg_mtp_loss: 14.1641 +[titan] 2025-07-10 01:32:15,937 - root - INFO - lr: 1.4886e-04 gnorm: 0.83 [11:57:15<10:04:16] +[titan] 2025-07-10 01:32:19,865 - root - INFO - step: 54280 loss: 16.6587 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5696 global_avg_mtp_loss: 14.0891 +[titan] 2025-07-10 01:32:19,865 - root - INFO - lr: 1.4884e-04 gnorm: 0.86 [11:57:19<10:04:12] +[titan] 2025-07-10 01:32:23,818 - root - INFO - step: 54285 loss: 16.4230 memory: 44.58GiB(31.99%) tps: 82,905 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.8989 +[titan] 2025-07-10 01:32:23,818 - root - INFO - lr: 1.4882e-04 gnorm: 0.86 [11:57:23<10:04:08] +[titan] 2025-07-10 01:32:27,751 - root - INFO - step: 54290 loss: 16.5641 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 14.0181 +[titan] 2025-07-10 01:32:27,752 - root - INFO - lr: 1.4880e-04 gnorm: 0.83 [11:57:27<10:04:04] +[titan] 2025-07-10 01:32:31,670 - root - INFO - step: 54295 loss: 16.4904 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9565 +[titan] 2025-07-10 01:32:31,671 - root - INFO - lr: 1.4878e-04 gnorm: 0.83 [11:57:31<10:04:00] +[titan] 2025-07-10 01:32:34,796 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:32:35,589 - root - INFO - step: 54300 loss: 16.5518 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 14.0067 +[titan] 2025-07-10 01:32:35,589 - root - INFO - lr: 1.4876e-04 gnorm: 0.87 [11:57:35<10:03:56] +[titan] 2025-07-10 01:32:39,501 - root - INFO - step: 54305 loss: 16.6213 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0676 +[titan] 2025-07-10 01:32:39,501 - root - INFO - lr: 1.4874e-04 gnorm: 0.86 [11:57:39<10:03:52] +[titan] 2025-07-10 01:32:43,416 - root - INFO - step: 54310 loss: 16.5656 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5570 global_avg_mtp_loss: 14.0086 +[titan] 2025-07-10 01:32:43,416 - root - INFO - lr: 1.4872e-04 gnorm: 0.82 [11:57:43<10:03:48] +[titan] 2025-07-10 01:32:47,312 - root - INFO - step: 54315 loss: 16.1480 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6770 +[titan] 2025-07-10 01:32:47,312 - root - INFO - lr: 1.4869e-04 gnorm: 0.85 [11:57:46<10:03:44] +[titan] 2025-07-10 01:32:51,209 - root - INFO - step: 54320 loss: 16.6417 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5737 global_avg_mtp_loss: 14.0680 +[titan] 2025-07-10 01:32:51,210 - root - INFO - lr: 1.4867e-04 gnorm: 0.85 [11:57:50<10:03:40] +[titan] 2025-07-10 01:32:55,129 - root - INFO - step: 54325 loss: 16.3066 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.7961 +[titan] 2025-07-10 01:32:55,130 - root - INFO - lr: 1.4865e-04 gnorm: 0.79 [11:57:54<10:03:36] +[titan] 2025-07-10 01:32:59,053 - root - INFO - step: 54330 loss: 16.4875 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.9546 +[titan] 2025-07-10 01:32:59,053 - root - INFO - lr: 1.4863e-04 gnorm: 0.82 [11:57:58<10:03:32] +[titan] 2025-07-10 01:33:02,955 - root - INFO - step: 54335 loss: 16.6696 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.0980 +[titan] 2025-07-10 01:33:02,955 - root - INFO - lr: 1.4861e-04 gnorm: 0.80 [11:58:02<10:03:28] +[titan] 2025-07-10 01:33:06,869 - root - INFO - step: 54340 loss: 16.2671 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.7698 +[titan] 2025-07-10 01:33:06,870 - root - INFO - lr: 1.4859e-04 gnorm: 0.79 [11:58:06<10:03:24] +[titan] 2025-07-10 01:33:10,761 - root - INFO - step: 54345 loss: 16.5067 memory: 44.58GiB(31.99%) tps: 84,207 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9668 +[titan] 2025-07-10 01:33:10,761 - root - INFO - lr: 1.4857e-04 gnorm: 0.83 [11:58:10<10:03:20] +[titan] 2025-07-10 01:33:13,866 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:33:14,659 - root - INFO - step: 54350 loss: 16.5010 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5506 global_avg_mtp_loss: 13.9505 +[titan] 2025-07-10 01:33:14,660 - root - INFO - lr: 1.4855e-04 gnorm: 0.85 [11:58:14<10:03:16] +[titan] 2025-07-10 01:33:18,610 - root - INFO - step: 54355 loss: 16.4431 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9120 +[titan] 2025-07-10 01:33:18,610 - root - INFO - lr: 1.4852e-04 gnorm: 0.87 [11:58:18<10:03:12] +[titan] 2025-07-10 01:33:22,507 - root - INFO - step: 54360 loss: 16.3411 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8446 +[titan] 2025-07-10 01:33:22,507 - root - INFO - lr: 1.4850e-04 gnorm: 0.80 [11:58:22<10:03:08] +[titan] 2025-07-10 01:33:26,413 - root - INFO - step: 54365 loss: 16.5863 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0425 +[titan] 2025-07-10 01:33:26,413 - root - INFO - lr: 1.4848e-04 gnorm: 0.83 [11:58:26<10:03:04] +[titan] 2025-07-10 01:33:30,347 - root - INFO - step: 54370 loss: 16.6326 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-10 01:33:30,347 - root - INFO - lr: 1.4846e-04 gnorm: 0.78 [11:58:29<10:03:00] +[titan] 2025-07-10 01:33:34,298 - root - INFO - step: 54375 loss: 16.7808 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.1899 +[titan] 2025-07-10 01:33:34,298 - root - INFO - lr: 1.4844e-04 gnorm: 0.83 [11:58:33<10:02:56] +[titan] 2025-07-10 01:33:38,227 - root - INFO - step: 54380 loss: 16.2481 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7626 +[titan] 2025-07-10 01:33:38,227 - root - INFO - lr: 1.4842e-04 gnorm: 0.85 [11:58:37<10:02:52] +[titan] 2025-07-10 01:33:42,151 - root - INFO - step: 54385 loss: 16.5208 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5525 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-10 01:33:42,151 - root - INFO - lr: 1.4840e-04 gnorm: 0.86 [11:58:41<10:02:48] +[titan] 2025-07-10 01:33:46,109 - root - INFO - step: 54390 loss: 16.3779 memory: 44.58GiB(31.99%) tps: 82,787 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.8635 +[titan] 2025-07-10 01:33:46,110 - root - INFO - lr: 1.4838e-04 gnorm: 0.84 [11:58:45<10:02:44] +[titan] 2025-07-10 01:33:50,018 - root - INFO - step: 54395 loss: 16.5111 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9846 +[titan] 2025-07-10 01:33:50,019 - root - INFO - lr: 1.4835e-04 gnorm: 0.86 [11:58:49<10:02:40] +[titan] 2025-07-10 01:33:53,147 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:33:53,936 - root - INFO - step: 54400 loss: 16.1310 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.6584 +[titan] 2025-07-10 01:33:53,936 - root - INFO - lr: 1.4833e-04 gnorm: 0.81 [11:58:53<10:02:36] +[titan] 2025-07-10 01:33:57,862 - root - INFO - step: 54405 loss: 16.5089 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-10 01:33:57,863 - root - INFO - lr: 1.4831e-04 gnorm: 0.83 [11:58:57<10:02:32] +[titan] 2025-07-10 01:34:01,807 - root - INFO - step: 54410 loss: 16.6479 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0910 +[titan] 2025-07-10 01:34:01,808 - root - INFO - lr: 1.4829e-04 gnorm: 0.84 [11:59:01<10:02:28] +[titan] 2025-07-10 01:34:05,724 - root - INFO - step: 54415 loss: 16.5406 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-10 01:34:05,724 - root - INFO - lr: 1.4827e-04 gnorm: 0.85 [11:59:05<10:02:24] +[titan] 2025-07-10 01:34:09,666 - root - INFO - step: 54420 loss: 16.4543 memory: 44.58GiB(31.99%) tps: 83,129 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.9358 +[titan] 2025-07-10 01:34:09,666 - root - INFO - lr: 1.4825e-04 gnorm: 0.83 [11:59:09<10:02:20] +[titan] 2025-07-10 01:34:13,568 - root - INFO - step: 54425 loss: 16.5798 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0227 +[titan] 2025-07-10 01:34:13,568 - root - INFO - lr: 1.4823e-04 gnorm: 0.85 [11:59:13<10:02:16] +[titan] 2025-07-10 01:34:17,487 - root - INFO - step: 54430 loss: 16.4827 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.9616 +[titan] 2025-07-10 01:34:17,487 - root - INFO - lr: 1.4821e-04 gnorm: 0.83 [11:59:17<10:02:12] +[titan] 2025-07-10 01:34:21,398 - root - INFO - step: 54435 loss: 16.4847 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9557 +[titan] 2025-07-10 01:34:21,398 - root - INFO - lr: 1.4818e-04 gnorm: 0.86 [11:59:21<10:02:08] +[titan] 2025-07-10 01:34:25,312 - root - INFO - step: 54440 loss: 16.4651 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9340 +[titan] 2025-07-10 01:34:25,312 - root - INFO - lr: 1.4816e-04 gnorm: 0.81 [11:59:24<10:02:04] +[titan] 2025-07-10 01:34:29,223 - root - INFO - step: 54445 loss: 16.5758 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0100 +[titan] 2025-07-10 01:34:29,223 - root - INFO - lr: 1.4814e-04 gnorm: 0.88 [11:59:28<10:02:00] +[titan] 2025-07-10 01:34:32,356 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:34:33,150 - root - INFO - step: 54450 loss: 16.5370 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5381 global_avg_mtp_loss: 13.9990 +[titan] 2025-07-10 01:34:33,150 - root - INFO - lr: 1.4812e-04 gnorm: 0.82 [11:59:32<10:01:56] +[titan] 2025-07-10 01:34:37,112 - root - INFO - step: 54455 loss: 16.3365 memory: 44.58GiB(31.99%) tps: 82,717 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.8142 +[titan] 2025-07-10 01:34:37,112 - root - INFO - lr: 1.4810e-04 gnorm: 0.86 [11:59:36<10:01:52] +[titan] 2025-07-10 01:34:41,043 - root - INFO - step: 54460 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.9101 +[titan] 2025-07-10 01:34:41,043 - root - INFO - lr: 1.4808e-04 gnorm: 0.93 [11:59:40<10:01:48] +[titan] 2025-07-10 01:34:44,987 - root - INFO - step: 54465 loss: 16.3613 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8543 +[titan] 2025-07-10 01:34:44,988 - root - INFO - lr: 1.4806e-04 gnorm: 0.83 [11:59:44<10:01:44] +[titan] 2025-07-10 01:34:48,929 - root - INFO - step: 54470 loss: 16.7617 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5906 global_avg_mtp_loss: 14.1711 +[titan] 2025-07-10 01:34:48,930 - root - INFO - lr: 1.4804e-04 gnorm: 0.80 [11:59:48<10:01:40] +[titan] 2025-07-10 01:34:52,863 - root - INFO - step: 54475 loss: 16.4556 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9308 +[titan] 2025-07-10 01:34:52,863 - root - INFO - lr: 1.4801e-04 gnorm: 0.82 [11:59:52<10:01:36] +[titan] 2025-07-10 01:34:56,771 - root - INFO - step: 54480 loss: 16.2687 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.7747 +[titan] 2025-07-10 01:34:56,771 - root - INFO - lr: 1.4799e-04 gnorm: 0.78 [11:59:56<10:01:32] +[titan] 2025-07-10 01:35:00,694 - root - INFO - step: 54485 loss: 16.3588 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.8716 +[titan] 2025-07-10 01:35:00,694 - root - INFO - lr: 1.4797e-04 gnorm: 0.83 [12:00:00<10:01:28] +[titan] 2025-07-10 01:35:04,628 - root - INFO - step: 54490 loss: 16.1256 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.6360 +[titan] 2025-07-10 01:35:04,628 - root - INFO - lr: 1.4795e-04 gnorm: 0.85 [12:00:04<10:01:24] +[titan] 2025-07-10 01:35:08,571 - root - INFO - step: 54495 loss: 16.4246 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9006 +[titan] 2025-07-10 01:35:08,571 - root - INFO - lr: 1.4793e-04 gnorm: 0.82 [12:00:08<10:01:20] +[titan] 2025-07-10 01:35:11,675 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:35:12,467 - root - INFO - step: 54500 loss: 16.2214 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7354 +[titan] 2025-07-10 01:35:12,467 - root - INFO - lr: 1.4791e-04 gnorm: 0.96 [12:00:12<10:01:16] +[titan] 2025-07-10 01:35:16,379 - root - INFO - step: 54505 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 14.0298 +[titan] 2025-07-10 01:35:16,380 - root - INFO - lr: 1.4789e-04 gnorm: 0.83 [12:00:16<10:01:12] +[titan] 2025-07-10 01:35:20,313 - root - INFO - step: 54510 loss: 16.3272 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8198 +[titan] 2025-07-10 01:35:20,313 - root - INFO - lr: 1.4787e-04 gnorm: 0.86 [12:00:19<10:01:08] +[titan] 2025-07-10 01:35:24,243 - root - INFO - step: 54515 loss: 16.5029 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9611 +[titan] 2025-07-10 01:35:24,243 - root - INFO - lr: 1.4784e-04 gnorm: 0.81 [12:00:23<10:01:04] +[titan] 2025-07-10 01:35:28,202 - root - INFO - step: 54520 loss: 16.1867 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7051 +[titan] 2025-07-10 01:35:28,203 - root - INFO - lr: 1.4782e-04 gnorm: 0.79 [12:00:27<10:01:00] +[titan] 2025-07-10 01:35:32,119 - root - INFO - step: 54525 loss: 16.5803 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5685 global_avg_mtp_loss: 14.0118 +[titan] 2025-07-10 01:35:32,119 - root - INFO - lr: 1.4780e-04 gnorm: 0.83 [12:00:31<10:00:56] +[titan] 2025-07-10 01:35:36,051 - root - INFO - step: 54530 loss: 16.0441 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4578 global_avg_mtp_loss: 13.5863 +[titan] 2025-07-10 01:35:36,051 - root - INFO - lr: 1.4778e-04 gnorm: 0.84 [12:00:35<10:00:52] +[titan] 2025-07-10 01:35:40,090 - root - INFO - step: 54535 loss: 16.2479 memory: 44.58GiB(31.99%) tps: 81,130 tflops: 279.99 mfu: 28.31% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7605 +[titan] 2025-07-10 01:35:40,090 - root - INFO - lr: 1.4776e-04 gnorm: 0.78 [12:00:39<10:00:48] +[titan] 2025-07-10 01:35:44,056 - root - INFO - step: 54540 loss: 16.5252 memory: 44.58GiB(31.99%) tps: 82,625 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 13.9745 +[titan] 2025-07-10 01:35:44,056 - root - INFO - lr: 1.4774e-04 gnorm: 0.82 [12:00:43<10:00:44] +[titan] 2025-07-10 01:35:47,996 - root - INFO - step: 54545 loss: 16.1042 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.6307 +[titan] 2025-07-10 01:35:47,996 - root - INFO - lr: 1.4772e-04 gnorm: 0.86 [12:00:47<10:00:40] +[titan] 2025-07-10 01:35:51,111 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:35:51,918 - root - INFO - step: 54550 loss: 16.4437 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.9284 +[titan] 2025-07-10 01:35:51,918 - root - INFO - lr: 1.4770e-04 gnorm: 0.79 [12:00:51<10:00:36] +[titan] 2025-07-10 01:35:55,851 - root - INFO - step: 54555 loss: 16.3730 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.8607 +[titan] 2025-07-10 01:35:55,851 - root - INFO - lr: 1.4767e-04 gnorm: 0.80 [12:00:55<10:00:32] +[titan] 2025-07-10 01:35:59,780 - root - INFO - step: 54560 loss: 16.3555 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8439 +[titan] 2025-07-10 01:35:59,780 - root - INFO - lr: 1.4765e-04 gnorm: 0.86 [12:00:59<10:00:28] +[titan] 2025-07-10 01:36:03,688 - root - INFO - step: 54565 loss: 16.4414 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9086 +[titan] 2025-07-10 01:36:03,689 - root - INFO - lr: 1.4763e-04 gnorm: 0.88 [12:01:03<10:00:24] +[titan] 2025-07-10 01:36:07,598 - root - INFO - step: 54570 loss: 16.4637 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9271 +[titan] 2025-07-10 01:36:07,598 - root - INFO - lr: 1.4761e-04 gnorm: 0.81 [12:01:07<10:00:20] +[titan] 2025-07-10 01:36:11,519 - root - INFO - step: 54575 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 14.0031 +[titan] 2025-07-10 01:36:11,519 - root - INFO - lr: 1.4759e-04 gnorm: 0.88 [12:01:11<10:00:16] +[titan] 2025-07-10 01:36:15,429 - root - INFO - step: 54580 loss: 16.3454 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-10 01:36:15,429 - root - INFO - lr: 1.4757e-04 gnorm: 0.81 [12:01:15<10:00:12] +[titan] 2025-07-10 01:36:19,359 - root - INFO - step: 54585 loss: 16.4223 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9022 +[titan] 2025-07-10 01:36:19,360 - root - INFO - lr: 1.4755e-04 gnorm: 0.83 [12:01:18<10:00:08] +[titan] 2025-07-10 01:36:23,269 - root - INFO - step: 54590 loss: 16.5788 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0174 +[titan] 2025-07-10 01:36:23,269 - root - INFO - lr: 1.4753e-04 gnorm: 0.83 [12:01:22<10:00:04] +[titan] 2025-07-10 01:36:27,171 - root - INFO - step: 54595 loss: 16.3510 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8325 +[titan] 2025-07-10 01:36:27,172 - root - INFO - lr: 1.4750e-04 gnorm: 0.92 [12:01:26<10:00:00] +[titan] 2025-07-10 01:36:30,283 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:36:31,074 - root - INFO - step: 54600 loss: 16.6685 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.1025 +[titan] 2025-07-10 01:36:31,074 - root - INFO - lr: 1.4748e-04 gnorm: 0.81 [12:01:30< 9:59:56] +[titan] 2025-07-10 01:36:34,979 - root - INFO - step: 54605 loss: 16.5630 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 14.0349 +[titan] 2025-07-10 01:36:34,980 - root - INFO - lr: 1.4746e-04 gnorm: 0.84 [12:01:34< 9:59:52] +[titan] 2025-07-10 01:36:38,902 - root - INFO - step: 54610 loss: 16.4937 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9507 +[titan] 2025-07-10 01:36:38,902 - root - INFO - lr: 1.4744e-04 gnorm: 0.87 [12:01:38< 9:59:48] +[titan] 2025-07-10 01:36:42,818 - root - INFO - step: 54615 loss: 16.1826 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7039 +[titan] 2025-07-10 01:36:42,819 - root - INFO - lr: 1.4742e-04 gnorm: 0.84 [12:01:42< 9:59:44] +[titan] 2025-07-10 01:36:46,749 - root - INFO - step: 54620 loss: 16.6621 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5652 global_avg_mtp_loss: 14.0969 +[titan] 2025-07-10 01:36:46,750 - root - INFO - lr: 1.4740e-04 gnorm: 0.97 [12:01:46< 9:59:40] +[titan] 2025-07-10 01:36:50,664 - root - INFO - step: 54625 loss: 16.6897 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.1167 +[titan] 2025-07-10 01:36:50,664 - root - INFO - lr: 1.4738e-04 gnorm: 0.91 [12:01:50< 9:59:36] +[titan] 2025-07-10 01:36:54,602 - root - INFO - step: 54630 loss: 16.2767 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.7778 +[titan] 2025-07-10 01:36:54,602 - root - INFO - lr: 1.4736e-04 gnorm: 0.84 [12:01:54< 9:59:32] +[titan] 2025-07-10 01:36:58,550 - root - INFO - step: 54635 loss: 16.4346 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9151 +[titan] 2025-07-10 01:36:58,550 - root - INFO - lr: 1.4733e-04 gnorm: 0.83 [12:01:58< 9:59:28] +[titan] 2025-07-10 01:37:02,484 - root - INFO - step: 54640 loss: 16.4183 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.8992 +[titan] 2025-07-10 01:37:02,484 - root - INFO - lr: 1.4731e-04 gnorm: 0.80 [12:02:02< 9:59:24] +[titan] 2025-07-10 01:37:06,434 - root - INFO - step: 54645 loss: 16.6409 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5617 global_avg_mtp_loss: 14.0793 +[titan] 2025-07-10 01:37:06,434 - root - INFO - lr: 1.4729e-04 gnorm: 0.80 [12:02:06< 9:59:20] +[titan] 2025-07-10 01:37:09,576 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:37:10,372 - root - INFO - step: 54650 loss: 16.4365 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9169 +[titan] 2025-07-10 01:37:10,372 - root - INFO - lr: 1.4727e-04 gnorm: 0.81 [12:02:09< 9:59:16] +[titan] 2025-07-10 01:37:14,289 - root - INFO - step: 54655 loss: 16.9292 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.6007 global_avg_mtp_loss: 14.3285 +[titan] 2025-07-10 01:37:14,289 - root - INFO - lr: 1.4725e-04 gnorm: 0.85 [12:02:13< 9:59:12] +[titan] 2025-07-10 01:37:18,205 - root - INFO - step: 54660 loss: 16.2942 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.7880 +[titan] 2025-07-10 01:37:18,206 - root - INFO - lr: 1.4723e-04 gnorm: 0.86 [12:02:17< 9:59:08] +[titan] 2025-07-10 01:37:22,123 - root - INFO - step: 54665 loss: 16.6424 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0848 +[titan] 2025-07-10 01:37:22,124 - root - INFO - lr: 1.4721e-04 gnorm: 0.77 [12:02:21< 9:59:04] +[titan] 2025-07-10 01:37:26,061 - root - INFO - step: 54670 loss: 16.4328 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 13.8978 +[titan] 2025-07-10 01:37:26,061 - root - INFO - lr: 1.4719e-04 gnorm: 0.81 [12:02:25< 9:59:00] +[titan] 2025-07-10 01:37:30,009 - root - INFO - step: 54675 loss: 16.7514 memory: 44.58GiB(31.99%) tps: 83,004 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1671 +[titan] 2025-07-10 01:37:30,009 - root - INFO - lr: 1.4716e-04 gnorm: 0.84 [12:02:29< 9:58:56] +[titan] 2025-07-10 01:37:33,928 - root - INFO - step: 54680 loss: 16.7066 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.1327 +[titan] 2025-07-10 01:37:33,928 - root - INFO - lr: 1.4714e-04 gnorm: 0.84 [12:02:33< 9:58:52] +[titan] 2025-07-10 01:37:37,838 - root - INFO - step: 54685 loss: 16.4188 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 01:37:37,839 - root - INFO - lr: 1.4712e-04 gnorm: 0.86 [12:02:37< 9:58:48] +[titan] 2025-07-10 01:37:41,748 - root - INFO - step: 54690 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.8986 +[titan] 2025-07-10 01:37:41,748 - root - INFO - lr: 1.4710e-04 gnorm: 0.84 [12:02:41< 9:58:44] +[titan] 2025-07-10 01:37:45,688 - root - INFO - step: 54695 loss: 16.4605 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9289 +[titan] 2025-07-10 01:37:45,688 - root - INFO - lr: 1.4708e-04 gnorm: 0.80 [12:02:45< 9:58:40] +[titan] 2025-07-10 01:37:48,815 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:37:49,617 - root - INFO - step: 54700 loss: 16.5205 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 13.9893 +[titan] 2025-07-10 01:37:49,617 - root - INFO - lr: 1.4706e-04 gnorm: 0.80 [12:02:49< 9:58:36] +[titan] 2025-07-10 01:37:53,526 - root - INFO - step: 54705 loss: 16.6229 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5673 global_avg_mtp_loss: 14.0556 +[titan] 2025-07-10 01:37:53,526 - root - INFO - lr: 1.4704e-04 gnorm: 0.82 [12:02:53< 9:58:32] +[titan] 2025-07-10 01:37:57,434 - root - INFO - step: 54710 loss: 16.6971 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5783 global_avg_mtp_loss: 14.1188 +[titan] 2025-07-10 01:37:57,435 - root - INFO - lr: 1.4702e-04 gnorm: 0.83 [12:02:57< 9:58:28] +[titan] 2025-07-10 01:38:01,348 - root - INFO - step: 54715 loss: 16.5670 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 14.0203 +[titan] 2025-07-10 01:38:01,348 - root - INFO - lr: 1.4699e-04 gnorm: 0.83 [12:03:00< 9:58:24] +[titan] 2025-07-10 01:38:05,260 - root - INFO - step: 54720 loss: 16.6107 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.5593 global_avg_mtp_loss: 14.0514 +[titan] 2025-07-10 01:38:05,260 - root - INFO - lr: 1.4697e-04 gnorm: 0.82 [12:03:04< 9:58:20] +[titan] 2025-07-10 01:38:09,176 - root - INFO - step: 54725 loss: 16.6936 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5703 global_avg_mtp_loss: 14.1233 +[titan] 2025-07-10 01:38:09,176 - root - INFO - lr: 1.4695e-04 gnorm: 0.79 [12:03:08< 9:58:16] +[titan] 2025-07-10 01:38:13,094 - root - INFO - step: 54730 loss: 16.5593 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5490 global_avg_mtp_loss: 14.0103 +[titan] 2025-07-10 01:38:13,094 - root - INFO - lr: 1.4693e-04 gnorm: 0.99 [12:03:12< 9:58:12] +[titan] 2025-07-10 01:38:17,017 - root - INFO - step: 54735 loss: 16.4969 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9703 +[titan] 2025-07-10 01:38:17,017 - root - INFO - lr: 1.4691e-04 gnorm: 0.83 [12:03:16< 9:58:08] +[titan] 2025-07-10 01:38:20,939 - root - INFO - step: 54740 loss: 16.1692 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.6929 +[titan] 2025-07-10 01:38:20,940 - root - INFO - lr: 1.4689e-04 gnorm: 0.87 [12:03:20< 9:58:04] +[titan] 2025-07-10 01:38:24,860 - root - INFO - step: 54745 loss: 16.4537 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.9430 +[titan] 2025-07-10 01:38:24,860 - root - INFO - lr: 1.4687e-04 gnorm: 0.83 [12:03:24< 9:58:00] +[titan] 2025-07-10 01:38:27,990 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:38:28,777 - root - INFO - step: 54750 loss: 16.6973 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1270 +[titan] 2025-07-10 01:38:28,777 - root - INFO - lr: 1.4685e-04 gnorm: 0.82 [12:03:28< 9:57:56] +[titan] 2025-07-10 01:38:32,720 - root - INFO - step: 54755 loss: 16.3778 memory: 44.58GiB(31.99%) tps: 83,109 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8670 +[titan] 2025-07-10 01:38:32,720 - root - INFO - lr: 1.4683e-04 gnorm: 0.79 [12:03:32< 9:57:52] +[titan] 2025-07-10 01:38:36,647 - root - INFO - step: 54760 loss: 16.2808 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.7774 +[titan] 2025-07-10 01:38:36,647 - root - INFO - lr: 1.4680e-04 gnorm: 0.80 [12:03:36< 9:57:48] +[titan] 2025-07-10 01:38:40,610 - root - INFO - step: 54765 loss: 16.5602 memory: 44.58GiB(31.99%) tps: 82,694 tflops: 285.39 mfu: 28.86% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 14.0123 +[titan] 2025-07-10 01:38:40,610 - root - INFO - lr: 1.4678e-04 gnorm: 0.81 [12:03:40< 9:57:44] +[titan] 2025-07-10 01:38:44,533 - root - INFO - step: 54770 loss: 16.4248 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.8980 +[titan] 2025-07-10 01:38:44,533 - root - INFO - lr: 1.4676e-04 gnorm: 0.84 [12:03:44< 9:57:40] +[titan] 2025-07-10 01:38:48,519 - root - INFO - step: 54775 loss: 16.5628 memory: 44.58GiB(31.99%) tps: 82,205 tflops: 283.70 mfu: 28.69% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 14.0147 +[titan] 2025-07-10 01:38:48,520 - root - INFO - lr: 1.4674e-04 gnorm: 0.82 [12:03:48< 9:57:36] +[titan] 2025-07-10 01:38:52,424 - root - INFO - step: 54780 loss: 16.5429 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9951 +[titan] 2025-07-10 01:38:52,424 - root - INFO - lr: 1.4672e-04 gnorm: 0.79 [12:03:52< 9:57:32] +[titan] 2025-07-10 01:38:55,827 - root - INFO - Dumping profiler traces at step 54784 +[titan] 2025-07-10 01:38:55,861 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:38:56,651 - root - INFO - step: 54785 loss: 16.6662 memory: 44.58GiB(31.99%) tps: 77,515 tflops: 267.52 mfu: 27.05% global_avg_ntp_loss: 2.5697 global_avg_mtp_loss: 14.0964 +[titan] 2025-07-10 01:38:56,652 - root - INFO - lr: 1.4670e-04 gnorm: 0.84 [12:03:56< 9:57:28] +[titan] 2025-07-10 01:39:00,610 - root - INFO - step: 54790 loss: 16.5463 memory: 44.58GiB(31.99%) tps: 82,790 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5458 global_avg_mtp_loss: 14.0005 +[titan] 2025-07-10 01:39:00,610 - root - INFO - lr: 1.4668e-04 gnorm: 0.86 [12:04:00< 9:57:24] +[titan] 2025-07-10 01:39:04,517 - root - INFO - step: 54795 loss: 16.5452 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 13.9935 +[titan] 2025-07-10 01:39:04,517 - root - INFO - lr: 1.4666e-04 gnorm: 0.85 [12:04:04< 9:57:20] +[titan] 2025-07-10 01:39:07,689 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:39:08,482 - root - INFO - step: 54800 loss: 16.6739 memory: 44.58GiB(31.99%) tps: 82,659 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.1109 +[titan] 2025-07-10 01:39:08,482 - root - INFO - lr: 1.4663e-04 gnorm: 0.79 [12:04:08< 9:57:16] +[titan] 2025-07-10 01:39:12,399 - root - INFO - step: 54805 loss: 16.4365 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9025 +[titan] 2025-07-10 01:39:12,399 - root - INFO - lr: 1.4661e-04 gnorm: 0.84 [12:04:12< 9:57:12] +[titan] 2025-07-10 01:39:16,309 - root - INFO - step: 54810 loss: 16.4924 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9578 +[titan] 2025-07-10 01:39:16,309 - root - INFO - lr: 1.4659e-04 gnorm: 0.80 [12:04:15< 9:57:08] +[titan] 2025-07-10 01:39:20,235 - root - INFO - step: 54815 loss: 16.6041 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0480 +[titan] 2025-07-10 01:39:20,235 - root - INFO - lr: 1.4657e-04 gnorm: 0.83 [12:04:19< 9:57:04] +[titan] 2025-07-10 01:39:24,144 - root - INFO - step: 54820 loss: 16.4611 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.9412 +[titan] 2025-07-10 01:39:24,144 - root - INFO - lr: 1.4655e-04 gnorm: 0.86 [12:04:23< 9:57:00] +[titan] 2025-07-10 01:39:28,046 - root - INFO - step: 54825 loss: 16.5538 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5555 global_avg_mtp_loss: 13.9983 +[titan] 2025-07-10 01:39:28,046 - root - INFO - lr: 1.4653e-04 gnorm: 0.92 [12:04:27< 9:56:56] +[titan] 2025-07-10 01:39:31,967 - root - INFO - step: 54830 loss: 16.6528 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5615 global_avg_mtp_loss: 14.0913 +[titan] 2025-07-10 01:39:31,967 - root - INFO - lr: 1.4651e-04 gnorm: 0.82 [12:04:31< 9:56:52] +[titan] 2025-07-10 01:39:35,865 - root - INFO - step: 54835 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 14.0021 +[titan] 2025-07-10 01:39:35,865 - root - INFO - lr: 1.4649e-04 gnorm: 0.85 [12:04:35< 9:56:48] +[titan] 2025-07-10 01:39:39,773 - root - INFO - step: 54840 loss: 16.8068 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5979 global_avg_mtp_loss: 14.2089 +[titan] 2025-07-10 01:39:39,774 - root - INFO - lr: 1.4646e-04 gnorm: 0.83 [12:04:39< 9:56:44] +[titan] 2025-07-10 01:39:43,726 - root - INFO - step: 54845 loss: 16.5758 memory: 44.58GiB(31.99%) tps: 82,914 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 14.0324 +[titan] 2025-07-10 01:39:43,726 - root - INFO - lr: 1.4644e-04 gnorm: 0.87 [12:04:43< 9:56:40] +[titan] 2025-07-10 01:39:46,874 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:39:47,676 - root - INFO - step: 54850 loss: 16.3658 memory: 44.58GiB(31.99%) tps: 82,966 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8538 +[titan] 2025-07-10 01:39:47,676 - root - INFO - lr: 1.4642e-04 gnorm: 0.88 [12:04:47< 9:56:36] +[titan] 2025-07-10 01:39:51,641 - root - INFO - step: 54855 loss: 16.4662 memory: 44.58GiB(31.99%) tps: 82,641 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.5371 global_avg_mtp_loss: 13.9290 +[titan] 2025-07-10 01:39:51,642 - root - INFO - lr: 1.4640e-04 gnorm: 0.90 [12:04:51< 9:56:32] +[titan] 2025-07-10 01:39:55,541 - root - INFO - step: 54860 loss: 16.6659 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0969 +[titan] 2025-07-10 01:39:55,542 - root - INFO - lr: 1.4638e-04 gnorm: 0.86 [12:04:55< 9:56:28] +[titan] 2025-07-10 01:39:59,474 - root - INFO - step: 54865 loss: 16.7690 memory: 44.58GiB(31.99%) tps: 83,330 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1890 +[titan] 2025-07-10 01:39:59,474 - root - INFO - lr: 1.4636e-04 gnorm: 0.83 [12:04:59< 9:56:24] +[titan] 2025-07-10 01:40:03,389 - root - INFO - step: 54870 loss: 16.2837 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5015 global_avg_mtp_loss: 13.7822 +[titan] 2025-07-10 01:40:03,390 - root - INFO - lr: 1.4634e-04 gnorm: 0.83 [12:05:02< 9:56:20] +[titan] 2025-07-10 01:40:07,347 - root - INFO - step: 54875 loss: 16.3345 memory: 44.58GiB(31.99%) tps: 82,807 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8225 +[titan] 2025-07-10 01:40:07,347 - root - INFO - lr: 1.4632e-04 gnorm: 0.82 [12:05:06< 9:56:16] +[titan] 2025-07-10 01:40:11,252 - root - INFO - step: 54880 loss: 16.6192 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0593 +[titan] 2025-07-10 01:40:11,252 - root - INFO - lr: 1.4629e-04 gnorm: 0.81 [12:05:10< 9:56:12] +[titan] 2025-07-10 01:40:15,174 - root - INFO - step: 54885 loss: 16.5912 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5604 global_avg_mtp_loss: 14.0308 +[titan] 2025-07-10 01:40:15,174 - root - INFO - lr: 1.4627e-04 gnorm: 0.83 [12:05:14< 9:56:08] +[titan] 2025-07-10 01:40:19,098 - root - INFO - step: 54890 loss: 16.4741 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.9571 +[titan] 2025-07-10 01:40:19,098 - root - INFO - lr: 1.4625e-04 gnorm: 0.81 [12:05:18< 9:56:04] +[titan] 2025-07-10 01:40:23,020 - root - INFO - step: 54895 loss: 16.3757 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8760 +[titan] 2025-07-10 01:40:23,020 - root - INFO - lr: 1.4623e-04 gnorm: 0.89 [12:05:22< 9:56:00] +[titan] 2025-07-10 01:40:26,140 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:40:26,937 - root - INFO - step: 54900 loss: 16.4301 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.9074 +[titan] 2025-07-10 01:40:26,937 - root - INFO - lr: 1.4621e-04 gnorm: 0.87 [12:05:26< 9:55:56] +[titan] 2025-07-10 01:40:30,863 - root - INFO - step: 54905 loss: 16.3552 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.8617 +[titan] 2025-07-10 01:40:30,863 - root - INFO - lr: 1.4619e-04 gnorm: 0.83 [12:05:30< 9:55:52] +[titan] 2025-07-10 01:40:34,784 - root - INFO - step: 54910 loss: 16.1974 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.6985 +[titan] 2025-07-10 01:40:34,784 - root - INFO - lr: 1.4617e-04 gnorm: 0.88 [12:05:34< 9:55:48] +[titan] 2025-07-10 01:40:38,697 - root - INFO - step: 54915 loss: 16.5405 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 13.9885 +[titan] 2025-07-10 01:40:38,697 - root - INFO - lr: 1.4615e-04 gnorm: 0.92 [12:05:38< 9:55:44] +[titan] 2025-07-10 01:40:42,868 - root - INFO - step: 54920 loss: 16.4062 memory: 44.58GiB(31.99%) tps: 78,555 tflops: 271.11 mfu: 27.41% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8979 +[titan] 2025-07-10 01:40:42,869 - root - INFO - lr: 1.4612e-04 gnorm: 0.86 [12:05:42< 9:55:40] +[titan] 2025-07-10 01:40:46,785 - root - INFO - step: 54925 loss: 16.5260 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.9819 +[titan] 2025-07-10 01:40:46,786 - root - INFO - lr: 1.4610e-04 gnorm: 0.88 [12:05:46< 9:55:36] +[titan] 2025-07-10 01:40:50,708 - root - INFO - step: 54930 loss: 16.1091 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.6422 +[titan] 2025-07-10 01:40:50,708 - root - INFO - lr: 1.4608e-04 gnorm: 0.88 [12:05:50< 9:55:32] +[titan] 2025-07-10 01:40:54,630 - root - INFO - step: 54935 loss: 16.7667 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5714 global_avg_mtp_loss: 14.1954 +[titan] 2025-07-10 01:40:54,630 - root - INFO - lr: 1.4606e-04 gnorm: 0.81 [12:05:54< 9:55:28] +[titan] 2025-07-10 01:40:58,553 - root - INFO - step: 54940 loss: 16.6737 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.1045 +[titan] 2025-07-10 01:40:58,553 - root - INFO - lr: 1.4604e-04 gnorm: 0.87 [12:05:58< 9:55:24] +[titan] 2025-07-10 01:41:02,445 - root - INFO - step: 54945 loss: 16.4832 memory: 44.58GiB(31.99%) tps: 84,198 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-10 01:41:02,445 - root - INFO - lr: 1.4602e-04 gnorm: 0.88 [12:06:02< 9:55:20] +[titan] 2025-07-10 01:41:05,558 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:41:06,345 - root - INFO - step: 54950 loss: 16.5157 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-10 01:41:06,345 - root - INFO - lr: 1.4600e-04 gnorm: 0.87 [12:06:05< 9:55:16] +[titan] 2025-07-10 01:41:10,299 - root - INFO - step: 54955 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9079 +[titan] 2025-07-10 01:41:10,299 - root - INFO - lr: 1.4598e-04 gnorm: 0.84 [12:06:09< 9:55:12] +[titan] 2025-07-10 01:41:14,272 - root - INFO - step: 54960 loss: 16.4144 memory: 44.58GiB(31.99%) tps: 82,477 tflops: 284.64 mfu: 28.78% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.8901 +[titan] 2025-07-10 01:41:14,272 - root - INFO - lr: 1.4596e-04 gnorm: 0.89 [12:06:13< 9:55:09] +[titan] 2025-07-10 01:41:18,181 - root - INFO - step: 54965 loss: 16.3903 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.8688 +[titan] 2025-07-10 01:41:18,181 - root - INFO - lr: 1.4593e-04 gnorm: 0.80 [12:06:17< 9:55:04] +[titan] 2025-07-10 01:41:22,084 - root - INFO - step: 54970 loss: 16.6385 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5627 global_avg_mtp_loss: 14.0758 +[titan] 2025-07-10 01:41:22,084 - root - INFO - lr: 1.4591e-04 gnorm: 0.81 [12:06:21< 9:55:00] +[titan] 2025-07-10 01:41:25,994 - root - INFO - step: 54975 loss: 16.5177 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 13.9743 +[titan] 2025-07-10 01:41:25,994 - root - INFO - lr: 1.4589e-04 gnorm: 0.85 [12:06:25< 9:54:56] +[titan] 2025-07-10 01:41:29,903 - root - INFO - step: 54980 loss: 16.4602 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.9383 +[titan] 2025-07-10 01:41:29,903 - root - INFO - lr: 1.4587e-04 gnorm: 0.81 [12:06:29< 9:54:52] +[titan] 2025-07-10 01:41:33,857 - root - INFO - step: 54985 loss: 16.2026 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.7030 +[titan] 2025-07-10 01:41:33,857 - root - INFO - lr: 1.4585e-04 gnorm: 0.82 [12:06:33< 9:54:48] +[titan] 2025-07-10 01:41:37,769 - root - INFO - step: 54990 loss: 16.3765 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5057 global_avg_mtp_loss: 13.8708 +[titan] 2025-07-10 01:41:37,769 - root - INFO - lr: 1.4583e-04 gnorm: 0.83 [12:06:37< 9:54:44] +[titan] 2025-07-10 01:41:41,748 - root - INFO - step: 54995 loss: 16.7643 memory: 44.58GiB(31.99%) tps: 82,343 tflops: 284.18 mfu: 28.73% global_avg_ntp_loss: 2.5930 global_avg_mtp_loss: 14.1713 +[titan] 2025-07-10 01:41:41,749 - root - INFO - lr: 1.4581e-04 gnorm: 0.86 [12:06:41< 9:54:41] +[titan] 2025-07-10 01:41:44,864 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:41:45,654 - root - INFO - step: 55000 loss: 16.3537 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8449 +[titan] 2025-07-10 01:41:45,654 - root - INFO - lr: 1.4579e-04 gnorm: 0.87 [12:06:45< 9:54:37] +[titan] 2025-07-10 01:41:49,557 - root - INFO - step: 55005 loss: 16.3937 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8763 +[titan] 2025-07-10 01:41:49,557 - root - INFO - lr: 1.4576e-04 gnorm: 0.82 [12:06:49< 9:54:33] +[titan] 2025-07-10 01:41:53,513 - root - INFO - step: 55010 loss: 16.1767 memory: 44.58GiB(31.99%) tps: 82,847 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.6876 +[titan] 2025-07-10 01:41:53,513 - root - INFO - lr: 1.4574e-04 gnorm: 0.81 [12:06:53< 9:54:29] +[titan] 2025-07-10 01:41:57,436 - root - INFO - step: 55015 loss: 16.2371 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7447 +[titan] 2025-07-10 01:41:57,437 - root - INFO - lr: 1.4572e-04 gnorm: 0.85 [12:06:57< 9:54:25] +[titan] 2025-07-10 01:42:01,389 - root - INFO - step: 55020 loss: 16.4026 memory: 44.58GiB(31.99%) tps: 82,903 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.8796 +[titan] 2025-07-10 01:42:01,390 - root - INFO - lr: 1.4570e-04 gnorm: 0.93 [12:07:00< 9:54:21] +[titan] 2025-07-10 01:42:05,287 - root - INFO - step: 55025 loss: 16.3766 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8616 +[titan] 2025-07-10 01:42:05,288 - root - INFO - lr: 1.4568e-04 gnorm: 0.82 [12:07:04< 9:54:17] +[titan] 2025-07-10 01:42:09,202 - root - INFO - step: 55030 loss: 16.5448 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 13.9975 +[titan] 2025-07-10 01:42:09,202 - root - INFO - lr: 1.4566e-04 gnorm: 0.80 [12:07:08< 9:54:13] +[titan] 2025-07-10 01:42:13,178 - root - INFO - step: 55035 loss: 16.7003 memory: 44.58GiB(31.99%) tps: 82,425 tflops: 284.46 mfu: 28.76% global_avg_ntp_loss: 2.5848 global_avg_mtp_loss: 14.1155 +[titan] 2025-07-10 01:42:13,178 - root - INFO - lr: 1.4564e-04 gnorm: 0.84 [12:07:12< 9:54:09] +[titan] 2025-07-10 01:42:17,091 - root - INFO - step: 55040 loss: 16.4906 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9543 +[titan] 2025-07-10 01:42:17,092 - root - INFO - lr: 1.4562e-04 gnorm: 0.84 [12:07:16< 9:54:05] +[titan] 2025-07-10 01:42:21,000 - root - INFO - step: 55045 loss: 16.6265 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0699 +[titan] 2025-07-10 01:42:21,000 - root - INFO - lr: 1.4559e-04 gnorm: 0.92 [12:07:20< 9:54:01] +[titan] 2025-07-10 01:42:24,145 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:42:24,934 - root - INFO - step: 55050 loss: 16.4799 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9431 +[titan] 2025-07-10 01:42:24,934 - root - INFO - lr: 1.4557e-04 gnorm: 0.82 [12:07:24< 9:53:57] +[titan] 2025-07-10 01:42:28,864 - root - INFO - step: 55055 loss: 16.3938 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8753 +[titan] 2025-07-10 01:42:28,864 - root - INFO - lr: 1.4555e-04 gnorm: 0.85 [12:07:28< 9:53:53] +[titan] 2025-07-10 01:42:32,808 - root - INFO - step: 55060 loss: 16.5678 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 13.9973 +[titan] 2025-07-10 01:42:32,808 - root - INFO - lr: 1.4553e-04 gnorm: 0.89 [12:07:32< 9:53:49] +[titan] 2025-07-10 01:42:36,738 - root - INFO - step: 55065 loss: 16.6077 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.0445 +[titan] 2025-07-10 01:42:36,738 - root - INFO - lr: 1.4551e-04 gnorm: 0.85 [12:07:36< 9:53:45] +[titan] 2025-07-10 01:42:40,640 - root - INFO - step: 55070 loss: 16.6920 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.1087 +[titan] 2025-07-10 01:42:40,640 - root - INFO - lr: 1.4549e-04 gnorm: 0.83 [12:07:40< 9:53:41] +[titan] 2025-07-10 01:42:44,560 - root - INFO - step: 55075 loss: 16.4057 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8959 +[titan] 2025-07-10 01:42:44,560 - root - INFO - lr: 1.4547e-04 gnorm: 0.82 [12:07:44< 9:53:37] +[titan] 2025-07-10 01:42:48,474 - root - INFO - step: 55080 loss: 16.3472 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 01:42:48,474 - root - INFO - lr: 1.4545e-04 gnorm: 0.78 [12:07:48< 9:53:33] +[titan] 2025-07-10 01:42:52,393 - root - INFO - step: 55085 loss: 16.5176 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9733 +[titan] 2025-07-10 01:42:52,394 - root - INFO - lr: 1.4543e-04 gnorm: 0.86 [12:07:51< 9:53:29] +[titan] 2025-07-10 01:42:56,291 - root - INFO - step: 55090 loss: 16.3766 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 01:42:56,292 - root - INFO - lr: 1.4540e-04 gnorm: 0.81 [12:07:55< 9:53:25] +[titan] 2025-07-10 01:43:00,200 - root - INFO - step: 55095 loss: 16.4113 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 01:43:00,200 - root - INFO - lr: 1.4538e-04 gnorm: 0.81 [12:07:59< 9:53:21] +[titan] 2025-07-10 01:43:03,309 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:43:04,093 - root - INFO - step: 55100 loss: 16.5567 memory: 44.58GiB(31.99%) tps: 84,175 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0091 +[titan] 2025-07-10 01:43:04,094 - root - INFO - lr: 1.4536e-04 gnorm: 0.80 [12:08:03< 9:53:17] +[titan] 2025-07-10 01:43:08,005 - root - INFO - step: 55105 loss: 16.4077 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.8751 +[titan] 2025-07-10 01:43:08,005 - root - INFO - lr: 1.4534e-04 gnorm: 0.83 [12:08:07< 9:53:13] +[titan] 2025-07-10 01:43:11,929 - root - INFO - step: 55110 loss: 16.2222 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.7283 +[titan] 2025-07-10 01:43:11,930 - root - INFO - lr: 1.4532e-04 gnorm: 0.79 [12:08:11< 9:53:09] +[titan] 2025-07-10 01:43:15,832 - root - INFO - step: 55115 loss: 16.5692 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0182 +[titan] 2025-07-10 01:43:15,832 - root - INFO - lr: 1.4530e-04 gnorm: 0.88 [12:08:15< 9:53:05] +[titan] 2025-07-10 01:43:19,757 - root - INFO - step: 55120 loss: 16.3748 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8655 +[titan] 2025-07-10 01:43:19,757 - root - INFO - lr: 1.4528e-04 gnorm: 0.88 [12:08:19< 9:53:01] +[titan] 2025-07-10 01:43:23,671 - root - INFO - step: 55125 loss: 16.6764 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.1072 +[titan] 2025-07-10 01:43:23,671 - root - INFO - lr: 1.4526e-04 gnorm: 0.88 [12:08:23< 9:52:57] +[titan] 2025-07-10 01:43:27,581 - root - INFO - step: 55130 loss: 16.4591 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 13.9098 +[titan] 2025-07-10 01:43:27,581 - root - INFO - lr: 1.4523e-04 gnorm: 0.85 [12:08:27< 9:52:53] +[titan] 2025-07-10 01:43:31,498 - root - INFO - step: 55135 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 13.9088 +[titan] 2025-07-10 01:43:31,498 - root - INFO - lr: 1.4521e-04 gnorm: 0.87 [12:08:31< 9:52:49] +[titan] 2025-07-10 01:43:35,428 - root - INFO - step: 55140 loss: 16.2694 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.7609 +[titan] 2025-07-10 01:43:35,428 - root - INFO - lr: 1.4519e-04 gnorm: 0.87 [12:08:35< 9:52:45] +[titan] 2025-07-10 01:43:39,359 - root - INFO - step: 55145 loss: 16.2231 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7324 +[titan] 2025-07-10 01:43:39,359 - root - INFO - lr: 1.4517e-04 gnorm: 0.84 [12:08:38< 9:52:41] +[titan] 2025-07-10 01:43:42,510 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:43:43,299 - root - INFO - step: 55150 loss: 16.2763 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7927 +[titan] 2025-07-10 01:43:43,300 - root - INFO - lr: 1.4515e-04 gnorm: 0.85 [12:08:42< 9:52:37] +[titan] 2025-07-10 01:43:47,227 - root - INFO - step: 55155 loss: 16.3263 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 01:43:47,227 - root - INFO - lr: 1.4513e-04 gnorm: 0.87 [12:08:46< 9:52:33] +[titan] 2025-07-10 01:43:51,205 - root - INFO - step: 55160 loss: 16.5580 memory: 44.58GiB(31.99%) tps: 82,364 tflops: 284.25 mfu: 28.74% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 14.0047 +[titan] 2025-07-10 01:43:51,206 - root - INFO - lr: 1.4511e-04 gnorm: 0.86 [12:08:50< 9:52:29] +[titan] 2025-07-10 01:43:55,128 - root - INFO - step: 55165 loss: 16.5545 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0063 +[titan] 2025-07-10 01:43:55,129 - root - INFO - lr: 1.4509e-04 gnorm: 0.79 [12:08:54< 9:52:25] +[titan] 2025-07-10 01:43:59,082 - root - INFO - step: 55170 loss: 16.4902 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9423 +[titan] 2025-07-10 01:43:59,082 - root - INFO - lr: 1.4506e-04 gnorm: 0.84 [12:08:58< 9:52:21] +[titan] 2025-07-10 01:44:03,028 - root - INFO - step: 55175 loss: 16.3087 memory: 44.58GiB(31.99%) tps: 83,039 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.7988 +[titan] 2025-07-10 01:44:03,029 - root - INFO - lr: 1.4504e-04 gnorm: 0.82 [12:09:02< 9:52:17] +[titan] 2025-07-10 01:44:06,946 - root - INFO - step: 55180 loss: 16.3595 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5307 global_avg_mtp_loss: 13.8288 +[titan] 2025-07-10 01:44:06,946 - root - INFO - lr: 1.4502e-04 gnorm: 0.89 [12:09:06< 9:52:13] +[titan] 2025-07-10 01:44:10,875 - root - INFO - step: 55185 loss: 16.3357 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8305 +[titan] 2025-07-10 01:44:10,876 - root - INFO - lr: 1.4500e-04 gnorm: 0.92 [12:09:10< 9:52:09] +[titan] 2025-07-10 01:44:14,798 - root - INFO - step: 55190 loss: 16.5296 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9841 +[titan] 2025-07-10 01:44:14,798 - root - INFO - lr: 1.4498e-04 gnorm: 0.85 [12:09:14< 9:52:05] +[titan] 2025-07-10 01:44:18,705 - root - INFO - step: 55195 loss: 16.4148 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.8936 +[titan] 2025-07-10 01:44:18,705 - root - INFO - lr: 1.4496e-04 gnorm: 0.86 [12:09:18< 9:52:01] +[titan] 2025-07-10 01:44:21,825 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:44:22,613 - root - INFO - step: 55200 loss: 16.3082 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8142 +[titan] 2025-07-10 01:44:22,613 - root - INFO - lr: 1.4494e-04 gnorm: 0.87 [12:09:22< 9:51:57] +[titan] 2025-07-10 01:44:26,551 - root - INFO - step: 55205 loss: 16.4616 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9272 +[titan] 2025-07-10 01:44:26,552 - root - INFO - lr: 1.4492e-04 gnorm: 0.80 [12:09:26< 9:51:53] +[titan] 2025-07-10 01:44:30,444 - root - INFO - step: 55210 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 84,186 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.5230 global_avg_mtp_loss: 13.9010 +[titan] 2025-07-10 01:44:30,444 - root - INFO - lr: 1.4490e-04 gnorm: 0.87 [12:09:30< 9:51:49] +[titan] 2025-07-10 01:44:34,350 - root - INFO - step: 55215 loss: 16.4598 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9192 +[titan] 2025-07-10 01:44:34,351 - root - INFO - lr: 1.4487e-04 gnorm: 0.83 [12:09:33< 9:51:45] +[titan] 2025-07-10 01:44:38,287 - root - INFO - step: 55220 loss: 16.4379 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 01:44:38,287 - root - INFO - lr: 1.4485e-04 gnorm: 0.84 [12:09:37< 9:51:41] +[titan] 2025-07-10 01:44:42,199 - root - INFO - step: 55225 loss: 16.1900 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7090 +[titan] 2025-07-10 01:44:42,199 - root - INFO - lr: 1.4483e-04 gnorm: 0.81 [12:09:41< 9:51:37] +[titan] 2025-07-10 01:44:46,146 - root - INFO - step: 55230 loss: 16.4887 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9530 +[titan] 2025-07-10 01:44:46,146 - root - INFO - lr: 1.4481e-04 gnorm: 0.80 [12:09:45< 9:51:33] +[titan] 2025-07-10 01:44:50,068 - root - INFO - step: 55235 loss: 16.4214 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-10 01:44:50,068 - root - INFO - lr: 1.4479e-04 gnorm: 0.83 [12:09:49< 9:51:29] +[titan] 2025-07-10 01:44:54,008 - root - INFO - step: 55240 loss: 16.4409 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.9129 +[titan] 2025-07-10 01:44:54,008 - root - INFO - lr: 1.4477e-04 gnorm: 0.82 [12:09:53< 9:51:25] +[titan] 2025-07-10 01:44:57,945 - root - INFO - step: 55245 loss: 16.4303 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 01:44:57,946 - root - INFO - lr: 1.4475e-04 gnorm: 0.85 [12:09:57< 9:51:21] +[titan] 2025-07-10 01:45:01,068 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:45:01,866 - root - INFO - step: 55250 loss: 16.4400 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9167 +[titan] 2025-07-10 01:45:01,866 - root - INFO - lr: 1.4473e-04 gnorm: 0.84 [12:10:01< 9:51:17] +[titan] 2025-07-10 01:45:05,821 - root - INFO - step: 55255 loss: 16.6060 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0513 +[titan] 2025-07-10 01:45:05,821 - root - INFO - lr: 1.4470e-04 gnorm: 0.87 [12:10:05< 9:51:13] +[titan] 2025-07-10 01:45:09,755 - root - INFO - step: 55260 loss: 16.5446 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9994 +[titan] 2025-07-10 01:45:09,755 - root - INFO - lr: 1.4468e-04 gnorm: 0.81 [12:10:09< 9:51:09] +[titan] 2025-07-10 01:45:13,687 - root - INFO - step: 55265 loss: 16.7118 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5814 global_avg_mtp_loss: 14.1304 +[titan] 2025-07-10 01:45:13,687 - root - INFO - lr: 1.4466e-04 gnorm: 0.88 [12:10:13< 9:51:05] +[titan] 2025-07-10 01:45:17,605 - root - INFO - step: 55270 loss: 16.4629 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5543 global_avg_mtp_loss: 13.9086 +[titan] 2025-07-10 01:45:17,605 - root - INFO - lr: 1.4464e-04 gnorm: 0.87 [12:10:17< 9:51:01] +[titan] 2025-07-10 01:45:21,513 - root - INFO - step: 55275 loss: 16.4016 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8831 +[titan] 2025-07-10 01:45:21,513 - root - INFO - lr: 1.4462e-04 gnorm: 0.82 [12:10:21< 9:50:57] +[titan] 2025-07-10 01:45:25,498 - root - INFO - step: 55280 loss: 16.2899 memory: 44.58GiB(31.99%) tps: 82,225 tflops: 283.77 mfu: 28.69% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.7880 +[titan] 2025-07-10 01:45:25,498 - root - INFO - lr: 1.4460e-04 gnorm: 0.88 [12:10:25< 9:50:53] +[titan] 2025-07-10 01:45:29,414 - root - INFO - step: 55285 loss: 16.5080 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9691 +[titan] 2025-07-10 01:45:29,414 - root - INFO - lr: 1.4458e-04 gnorm: 0.80 [12:10:28< 9:50:49] +[titan] 2025-07-10 01:45:33,358 - root - INFO - step: 55290 loss: 16.2879 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.7744 +[titan] 2025-07-10 01:45:33,358 - root - INFO - lr: 1.4456e-04 gnorm: 0.82 [12:10:32< 9:50:45] +[titan] 2025-07-10 01:45:37,346 - root - INFO - step: 55295 loss: 16.3885 memory: 44.58GiB(31.99%) tps: 82,168 tflops: 283.57 mfu: 28.67% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8769 +[titan] 2025-07-10 01:45:37,346 - root - INFO - lr: 1.4454e-04 gnorm: 0.80 [12:10:36< 9:50:41] +[titan] 2025-07-10 01:45:38,294 - root - INFO - Dumping profiler traces at step 55296 +[titan] 2025-07-10 01:45:38,326 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:45:40,667 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:45:41,458 - root - INFO - step: 55300 loss: 16.6402 memory: 44.58GiB(31.99%) tps: 79,702 tflops: 275.07 mfu: 27.81% global_avg_ntp_loss: 2.5782 global_avg_mtp_loss: 14.0620 +[titan] 2025-07-10 01:45:41,458 - root - INFO - lr: 1.4451e-04 gnorm: 0.80 [12:10:41< 9:50:37] +[titan] 2025-07-10 01:45:45,403 - root - INFO - step: 55305 loss: 16.4029 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8898 +[titan] 2025-07-10 01:45:45,404 - root - INFO - lr: 1.4449e-04 gnorm: 0.85 [12:10:44< 9:50:33] +[titan] 2025-07-10 01:45:49,325 - root - INFO - step: 55310 loss: 16.2706 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.7726 +[titan] 2025-07-10 01:45:49,325 - root - INFO - lr: 1.4447e-04 gnorm: 0.94 [12:10:48< 9:50:29] +[titan] 2025-07-10 01:45:53,258 - root - INFO - step: 55315 loss: 16.3995 memory: 44.58GiB(31.99%) tps: 83,321 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.8750 +[titan] 2025-07-10 01:45:53,258 - root - INFO - lr: 1.4445e-04 gnorm: 0.90 [12:10:52< 9:50:25] +[titan] 2025-07-10 01:45:57,179 - root - INFO - step: 55320 loss: 16.5039 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9715 +[titan] 2025-07-10 01:45:57,179 - root - INFO - lr: 1.4443e-04 gnorm: 0.87 [12:10:56< 9:50:21] +[titan] 2025-07-10 01:46:01,091 - root - INFO - step: 55325 loss: 16.3248 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8225 +[titan] 2025-07-10 01:46:01,091 - root - INFO - lr: 1.4441e-04 gnorm: 0.85 [12:11:00< 9:50:17] +[titan] 2025-07-10 01:46:04,990 - root - INFO - step: 55330 loss: 16.5011 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5392 global_avg_mtp_loss: 13.9619 +[titan] 2025-07-10 01:46:04,991 - root - INFO - lr: 1.4439e-04 gnorm: 0.87 [12:11:04< 9:50:13] +[titan] 2025-07-10 01:46:08,891 - root - INFO - step: 55335 loss: 16.8120 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5939 global_avg_mtp_loss: 14.2182 +[titan] 2025-07-10 01:46:08,891 - root - INFO - lr: 1.4437e-04 gnorm: 0.87 [12:11:08< 9:50:09] +[titan] 2025-07-10 01:46:12,819 - root - INFO - step: 55340 loss: 16.2230 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7243 +[titan] 2025-07-10 01:46:12,819 - root - INFO - lr: 1.4435e-04 gnorm: 0.81 [12:11:12< 9:50:05] +[titan] 2025-07-10 01:46:16,750 - root - INFO - step: 55345 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9454 +[titan] 2025-07-10 01:46:16,750 - root - INFO - lr: 1.4432e-04 gnorm: 0.79 [12:11:16< 9:50:01] +[titan] 2025-07-10 01:46:19,872 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:46:20,665 - root - INFO - step: 55350 loss: 16.2731 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.7618 +[titan] 2025-07-10 01:46:20,665 - root - INFO - lr: 1.4430e-04 gnorm: 0.87 [12:11:20< 9:49:57] +[titan] 2025-07-10 01:46:24,582 - root - INFO - step: 55355 loss: 16.5631 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5552 global_avg_mtp_loss: 14.0079 +[titan] 2025-07-10 01:46:24,582 - root - INFO - lr: 1.4428e-04 gnorm: 0.91 [12:11:24< 9:49:53] +[titan] 2025-07-10 01:46:28,493 - root - INFO - step: 55360 loss: 16.2935 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.8055 +[titan] 2025-07-10 01:46:28,493 - root - INFO - lr: 1.4426e-04 gnorm: 0.85 [12:11:28< 9:49:49] +[titan] 2025-07-10 01:46:32,443 - root - INFO - step: 55365 loss: 16.3670 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.8458 +[titan] 2025-07-10 01:46:32,443 - root - INFO - lr: 1.4424e-04 gnorm: 0.87 [12:11:31< 9:49:45] +[titan] 2025-07-10 01:46:36,354 - root - INFO - step: 55370 loss: 16.5185 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 13.9617 +[titan] 2025-07-10 01:46:36,354 - root - INFO - lr: 1.4422e-04 gnorm: 0.92 [12:11:35< 9:49:41] +[titan] 2025-07-10 01:46:40,259 - root - INFO - step: 55375 loss: 16.4225 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9038 +[titan] 2025-07-10 01:46:40,260 - root - INFO - lr: 1.4420e-04 gnorm: 0.92 [12:11:39< 9:49:37] +[titan] 2025-07-10 01:46:44,174 - root - INFO - step: 55380 loss: 16.4180 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 13.8594 +[titan] 2025-07-10 01:46:44,174 - root - INFO - lr: 1.4418e-04 gnorm: 0.90 [12:11:43< 9:49:33] +[titan] 2025-07-10 01:46:48,102 - root - INFO - step: 55385 loss: 16.5124 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9722 +[titan] 2025-07-10 01:46:48,102 - root - INFO - lr: 1.4415e-04 gnorm: 0.84 [12:11:47< 9:49:29] +[titan] 2025-07-10 01:46:52,044 - root - INFO - step: 55390 loss: 16.5611 memory: 44.58GiB(31.99%) tps: 83,137 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 14.0081 +[titan] 2025-07-10 01:46:52,044 - root - INFO - lr: 1.4413e-04 gnorm: 0.85 [12:11:51< 9:49:25] +[titan] 2025-07-10 01:46:55,966 - root - INFO - step: 55395 loss: 16.3069 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8088 +[titan] 2025-07-10 01:46:55,966 - root - INFO - lr: 1.4411e-04 gnorm: 0.80 [12:11:55< 9:49:21] +[titan] 2025-07-10 01:46:59,104 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:46:59,893 - root - INFO - step: 55400 loss: 16.1857 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7044 +[titan] 2025-07-10 01:46:59,893 - root - INFO - lr: 1.4409e-04 gnorm: 0.84 [12:11:59< 9:49:17] +[titan] 2025-07-10 01:47:03,851 - root - INFO - step: 55405 loss: 16.3755 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.8498 +[titan] 2025-07-10 01:47:03,851 - root - INFO - lr: 1.4407e-04 gnorm: 0.81 [12:12:03< 9:49:13] +[titan] 2025-07-10 01:47:07,770 - root - INFO - step: 55410 loss: 16.5337 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9998 +[titan] 2025-07-10 01:47:07,771 - root - INFO - lr: 1.4405e-04 gnorm: 0.83 [12:12:07< 9:49:09] +[titan] 2025-07-10 01:47:11,719 - root - INFO - step: 55415 loss: 16.3123 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8135 +[titan] 2025-07-10 01:47:11,720 - root - INFO - lr: 1.4403e-04 gnorm: 0.86 [12:12:11< 9:49:05] +[titan] 2025-07-10 01:47:15,642 - root - INFO - step: 55420 loss: 16.6761 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1067 +[titan] 2025-07-10 01:47:15,642 - root - INFO - lr: 1.4401e-04 gnorm: 0.83 [12:12:15< 9:49:01] +[titan] 2025-07-10 01:47:19,551 - root - INFO - step: 55425 loss: 16.3610 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8564 +[titan] 2025-07-10 01:47:19,551 - root - INFO - lr: 1.4399e-04 gnorm: 0.90 [12:12:19< 9:48:57] +[titan] 2025-07-10 01:47:23,502 - root - INFO - step: 55430 loss: 16.1976 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7200 +[titan] 2025-07-10 01:47:23,502 - root - INFO - lr: 1.4396e-04 gnorm: 0.80 [12:12:23< 9:48:53] +[titan] 2025-07-10 01:47:27,467 - root - INFO - step: 55435 loss: 16.4413 memory: 44.58GiB(31.99%) tps: 82,639 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9129 +[titan] 2025-07-10 01:47:27,468 - root - INFO - lr: 1.4394e-04 gnorm: 0.90 [12:12:27< 9:48:49] +[titan] 2025-07-10 01:47:31,393 - root - INFO - step: 55440 loss: 16.1149 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6552 +[titan] 2025-07-10 01:47:31,393 - root - INFO - lr: 1.4392e-04 gnorm: 0.87 [12:12:30< 9:48:45] +[titan] 2025-07-10 01:47:35,314 - root - INFO - step: 55445 loss: 16.3698 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 01:47:35,315 - root - INFO - lr: 1.4390e-04 gnorm: 0.92 [12:12:34< 9:48:41] +[titan] 2025-07-10 01:47:38,440 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:47:39,230 - root - INFO - step: 55450 loss: 16.4721 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9452 +[titan] 2025-07-10 01:47:39,230 - root - INFO - lr: 1.4388e-04 gnorm: 0.88 [12:12:38< 9:48:37] +[titan] 2025-07-10 01:47:43,129 - root - INFO - step: 55455 loss: 16.6947 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.1150 +[titan] 2025-07-10 01:47:43,130 - root - INFO - lr: 1.4386e-04 gnorm: 0.84 [12:12:42< 9:48:33] +[titan] 2025-07-10 01:47:47,040 - root - INFO - step: 55460 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5267 global_avg_mtp_loss: 13.8972 +[titan] 2025-07-10 01:47:47,041 - root - INFO - lr: 1.4384e-04 gnorm: 0.86 [12:12:46< 9:48:29] +[titan] 2025-07-10 01:47:50,949 - root - INFO - step: 55465 loss: 16.4149 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 01:47:50,949 - root - INFO - lr: 1.4382e-04 gnorm: 0.82 [12:12:50< 9:48:25] +[titan] 2025-07-10 01:47:54,894 - root - INFO - step: 55470 loss: 16.4774 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9471 +[titan] 2025-07-10 01:47:54,894 - root - INFO - lr: 1.4379e-04 gnorm: 0.85 [12:12:54< 9:48:21] +[titan] 2025-07-10 01:47:58,805 - root - INFO - step: 55475 loss: 16.2130 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.7413 +[titan] 2025-07-10 01:47:58,805 - root - INFO - lr: 1.4377e-04 gnorm: 0.88 [12:12:58< 9:48:17] +[titan] 2025-07-10 01:48:02,700 - root - INFO - step: 55480 loss: 16.3833 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8711 +[titan] 2025-07-10 01:48:02,700 - root - INFO - lr: 1.4375e-04 gnorm: 0.83 [12:13:02< 9:48:13] +[titan] 2025-07-10 01:48:06,614 - root - INFO - step: 55485 loss: 16.6609 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5671 global_avg_mtp_loss: 14.0939 +[titan] 2025-07-10 01:48:06,614 - root - INFO - lr: 1.4373e-04 gnorm: 0.81 [12:13:06< 9:48:09] +[titan] 2025-07-10 01:48:10,536 - root - INFO - step: 55490 loss: 16.2451 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4920 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 01:48:10,536 - root - INFO - lr: 1.4371e-04 gnorm: 0.84 [12:13:10< 9:48:05] +[titan] 2025-07-10 01:48:14,464 - root - INFO - step: 55495 loss: 16.7171 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5753 global_avg_mtp_loss: 14.1419 +[titan] 2025-07-10 01:48:14,465 - root - INFO - lr: 1.4369e-04 gnorm: 0.88 [12:13:14< 9:48:01] +[titan] 2025-07-10 01:48:17,620 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:48:18,409 - root - INFO - step: 55500 loss: 16.5527 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5564 global_avg_mtp_loss: 13.9963 +[titan] 2025-07-10 01:48:18,409 - root - INFO - lr: 1.4367e-04 gnorm: 0.87 [12:13:17< 9:47:57] +[titan] 2025-07-10 01:48:22,354 - root - INFO - step: 55505 loss: 16.2434 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7479 +[titan] 2025-07-10 01:48:22,354 - root - INFO - lr: 1.4365e-04 gnorm: 0.82 [12:13:21< 9:47:53] +[titan] 2025-07-10 01:48:26,272 - root - INFO - step: 55510 loss: 16.5506 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 14.0061 +[titan] 2025-07-10 01:48:26,272 - root - INFO - lr: 1.4363e-04 gnorm: 0.81 [12:13:25< 9:47:49] +[titan] 2025-07-10 01:48:30,194 - root - INFO - step: 55515 loss: 16.4029 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 01:48:30,194 - root - INFO - lr: 1.4360e-04 gnorm: 0.87 [12:13:29< 9:47:45] +[titan] 2025-07-10 01:48:34,106 - root - INFO - step: 55520 loss: 16.6416 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0802 +[titan] 2025-07-10 01:48:34,107 - root - INFO - lr: 1.4358e-04 gnorm: 0.87 [12:13:33< 9:47:41] +[titan] 2025-07-10 01:48:38,035 - root - INFO - step: 55525 loss: 16.3285 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.8080 +[titan] 2025-07-10 01:48:38,035 - root - INFO - lr: 1.4356e-04 gnorm: 0.82 [12:13:37< 9:47:37] +[titan] 2025-07-10 01:48:41,966 - root - INFO - step: 55530 loss: 16.5083 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9817 +[titan] 2025-07-10 01:48:41,966 - root - INFO - lr: 1.4354e-04 gnorm: 0.80 [12:13:41< 9:47:33] +[titan] 2025-07-10 01:48:45,881 - root - INFO - step: 55535 loss: 16.3551 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-10 01:48:45,882 - root - INFO - lr: 1.4352e-04 gnorm: 0.80 [12:13:45< 9:47:29] +[titan] 2025-07-10 01:48:49,817 - root - INFO - step: 55540 loss: 16.6997 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.1240 +[titan] 2025-07-10 01:48:49,817 - root - INFO - lr: 1.4350e-04 gnorm: 0.85 [12:13:49< 9:47:25] +[titan] 2025-07-10 01:48:53,746 - root - INFO - step: 55545 loss: 16.4462 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9160 +[titan] 2025-07-10 01:48:53,746 - root - INFO - lr: 1.4348e-04 gnorm: 0.82 [12:13:53< 9:47:21] +[titan] 2025-07-10 01:48:56,857 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:48:57,666 - root - INFO - step: 55550 loss: 15.9322 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4333 global_avg_mtp_loss: 13.4989 +[titan] 2025-07-10 01:48:57,666 - root - INFO - lr: 1.4346e-04 gnorm: 0.87 [12:13:57< 9:47:17] +[titan] 2025-07-10 01:49:01,590 - root - INFO - step: 55555 loss: 16.5259 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9908 +[titan] 2025-07-10 01:49:01,590 - root - INFO - lr: 1.4344e-04 gnorm: 0.81 [12:14:01< 9:47:13] +[titan] 2025-07-10 01:49:05,525 - root - INFO - step: 55560 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 83,277 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 14.0201 +[titan] 2025-07-10 01:49:05,526 - root - INFO - lr: 1.4341e-04 gnorm: 0.82 [12:14:05< 9:47:09] +[titan] 2025-07-10 01:49:09,441 - root - INFO - step: 55565 loss: 16.5651 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 14.0162 +[titan] 2025-07-10 01:49:09,441 - root - INFO - lr: 1.4339e-04 gnorm: 0.81 [12:14:08< 9:47:05] +[titan] 2025-07-10 01:49:13,378 - root - INFO - step: 55570 loss: 16.4508 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5285 global_avg_mtp_loss: 13.9223 +[titan] 2025-07-10 01:49:13,378 - root - INFO - lr: 1.4337e-04 gnorm: 0.81 [12:14:12< 9:47:01] +[titan] 2025-07-10 01:49:17,299 - root - INFO - step: 55575 loss: 16.7328 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5808 global_avg_mtp_loss: 14.1519 +[titan] 2025-07-10 01:49:17,299 - root - INFO - lr: 1.4335e-04 gnorm: 0.88 [12:14:16< 9:46:57] +[titan] 2025-07-10 01:49:21,220 - root - INFO - step: 55580 loss: 16.3086 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8058 +[titan] 2025-07-10 01:49:21,221 - root - INFO - lr: 1.4333e-04 gnorm: 0.85 [12:14:20< 9:46:53] +[titan] 2025-07-10 01:49:25,137 - root - INFO - step: 55585 loss: 16.5354 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 14.0082 +[titan] 2025-07-10 01:49:25,137 - root - INFO - lr: 1.4331e-04 gnorm: 0.84 [12:14:24< 9:46:49] +[titan] 2025-07-10 01:49:29,049 - root - INFO - step: 55590 loss: 16.5935 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 01:49:29,050 - root - INFO - lr: 1.4329e-04 gnorm: 0.78 [12:14:28< 9:46:45] +[titan] 2025-07-10 01:49:32,985 - root - INFO - step: 55595 loss: 16.3514 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8390 +[titan] 2025-07-10 01:49:32,985 - root - INFO - lr: 1.4327e-04 gnorm: 0.80 [12:14:32< 9:46:41] +[titan] 2025-07-10 01:49:36,089 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:49:36,890 - root - INFO - step: 55600 loss: 16.8278 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.6096 global_avg_mtp_loss: 14.2181 +[titan] 2025-07-10 01:49:36,890 - root - INFO - lr: 1.4325e-04 gnorm: 0.85 [12:14:36< 9:46:37] +[titan] 2025-07-10 01:49:40,796 - root - INFO - step: 55605 loss: 16.6109 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0544 +[titan] 2025-07-10 01:49:40,796 - root - INFO - lr: 1.4322e-04 gnorm: 0.79 [12:14:40< 9:46:33] +[titan] 2025-07-10 01:49:44,717 - root - INFO - step: 55610 loss: 16.4837 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9591 +[titan] 2025-07-10 01:49:44,717 - root - INFO - lr: 1.4320e-04 gnorm: 0.83 [12:14:44< 9:46:29] +[titan] 2025-07-10 01:49:48,624 - root - INFO - step: 55615 loss: 16.5466 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5500 global_avg_mtp_loss: 13.9965 +[titan] 2025-07-10 01:49:48,624 - root - INFO - lr: 1.4318e-04 gnorm: 0.83 [12:14:48< 9:46:25] +[titan] 2025-07-10 01:49:52,547 - root - INFO - step: 55620 loss: 16.3223 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8227 +[titan] 2025-07-10 01:49:52,547 - root - INFO - lr: 1.4316e-04 gnorm: 0.85 [12:14:52< 9:46:21] +[titan] 2025-07-10 01:49:56,510 - root - INFO - step: 55625 loss: 16.4994 memory: 44.58GiB(31.99%) tps: 82,685 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9681 +[titan] 2025-07-10 01:49:56,511 - root - INFO - lr: 1.4314e-04 gnorm: 0.82 [12:14:56< 9:46:17] +[titan] 2025-07-10 01:50:00,464 - root - INFO - step: 55630 loss: 16.8204 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.6056 global_avg_mtp_loss: 14.2148 +[titan] 2025-07-10 01:50:00,465 - root - INFO - lr: 1.4312e-04 gnorm: 0.82 [12:14:59< 9:46:13] +[titan] 2025-07-10 01:50:04,405 - root - INFO - step: 55635 loss: 16.3773 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 01:50:04,405 - root - INFO - lr: 1.4310e-04 gnorm: 0.84 [12:15:03< 9:46:09] +[titan] 2025-07-10 01:50:08,322 - root - INFO - step: 55640 loss: 16.6099 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5670 global_avg_mtp_loss: 14.0429 +[titan] 2025-07-10 01:50:08,323 - root - INFO - lr: 1.4308e-04 gnorm: 0.82 [12:15:07< 9:46:05] +[titan] 2025-07-10 01:50:12,235 - root - INFO - step: 55645 loss: 15.9730 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.5157 +[titan] 2025-07-10 01:50:12,235 - root - INFO - lr: 1.4305e-04 gnorm: 0.81 [12:15:11< 9:46:01] +[titan] 2025-07-10 01:50:15,372 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:50:16,162 - root - INFO - step: 55650 loss: 16.4547 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9244 +[titan] 2025-07-10 01:50:16,163 - root - INFO - lr: 1.4303e-04 gnorm: 0.89 [12:15:15< 9:45:57] +[titan] 2025-07-10 01:50:20,058 - root - INFO - step: 55655 loss: 16.6020 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5573 global_avg_mtp_loss: 14.0447 +[titan] 2025-07-10 01:50:20,058 - root - INFO - lr: 1.4301e-04 gnorm: 0.81 [12:15:19< 9:45:53] +[titan] 2025-07-10 01:50:23,975 - root - INFO - step: 55660 loss: 16.5009 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9688 +[titan] 2025-07-10 01:50:23,975 - root - INFO - lr: 1.4299e-04 gnorm: 0.82 [12:15:23< 9:45:49] +[titan] 2025-07-10 01:50:27,915 - root - INFO - step: 55665 loss: 16.3391 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8327 +[titan] 2025-07-10 01:50:27,916 - root - INFO - lr: 1.4297e-04 gnorm: 0.85 [12:15:27< 9:45:45] +[titan] 2025-07-10 01:50:31,817 - root - INFO - step: 55670 loss: 16.2177 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7376 +[titan] 2025-07-10 01:50:31,818 - root - INFO - lr: 1.4295e-04 gnorm: 0.84 [12:15:31< 9:45:41] +[titan] 2025-07-10 01:50:35,745 - root - INFO - step: 55675 loss: 16.3786 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8748 +[titan] 2025-07-10 01:50:35,745 - root - INFO - lr: 1.4293e-04 gnorm: 0.82 [12:15:35< 9:45:37] +[titan] 2025-07-10 01:50:39,652 - root - INFO - step: 55680 loss: 16.5399 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9993 +[titan] 2025-07-10 01:50:39,652 - root - INFO - lr: 1.4291e-04 gnorm: 0.82 [12:15:39< 9:45:33] +[titan] 2025-07-10 01:50:43,551 - root - INFO - step: 55685 loss: 16.7587 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5849 global_avg_mtp_loss: 14.1739 +[titan] 2025-07-10 01:50:43,551 - root - INFO - lr: 1.4289e-04 gnorm: 0.84 [12:15:43< 9:45:29] +[titan] 2025-07-10 01:50:47,464 - root - INFO - step: 55690 loss: 16.5384 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 13.9964 +[titan] 2025-07-10 01:50:47,464 - root - INFO - lr: 1.4286e-04 gnorm: 0.78 [12:15:46< 9:45:25] +[titan] 2025-07-10 01:50:51,392 - root - INFO - step: 55695 loss: 16.0206 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.5645 +[titan] 2025-07-10 01:50:51,392 - root - INFO - lr: 1.4284e-04 gnorm: 0.82 [12:15:50< 9:45:21] +[titan] 2025-07-10 01:50:54,528 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:50:55,325 - root - INFO - step: 55700 loss: 16.3460 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-10 01:50:55,325 - root - INFO - lr: 1.4282e-04 gnorm: 0.90 [12:15:54< 9:45:17] +[titan] 2025-07-10 01:50:59,240 - root - INFO - step: 55705 loss: 16.2920 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.7940 +[titan] 2025-07-10 01:50:59,240 - root - INFO - lr: 1.4280e-04 gnorm: 0.84 [12:15:58< 9:45:13] +[titan] 2025-07-10 01:51:03,204 - root - INFO - step: 55710 loss: 16.3555 memory: 44.58GiB(31.99%) tps: 82,667 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8429 +[titan] 2025-07-10 01:51:03,204 - root - INFO - lr: 1.4278e-04 gnorm: 0.82 [12:16:02< 9:45:09] +[titan] 2025-07-10 01:51:07,152 - root - INFO - step: 55715 loss: 16.6052 memory: 44.58GiB(31.99%) tps: 82,996 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0505 +[titan] 2025-07-10 01:51:07,153 - root - INFO - lr: 1.4276e-04 gnorm: 0.85 [12:16:06< 9:45:05] +[titan] 2025-07-10 01:51:11,062 - root - INFO - step: 55720 loss: 16.4811 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9364 +[titan] 2025-07-10 01:51:11,063 - root - INFO - lr: 1.4274e-04 gnorm: 0.87 [12:16:10< 9:45:01] +[titan] 2025-07-10 01:51:14,991 - root - INFO - step: 55725 loss: 16.5261 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9881 +[titan] 2025-07-10 01:51:14,991 - root - INFO - lr: 1.4272e-04 gnorm: 0.90 [12:16:14< 9:44:57] +[titan] 2025-07-10 01:51:18,919 - root - INFO - step: 55730 loss: 16.3350 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.8244 +[titan] 2025-07-10 01:51:18,919 - root - INFO - lr: 1.4270e-04 gnorm: 0.81 [12:16:18< 9:44:53] +[titan] 2025-07-10 01:51:22,820 - root - INFO - step: 55735 loss: 16.3562 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8476 +[titan] 2025-07-10 01:51:22,820 - root - INFO - lr: 1.4267e-04 gnorm: 0.83 [12:16:22< 9:44:49] +[titan] 2025-07-10 01:51:26,743 - root - INFO - step: 55740 loss: 16.4130 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8959 +[titan] 2025-07-10 01:51:26,744 - root - INFO - lr: 1.4265e-04 gnorm: 0.82 [12:16:26< 9:44:45] +[titan] 2025-07-10 01:51:30,660 - root - INFO - step: 55745 loss: 16.6466 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0844 +[titan] 2025-07-10 01:51:30,660 - root - INFO - lr: 1.4263e-04 gnorm: 0.88 [12:16:30< 9:44:41] +[titan] 2025-07-10 01:51:33,787 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:51:34,574 - root - INFO - step: 55750 loss: 16.3612 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8474 +[titan] 2025-07-10 01:51:34,574 - root - INFO - lr: 1.4261e-04 gnorm: 0.79 [12:16:34< 9:44:37] +[titan] 2025-07-10 01:51:38,481 - root - INFO - step: 55755 loss: 16.4184 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.8892 +[titan] 2025-07-10 01:51:38,481 - root - INFO - lr: 1.4259e-04 gnorm: 0.80 [12:16:37< 9:44:33] +[titan] 2025-07-10 01:51:42,429 - root - INFO - step: 55760 loss: 16.2754 memory: 44.58GiB(31.99%) tps: 83,006 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.7728 +[titan] 2025-07-10 01:51:42,429 - root - INFO - lr: 1.4257e-04 gnorm: 0.82 [12:16:41< 9:44:29] +[titan] 2025-07-10 01:51:46,344 - root - INFO - step: 55765 loss: 16.1984 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.7141 +[titan] 2025-07-10 01:51:46,344 - root - INFO - lr: 1.4255e-04 gnorm: 0.85 [12:16:45< 9:44:25] +[titan] 2025-07-10 01:51:50,274 - root - INFO - step: 55770 loss: 16.2328 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7375 +[titan] 2025-07-10 01:51:50,274 - root - INFO - lr: 1.4253e-04 gnorm: 0.90 [12:16:49< 9:44:21] +[titan] 2025-07-10 01:51:54,191 - root - INFO - step: 55775 loss: 16.5050 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9659 +[titan] 2025-07-10 01:51:54,191 - root - INFO - lr: 1.4251e-04 gnorm: 0.82 [12:16:53< 9:44:17] +[titan] 2025-07-10 01:51:58,174 - root - INFO - step: 55780 loss: 16.6791 memory: 44.58GiB(31.99%) tps: 82,285 tflops: 283.98 mfu: 28.71% global_avg_ntp_loss: 2.5669 global_avg_mtp_loss: 14.1122 +[titan] 2025-07-10 01:51:58,174 - root - INFO - lr: 1.4248e-04 gnorm: 0.90 [12:16:57< 9:44:13] +[titan] 2025-07-10 01:52:02,103 - root - INFO - step: 55785 loss: 16.5656 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 14.0189 +[titan] 2025-07-10 01:52:02,103 - root - INFO - lr: 1.4246e-04 gnorm: 0.89 [12:17:01< 9:44:09] +[titan] 2025-07-10 01:52:06,018 - root - INFO - step: 55790 loss: 16.4033 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8882 +[titan] 2025-07-10 01:52:06,018 - root - INFO - lr: 1.4244e-04 gnorm: 0.89 [12:17:05< 9:44:05] +[titan] 2025-07-10 01:52:09,961 - root - INFO - step: 55795 loss: 16.1986 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7166 +[titan] 2025-07-10 01:52:09,961 - root - INFO - lr: 1.4242e-04 gnorm: 0.87 [12:17:09< 9:44:01] +[titan] 2025-07-10 01:52:13,095 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:52:13,884 - root - INFO - step: 55800 loss: 16.3611 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.8397 +[titan] 2025-07-10 01:52:13,884 - root - INFO - lr: 1.4240e-04 gnorm: 0.85 [12:17:13< 9:43:57] +[titan] 2025-07-10 01:52:17,873 - root - INFO - step: 55805 loss: 16.5276 memory: 44.58GiB(31.99%) tps: 82,150 tflops: 283.51 mfu: 28.67% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9821 +[titan] 2025-07-10 01:52:17,874 - root - INFO - lr: 1.4238e-04 gnorm: 0.85 [12:17:17< 9:43:53] +[titan] 2025-07-10 01:52:20,460 - root - INFO - Dumping profiler traces at step 55808 +[titan] 2025-07-10 01:52:20,492 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:52:22,079 - root - INFO - step: 55810 loss: 16.6765 memory: 44.58GiB(31.99%) tps: 77,914 tflops: 268.89 mfu: 27.19% global_avg_ntp_loss: 2.5627 global_avg_mtp_loss: 14.1138 +[titan] 2025-07-10 01:52:22,080 - root - INFO - lr: 1.4236e-04 gnorm: 0.77 [12:17:21< 9:43:50] +[titan] 2025-07-10 01:52:25,974 - root - INFO - step: 55815 loss: 16.5638 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 14.0150 +[titan] 2025-07-10 01:52:25,974 - root - INFO - lr: 1.4234e-04 gnorm: 0.81 [12:17:25< 9:43:46] +[titan] 2025-07-10 01:52:29,900 - root - INFO - step: 55820 loss: 16.5167 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9756 +[titan] 2025-07-10 01:52:29,900 - root - INFO - lr: 1.4232e-04 gnorm: 0.80 [12:17:29< 9:43:42] +[titan] 2025-07-10 01:52:33,837 - root - INFO - step: 55825 loss: 16.5764 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5541 global_avg_mtp_loss: 14.0222 +[titan] 2025-07-10 01:52:33,837 - root - INFO - lr: 1.4229e-04 gnorm: 0.85 [12:17:33< 9:43:38] +[titan] 2025-07-10 01:52:37,733 - root - INFO - step: 55830 loss: 16.5789 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 14.0346 +[titan] 2025-07-10 01:52:37,734 - root - INFO - lr: 1.4227e-04 gnorm: 0.77 [12:17:37< 9:43:34] +[titan] 2025-07-10 01:52:41,628 - root - INFO - step: 55835 loss: 16.6696 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.1007 +[titan] 2025-07-10 01:52:41,629 - root - INFO - lr: 1.4225e-04 gnorm: 0.82 [12:17:41< 9:43:30] +[titan] 2025-07-10 01:52:45,546 - root - INFO - step: 55840 loss: 16.6799 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5811 global_avg_mtp_loss: 14.0988 +[titan] 2025-07-10 01:52:45,546 - root - INFO - lr: 1.4223e-04 gnorm: 0.84 [12:17:45< 9:43:26] +[titan] 2025-07-10 01:52:49,442 - root - INFO - step: 55845 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5217 global_avg_mtp_loss: 13.9135 +[titan] 2025-07-10 01:52:49,442 - root - INFO - lr: 1.4221e-04 gnorm: 0.86 [12:17:48< 9:43:22] +[titan] 2025-07-10 01:52:52,564 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:52:53,356 - root - INFO - step: 55850 loss: 16.2210 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7353 +[titan] 2025-07-10 01:52:53,356 - root - INFO - lr: 1.4219e-04 gnorm: 0.86 [12:17:52< 9:43:18] +[titan] 2025-07-10 01:52:57,282 - root - INFO - step: 55855 loss: 16.4077 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.8831 +[titan] 2025-07-10 01:52:57,282 - root - INFO - lr: 1.4217e-04 gnorm: 0.79 [12:17:56< 9:43:14] +[titan] 2025-07-10 01:53:01,231 - root - INFO - step: 55860 loss: 16.5048 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9634 +[titan] 2025-07-10 01:53:01,231 - root - INFO - lr: 1.4215e-04 gnorm: 0.83 [12:18:00< 9:43:10] +[titan] 2025-07-10 01:53:05,133 - root - INFO - step: 55865 loss: 16.5243 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9795 +[titan] 2025-07-10 01:53:05,133 - root - INFO - lr: 1.4213e-04 gnorm: 0.83 [12:18:04< 9:43:06] +[titan] 2025-07-10 01:53:09,045 - root - INFO - step: 55870 loss: 16.6310 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0602 +[titan] 2025-07-10 01:53:09,046 - root - INFO - lr: 1.4210e-04 gnorm: 0.81 [12:18:08< 9:43:02] +[titan] 2025-07-10 01:53:12,949 - root - INFO - step: 55875 loss: 16.4058 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.8843 +[titan] 2025-07-10 01:53:12,949 - root - INFO - lr: 1.4208e-04 gnorm: 0.82 [12:18:12< 9:42:58] +[titan] 2025-07-10 01:53:16,892 - root - INFO - step: 55880 loss: 16.4419 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9125 +[titan] 2025-07-10 01:53:16,892 - root - INFO - lr: 1.4206e-04 gnorm: 0.88 [12:18:16< 9:42:54] +[titan] 2025-07-10 01:53:20,835 - root - INFO - step: 55885 loss: 16.4702 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.9451 +[titan] 2025-07-10 01:53:20,835 - root - INFO - lr: 1.4204e-04 gnorm: 0.81 [12:18:20< 9:42:50] +[titan] 2025-07-10 01:53:24,732 - root - INFO - step: 55890 loss: 16.6110 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 14.0490 +[titan] 2025-07-10 01:53:24,732 - root - INFO - lr: 1.4202e-04 gnorm: 0.82 [12:18:24< 9:42:46] +[titan] 2025-07-10 01:53:28,651 - root - INFO - step: 55895 loss: 16.6119 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5542 global_avg_mtp_loss: 14.0577 +[titan] 2025-07-10 01:53:28,651 - root - INFO - lr: 1.4200e-04 gnorm: 0.83 [12:18:28< 9:42:42] +[titan] 2025-07-10 01:53:31,771 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:53:32,559 - root - INFO - step: 55900 loss: 16.3767 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 01:53:32,559 - root - INFO - lr: 1.4198e-04 gnorm: 0.88 [12:18:32< 9:42:38] +[titan] 2025-07-10 01:53:36,466 - root - INFO - step: 55905 loss: 16.7145 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5850 global_avg_mtp_loss: 14.1295 +[titan] 2025-07-10 01:53:36,466 - root - INFO - lr: 1.4196e-04 gnorm: 0.91 [12:18:35< 9:42:34] +[titan] 2025-07-10 01:53:40,366 - root - INFO - step: 55910 loss: 16.2212 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7250 +[titan] 2025-07-10 01:53:40,367 - root - INFO - lr: 1.4194e-04 gnorm: 0.91 [12:18:39< 9:42:30] +[titan] 2025-07-10 01:53:44,315 - root - INFO - step: 55915 loss: 16.3923 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-10 01:53:44,315 - root - INFO - lr: 1.4191e-04 gnorm: 0.81 [12:18:43< 9:42:26] +[titan] 2025-07-10 01:53:48,261 - root - INFO - step: 55920 loss: 16.3677 memory: 44.58GiB(31.99%) tps: 83,037 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8555 +[titan] 2025-07-10 01:53:48,262 - root - INFO - lr: 1.4189e-04 gnorm: 0.81 [12:18:47< 9:42:22] +[titan] 2025-07-10 01:53:52,169 - root - INFO - step: 55925 loss: 16.1897 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7031 +[titan] 2025-07-10 01:53:52,169 - root - INFO - lr: 1.4187e-04 gnorm: 0.84 [12:18:51< 9:42:18] +[titan] 2025-07-10 01:53:56,066 - root - INFO - step: 55930 loss: 16.1734 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.6898 +[titan] 2025-07-10 01:53:56,067 - root - INFO - lr: 1.4185e-04 gnorm: 0.84 [12:18:55< 9:42:14] +[titan] 2025-07-10 01:53:59,969 - root - INFO - step: 55935 loss: 16.4705 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9367 +[titan] 2025-07-10 01:53:59,969 - root - INFO - lr: 1.4183e-04 gnorm: 0.92 [12:18:59< 9:42:10] +[titan] 2025-07-10 01:54:03,874 - root - INFO - step: 55940 loss: 16.4146 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8962 +[titan] 2025-07-10 01:54:03,874 - root - INFO - lr: 1.4181e-04 gnorm: 0.84 [12:19:03< 9:42:06] +[titan] 2025-07-10 01:54:07,820 - root - INFO - step: 55945 loss: 16.4206 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.8964 +[titan] 2025-07-10 01:54:07,820 - root - INFO - lr: 1.4179e-04 gnorm: 0.88 [12:19:07< 9:42:02] +[titan] 2025-07-10 01:54:10,981 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:54:11,777 - root - INFO - step: 55950 loss: 16.3261 memory: 44.58GiB(31.99%) tps: 82,809 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8180 +[titan] 2025-07-10 01:54:11,777 - root - INFO - lr: 1.4177e-04 gnorm: 0.82 [12:19:11< 9:41:58] +[titan] 2025-07-10 01:54:15,719 - root - INFO - step: 55955 loss: 16.3951 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.8575 +[titan] 2025-07-10 01:54:15,719 - root - INFO - lr: 1.4175e-04 gnorm: 0.84 [12:19:15< 9:41:54] +[titan] 2025-07-10 01:54:19,627 - root - INFO - step: 55960 loss: 16.4520 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9241 +[titan] 2025-07-10 01:54:19,627 - root - INFO - lr: 1.4172e-04 gnorm: 0.87 [12:19:19< 9:41:50] +[titan] 2025-07-10 01:54:23,581 - root - INFO - step: 55965 loss: 16.3541 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8488 +[titan] 2025-07-10 01:54:23,582 - root - INFO - lr: 1.4170e-04 gnorm: 0.88 [12:19:23< 9:41:46] +[titan] 2025-07-10 01:54:27,498 - root - INFO - step: 55970 loss: 16.2706 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.7742 +[titan] 2025-07-10 01:54:27,498 - root - INFO - lr: 1.4168e-04 gnorm: 0.83 [12:19:26< 9:41:42] +[titan] 2025-07-10 01:54:31,418 - root - INFO - step: 55975 loss: 16.6390 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5613 global_avg_mtp_loss: 14.0777 +[titan] 2025-07-10 01:54:31,418 - root - INFO - lr: 1.4166e-04 gnorm: 0.80 [12:19:30< 9:41:38] +[titan] 2025-07-10 01:54:35,326 - root - INFO - step: 55980 loss: 16.3276 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.8348 +[titan] 2025-07-10 01:54:35,326 - root - INFO - lr: 1.4164e-04 gnorm: 0.77 [12:19:34< 9:41:34] +[titan] 2025-07-10 01:54:39,276 - root - INFO - step: 55985 loss: 16.7005 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1242 +[titan] 2025-07-10 01:54:39,277 - root - INFO - lr: 1.4162e-04 gnorm: 0.81 [12:19:38< 9:41:30] +[titan] 2025-07-10 01:54:43,179 - root - INFO - step: 55990 loss: 16.6688 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5790 global_avg_mtp_loss: 14.0898 +[titan] 2025-07-10 01:54:43,179 - root - INFO - lr: 1.4160e-04 gnorm: 0.81 [12:19:42< 9:41:26] +[titan] 2025-07-10 01:54:47,096 - root - INFO - step: 55995 loss: 16.6046 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 14.0529 +[titan] 2025-07-10 01:54:47,097 - root - INFO - lr: 1.4158e-04 gnorm: 0.84 [12:19:46< 9:41:22] +[titan] 2025-07-10 01:54:50,240 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:54:51,028 - root - INFO - step: 56000 loss: 16.5559 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0012 +[titan] 2025-07-10 01:54:51,028 - root - INFO - lr: 1.4156e-04 gnorm: 0.82 [12:19:50< 9:41:18] +[titan] 2025-07-10 01:54:54,924 - root - INFO - step: 56005 loss: 16.4004 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8923 +[titan] 2025-07-10 01:54:54,924 - root - INFO - lr: 1.4153e-04 gnorm: 0.84 [12:19:54< 9:41:14] +[titan] 2025-07-10 01:54:58,840 - root - INFO - step: 56010 loss: 16.5176 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9827 +[titan] 2025-07-10 01:54:58,840 - root - INFO - lr: 1.4151e-04 gnorm: 0.89 [12:19:58< 9:41:10] +[titan] 2025-07-10 01:55:02,757 - root - INFO - step: 56015 loss: 16.3659 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8585 +[titan] 2025-07-10 01:55:02,757 - root - INFO - lr: 1.4149e-04 gnorm: 0.83 [12:20:02< 9:41:06] +[titan] 2025-07-10 01:55:06,681 - root - INFO - step: 56020 loss: 16.4152 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.9021 +[titan] 2025-07-10 01:55:06,682 - root - INFO - lr: 1.4147e-04 gnorm: 0.89 [12:20:06< 9:41:02] +[titan] 2025-07-10 01:55:10,589 - root - INFO - step: 56025 loss: 16.2478 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.7494 +[titan] 2025-07-10 01:55:10,589 - root - INFO - lr: 1.4145e-04 gnorm: 0.83 [12:20:10< 9:40:58] +[titan] 2025-07-10 01:55:14,491 - root - INFO - step: 56030 loss: 16.5692 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0172 +[titan] 2025-07-10 01:55:14,491 - root - INFO - lr: 1.4143e-04 gnorm: 0.78 [12:20:13< 9:40:54] +[titan] 2025-07-10 01:55:18,397 - root - INFO - step: 56035 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.1357 +[titan] 2025-07-10 01:55:18,398 - root - INFO - lr: 1.4141e-04 gnorm: 0.82 [12:20:17< 9:40:50] +[titan] 2025-07-10 01:55:22,308 - root - INFO - step: 56040 loss: 16.0985 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.6242 +[titan] 2025-07-10 01:55:22,309 - root - INFO - lr: 1.4139e-04 gnorm: 0.83 [12:20:21< 9:40:46] +[titan] 2025-07-10 01:55:26,258 - root - INFO - step: 56045 loss: 16.2217 memory: 44.58GiB(31.99%) tps: 82,980 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7410 +[titan] 2025-07-10 01:55:26,258 - root - INFO - lr: 1.4137e-04 gnorm: 0.82 [12:20:25< 9:40:42] +[titan] 2025-07-10 01:55:29,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:55:30,159 - root - INFO - step: 56050 loss: 16.6330 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5690 global_avg_mtp_loss: 14.0640 +[titan] 2025-07-10 01:55:30,159 - root - INFO - lr: 1.4134e-04 gnorm: 0.84 [12:20:29< 9:40:38] +[titan] 2025-07-10 01:55:34,069 - root - INFO - step: 56055 loss: 16.5442 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 14.0000 +[titan] 2025-07-10 01:55:34,070 - root - INFO - lr: 1.4132e-04 gnorm: 0.87 [12:20:33< 9:40:34] +[titan] 2025-07-10 01:55:37,973 - root - INFO - step: 56060 loss: 16.6765 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.0901 +[titan] 2025-07-10 01:55:37,973 - root - INFO - lr: 1.4130e-04 gnorm: 0.96 [12:20:37< 9:40:30] +[titan] 2025-07-10 01:55:41,932 - root - INFO - step: 56065 loss: 16.7628 memory: 44.58GiB(31.99%) tps: 82,783 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.5876 global_avg_mtp_loss: 14.1752 +[titan] 2025-07-10 01:55:41,932 - root - INFO - lr: 1.4128e-04 gnorm: 0.87 [12:20:41< 9:40:26] +[titan] 2025-07-10 01:55:45,854 - root - INFO - step: 56070 loss: 16.4264 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.8951 +[titan] 2025-07-10 01:55:45,854 - root - INFO - lr: 1.4126e-04 gnorm: 0.87 [12:20:45< 9:40:22] +[titan] 2025-07-10 01:55:49,764 - root - INFO - step: 56075 loss: 16.6378 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0790 +[titan] 2025-07-10 01:55:49,765 - root - INFO - lr: 1.4124e-04 gnorm: 0.83 [12:20:49< 9:40:18] +[titan] 2025-07-10 01:55:53,680 - root - INFO - step: 56080 loss: 16.0279 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4467 global_avg_mtp_loss: 13.5812 +[titan] 2025-07-10 01:55:53,680 - root - INFO - lr: 1.4122e-04 gnorm: 0.80 [12:20:53< 9:40:14] +[titan] 2025-07-10 01:55:57,609 - root - INFO - step: 56085 loss: 16.0475 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.6018 +[titan] 2025-07-10 01:55:57,610 - root - INFO - lr: 1.4120e-04 gnorm: 0.80 [12:20:57< 9:40:10] +[titan] 2025-07-10 01:56:01,506 - root - INFO - step: 56090 loss: 16.7510 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5894 global_avg_mtp_loss: 14.1615 +[titan] 2025-07-10 01:56:01,506 - root - INFO - lr: 1.4118e-04 gnorm: 0.88 [12:21:00< 9:40:06] +[titan] 2025-07-10 01:56:05,411 - root - INFO - step: 56095 loss: 16.4759 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.9473 +[titan] 2025-07-10 01:56:05,411 - root - INFO - lr: 1.4115e-04 gnorm: 0.87 [12:21:04< 9:40:02] +[titan] 2025-07-10 01:56:08,535 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:56:09,321 - root - INFO - step: 56100 loss: 16.5039 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 13.9640 +[titan] 2025-07-10 01:56:09,322 - root - INFO - lr: 1.4113e-04 gnorm: 0.90 [12:21:08< 9:39:58] +[titan] 2025-07-10 01:56:13,232 - root - INFO - step: 56105 loss: 16.5705 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5446 global_avg_mtp_loss: 14.0259 +[titan] 2025-07-10 01:56:13,232 - root - INFO - lr: 1.4111e-04 gnorm: 0.87 [12:21:12< 9:39:54] +[titan] 2025-07-10 01:56:17,177 - root - INFO - step: 56110 loss: 16.2614 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.7571 +[titan] 2025-07-10 01:56:17,178 - root - INFO - lr: 1.4109e-04 gnorm: 0.82 [12:21:16< 9:39:50] +[titan] 2025-07-10 01:56:21,078 - root - INFO - step: 56115 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9480 +[titan] 2025-07-10 01:56:21,078 - root - INFO - lr: 1.4107e-04 gnorm: 0.82 [12:21:20< 9:39:46] +[titan] 2025-07-10 01:56:25,001 - root - INFO - step: 56120 loss: 16.5553 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-10 01:56:25,001 - root - INFO - lr: 1.4105e-04 gnorm: 0.85 [12:21:24< 9:39:42] +[titan] 2025-07-10 01:56:28,980 - root - INFO - step: 56125 loss: 16.3850 memory: 44.58GiB(31.99%) tps: 82,355 tflops: 284.22 mfu: 28.74% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8856 +[titan] 2025-07-10 01:56:28,981 - root - INFO - lr: 1.4103e-04 gnorm: 0.85 [12:21:28< 9:39:38] +[titan] 2025-07-10 01:56:32,910 - root - INFO - step: 56130 loss: 16.1366 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6571 +[titan] 2025-07-10 01:56:32,910 - root - INFO - lr: 1.4101e-04 gnorm: 0.87 [12:21:32< 9:39:34] +[titan] 2025-07-10 01:56:36,814 - root - INFO - step: 56135 loss: 16.3080 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.8239 +[titan] 2025-07-10 01:56:36,814 - root - INFO - lr: 1.4099e-04 gnorm: 0.84 [12:21:36< 9:39:30] +[titan] 2025-07-10 01:56:40,710 - root - INFO - step: 56140 loss: 16.4576 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 13.9155 +[titan] 2025-07-10 01:56:40,710 - root - INFO - lr: 1.4097e-04 gnorm: 0.82 [12:21:40< 9:39:26] +[titan] 2025-07-10 01:56:44,615 - root - INFO - step: 56145 loss: 16.4288 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.8958 +[titan] 2025-07-10 01:56:44,615 - root - INFO - lr: 1.4094e-04 gnorm: 0.90 [12:21:44< 9:39:22] +[titan] 2025-07-10 01:56:47,757 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:56:48,544 - root - INFO - step: 56150 loss: 16.5634 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 14.0155 +[titan] 2025-07-10 01:56:48,545 - root - INFO - lr: 1.4092e-04 gnorm: 0.88 [12:21:48< 9:39:18] +[titan] 2025-07-10 01:56:52,473 - root - INFO - step: 56155 loss: 16.4651 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-10 01:56:52,473 - root - INFO - lr: 1.4090e-04 gnorm: 0.87 [12:21:51< 9:39:14] +[titan] 2025-07-10 01:56:56,397 - root - INFO - step: 56160 loss: 16.2820 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.7798 +[titan] 2025-07-10 01:56:56,397 - root - INFO - lr: 1.4088e-04 gnorm: 0.92 [12:21:55< 9:39:10] +[titan] 2025-07-10 01:57:00,290 - root - INFO - step: 56165 loss: 16.5596 memory: 44.58GiB(31.99%) tps: 84,182 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0014 +[titan] 2025-07-10 01:57:00,290 - root - INFO - lr: 1.4086e-04 gnorm: 0.94 [12:21:59< 9:39:06] +[titan] 2025-07-10 01:57:04,203 - root - INFO - step: 56170 loss: 16.6549 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0962 +[titan] 2025-07-10 01:57:04,204 - root - INFO - lr: 1.4084e-04 gnorm: 0.85 [12:22:03< 9:39:02] +[titan] 2025-07-10 01:57:08,144 - root - INFO - step: 56175 loss: 16.3745 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8559 +[titan] 2025-07-10 01:57:08,144 - root - INFO - lr: 1.4082e-04 gnorm: 0.87 [12:22:07< 9:38:58] +[titan] 2025-07-10 01:57:12,044 - root - INFO - step: 56180 loss: 16.5701 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 14.0139 +[titan] 2025-07-10 01:57:12,045 - root - INFO - lr: 1.4080e-04 gnorm: 0.87 [12:22:11< 9:38:54] +[titan] 2025-07-10 01:57:15,960 - root - INFO - step: 56185 loss: 16.4078 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.8865 +[titan] 2025-07-10 01:57:15,961 - root - INFO - lr: 1.4078e-04 gnorm: 0.83 [12:22:15< 9:38:50] +[titan] 2025-07-10 01:57:19,864 - root - INFO - step: 56190 loss: 16.1957 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7173 +[titan] 2025-07-10 01:57:19,864 - root - INFO - lr: 1.4075e-04 gnorm: 0.89 [12:22:19< 9:38:46] +[titan] 2025-07-10 01:57:23,774 - root - INFO - step: 56195 loss: 16.3168 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5184 global_avg_mtp_loss: 13.7985 +[titan] 2025-07-10 01:57:23,774 - root - INFO - lr: 1.4073e-04 gnorm: 0.82 [12:22:23< 9:38:42] +[titan] 2025-07-10 01:57:26,909 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:57:27,702 - root - INFO - step: 56200 loss: 16.4983 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9532 +[titan] 2025-07-10 01:57:27,702 - root - INFO - lr: 1.4071e-04 gnorm: 0.79 [12:22:27< 9:38:38] +[titan] 2025-07-10 01:57:31,605 - root - INFO - step: 56205 loss: 16.6205 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 14.0670 +[titan] 2025-07-10 01:57:31,605 - root - INFO - lr: 1.4069e-04 gnorm: 0.86 [12:22:31< 9:38:34] +[titan] 2025-07-10 01:57:35,521 - root - INFO - step: 56210 loss: 16.3880 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 01:57:35,522 - root - INFO - lr: 1.4067e-04 gnorm: 0.80 [12:22:34< 9:38:30] +[titan] 2025-07-10 01:57:39,463 - root - INFO - step: 56215 loss: 16.3427 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8299 +[titan] 2025-07-10 01:57:39,463 - root - INFO - lr: 1.4065e-04 gnorm: 0.93 [12:22:38< 9:38:26] +[titan] 2025-07-10 01:57:43,380 - root - INFO - step: 56220 loss: 16.2511 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7652 +[titan] 2025-07-10 01:57:43,380 - root - INFO - lr: 1.4063e-04 gnorm: 0.88 [12:22:42< 9:38:22] +[titan] 2025-07-10 01:57:47,280 - root - INFO - step: 56225 loss: 16.3957 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8802 +[titan] 2025-07-10 01:57:47,280 - root - INFO - lr: 1.4061e-04 gnorm: 0.85 [12:22:46< 9:38:18] +[titan] 2025-07-10 01:57:51,214 - root - INFO - step: 56230 loss: 16.3614 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.8484 +[titan] 2025-07-10 01:57:51,215 - root - INFO - lr: 1.4059e-04 gnorm: 0.88 [12:22:50< 9:38:14] +[titan] 2025-07-10 01:57:55,145 - root - INFO - step: 56235 loss: 16.4308 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.9120 +[titan] 2025-07-10 01:57:55,145 - root - INFO - lr: 1.4056e-04 gnorm: 0.84 [12:22:54< 9:38:10] +[titan] 2025-07-10 01:57:59,102 - root - INFO - step: 56240 loss: 16.6465 memory: 44.58GiB(31.99%) tps: 82,817 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0889 +[titan] 2025-07-10 01:57:59,102 - root - INFO - lr: 1.4054e-04 gnorm: 0.84 [12:22:58< 9:38:06] +[titan] 2025-07-10 01:58:03,014 - root - INFO - step: 56245 loss: 16.4046 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.8643 +[titan] 2025-07-10 01:58:03,014 - root - INFO - lr: 1.4052e-04 gnorm: 0.84 [12:23:02< 9:38:02] +[titan] 2025-07-10 01:58:06,142 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:58:06,945 - root - INFO - step: 56250 loss: 16.6733 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0979 +[titan] 2025-07-10 01:58:06,945 - root - INFO - lr: 1.4050e-04 gnorm: 0.83 [12:23:06< 9:37:58] +[titan] 2025-07-10 01:58:10,856 - root - INFO - step: 56255 loss: 16.5468 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 14.0103 +[titan] 2025-07-10 01:58:10,856 - root - INFO - lr: 1.4048e-04 gnorm: 0.79 [12:23:10< 9:37:54] +[titan] 2025-07-10 01:58:14,765 - root - INFO - step: 56260 loss: 16.5143 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5342 global_avg_mtp_loss: 13.9801 +[titan] 2025-07-10 01:58:14,765 - root - INFO - lr: 1.4046e-04 gnorm: 0.84 [12:23:14< 9:37:50] +[titan] 2025-07-10 01:58:18,666 - root - INFO - step: 56265 loss: 16.6667 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.1007 +[titan] 2025-07-10 01:58:18,666 - root - INFO - lr: 1.4044e-04 gnorm: 0.86 [12:23:18< 9:37:46] +[titan] 2025-07-10 01:58:22,572 - root - INFO - step: 56270 loss: 16.1824 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.7135 +[titan] 2025-07-10 01:58:22,572 - root - INFO - lr: 1.4042e-04 gnorm: 0.88 [12:23:22< 9:37:42] +[titan] 2025-07-10 01:58:26,474 - root - INFO - step: 56275 loss: 16.4814 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9478 +[titan] 2025-07-10 01:58:26,474 - root - INFO - lr: 1.4040e-04 gnorm: 0.83 [12:23:25< 9:37:38] +[titan] 2025-07-10 01:58:30,401 - root - INFO - step: 56280 loss: 16.0595 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.5856 +[titan] 2025-07-10 01:58:30,401 - root - INFO - lr: 1.4038e-04 gnorm: 0.80 [12:23:29< 9:37:34] +[titan] 2025-07-10 01:58:34,315 - root - INFO - step: 56285 loss: 16.5642 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 14.0214 +[titan] 2025-07-10 01:58:34,316 - root - INFO - lr: 1.4035e-04 gnorm: 0.84 [12:23:33< 9:37:30] +[titan] 2025-07-10 01:58:38,233 - root - INFO - step: 56290 loss: 16.3291 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8270 +[titan] 2025-07-10 01:58:38,233 - root - INFO - lr: 1.4033e-04 gnorm: 0.86 [12:23:37< 9:37:26] +[titan] 2025-07-10 01:58:42,134 - root - INFO - step: 56295 loss: 16.7424 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.1593 +[titan] 2025-07-10 01:58:42,134 - root - INFO - lr: 1.4031e-04 gnorm: 0.90 [12:23:41< 9:37:22] +[titan] 2025-07-10 01:58:45,251 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:58:46,044 - root - INFO - step: 56300 loss: 16.5100 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9772 +[titan] 2025-07-10 01:58:46,044 - root - INFO - lr: 1.4029e-04 gnorm: 0.87 [12:23:45< 9:37:18] +[titan] 2025-07-10 01:58:49,964 - root - INFO - step: 56305 loss: 16.2767 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.7705 +[titan] 2025-07-10 01:58:49,964 - root - INFO - lr: 1.4027e-04 gnorm: 0.85 [12:23:49< 9:37:14] +[titan] 2025-07-10 01:58:53,855 - root - INFO - step: 56310 loss: 16.5852 memory: 44.58GiB(31.99%) tps: 84,208 tflops: 290.62 mfu: 29.38% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0275 +[titan] 2025-07-10 01:58:53,856 - root - INFO - lr: 1.4025e-04 gnorm: 0.80 [12:23:53< 9:37:10] +[titan] 2025-07-10 01:58:57,757 - root - INFO - step: 56315 loss: 16.6416 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0782 +[titan] 2025-07-10 01:58:57,758 - root - INFO - lr: 1.4023e-04 gnorm: 0.80 [12:23:57< 9:37:06] +[titan] 2025-07-10 01:59:01,747 - root - INFO - step: 56320 loss: 16.4577 memory: 44.58GiB(31.99%) tps: 82,138 tflops: 283.47 mfu: 28.66% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 13.9182 +[titan] 2025-07-10 01:59:01,747 - root - INFO - lr: 1.4021e-04 gnorm: 0.80 [12:24:01< 9:37:02] +[titan] 2025-07-10 01:59:01,902 - root - INFO - Dumping profiler traces at step 56320 +[titan] 2025-07-10 01:59:01,935 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 01:59:05,847 - root - INFO - step: 56325 loss: 16.6031 memory: 44.58GiB(31.99%) tps: 79,938 tflops: 275.88 mfu: 27.89% global_avg_ntp_loss: 2.5529 global_avg_mtp_loss: 14.0501 +[titan] 2025-07-10 01:59:05,847 - root - INFO - lr: 1.4019e-04 gnorm: 0.85 [12:24:05< 9:36:58] +[titan] 2025-07-10 01:59:09,756 - root - INFO - step: 56330 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9444 +[titan] 2025-07-10 01:59:09,756 - root - INFO - lr: 1.4016e-04 gnorm: 0.88 [12:24:09< 9:36:54] +[titan] 2025-07-10 01:59:13,686 - root - INFO - step: 56335 loss: 16.4591 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.9238 +[titan] 2025-07-10 01:59:13,686 - root - INFO - lr: 1.4014e-04 gnorm: 0.87 [12:24:13< 9:36:50] +[titan] 2025-07-10 01:59:17,609 - root - INFO - step: 56340 loss: 16.5360 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 14.0125 +[titan] 2025-07-10 01:59:17,609 - root - INFO - lr: 1.4012e-04 gnorm: 0.94 [12:24:17< 9:36:46] +[titan] 2025-07-10 01:59:21,540 - root - INFO - step: 56345 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.8359 +[titan] 2025-07-10 01:59:21,540 - root - INFO - lr: 1.4010e-04 gnorm: 0.92 [12:24:21< 9:36:42] +[titan] 2025-07-10 01:59:24,658 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 01:59:25,442 - root - INFO - step: 56350 loss: 16.5286 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 13.9849 +[titan] 2025-07-10 01:59:25,442 - root - INFO - lr: 1.4008e-04 gnorm: 0.86 [12:24:24< 9:36:38] +[titan] 2025-07-10 01:59:29,359 - root - INFO - step: 56355 loss: 16.3957 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8760 +[titan] 2025-07-10 01:59:29,359 - root - INFO - lr: 1.4006e-04 gnorm: 0.85 [12:24:28< 9:36:34] +[titan] 2025-07-10 01:59:33,257 - root - INFO - step: 56360 loss: 16.3738 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 01:59:33,257 - root - INFO - lr: 1.4004e-04 gnorm: 0.81 [12:24:32< 9:36:30] +[titan] 2025-07-10 01:59:37,181 - root - INFO - step: 56365 loss: 16.5087 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9808 +[titan] 2025-07-10 01:59:37,182 - root - INFO - lr: 1.4002e-04 gnorm: 0.81 [12:24:36< 9:36:26] +[titan] 2025-07-10 01:59:41,087 - root - INFO - step: 56370 loss: 16.6111 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 14.0492 +[titan] 2025-07-10 01:59:41,088 - root - INFO - lr: 1.4000e-04 gnorm: 0.86 [12:24:40< 9:36:22] +[titan] 2025-07-10 01:59:45,056 - root - INFO - step: 56375 loss: 16.5412 memory: 44.58GiB(31.99%) tps: 82,582 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.5497 global_avg_mtp_loss: 13.9914 +[titan] 2025-07-10 01:59:45,056 - root - INFO - lr: 1.3998e-04 gnorm: 0.82 [12:24:44< 9:36:18] +[titan] 2025-07-10 01:59:48,987 - root - INFO - step: 56380 loss: 16.5576 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-10 01:59:48,987 - root - INFO - lr: 1.3995e-04 gnorm: 0.82 [12:24:48< 9:36:14] +[titan] 2025-07-10 01:59:52,886 - root - INFO - step: 56385 loss: 16.1707 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.6847 +[titan] 2025-07-10 01:59:52,886 - root - INFO - lr: 1.3993e-04 gnorm: 0.99 [12:24:52< 9:36:10] +[titan] 2025-07-10 01:59:56,784 - root - INFO - step: 56390 loss: 16.2919 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7943 +[titan] 2025-07-10 01:59:56,785 - root - INFO - lr: 1.3991e-04 gnorm: 0.85 [12:24:56< 9:36:06] +[titan] 2025-07-10 02:00:00,677 - root - INFO - step: 56395 loss: 16.3767 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8681 +[titan] 2025-07-10 02:00:00,677 - root - INFO - lr: 1.3989e-04 gnorm: 0.79 [12:25:00< 9:36:02] +[titan] 2025-07-10 02:00:03,778 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:00:04,577 - root - INFO - step: 56400 loss: 16.3667 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5165 global_avg_mtp_loss: 13.8502 +[titan] 2025-07-10 02:00:04,577 - root - INFO - lr: 1.3987e-04 gnorm: 0.80 [12:25:04< 9:35:58] +[titan] 2025-07-10 02:00:08,483 - root - INFO - step: 56405 loss: 16.3768 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.8719 +[titan] 2025-07-10 02:00:08,483 - root - INFO - lr: 1.3985e-04 gnorm: 0.81 [12:25:07< 9:35:54] +[titan] 2025-07-10 02:00:12,392 - root - INFO - step: 56410 loss: 16.3318 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8314 +[titan] 2025-07-10 02:00:12,392 - root - INFO - lr: 1.3983e-04 gnorm: 0.80 [12:25:11< 9:35:50] +[titan] 2025-07-10 02:00:16,327 - root - INFO - step: 56415 loss: 16.4053 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.8858 +[titan] 2025-07-10 02:00:16,327 - root - INFO - lr: 1.3981e-04 gnorm: 0.84 [12:25:15< 9:35:46] +[titan] 2025-07-10 02:00:20,294 - root - INFO - step: 56420 loss: 16.0224 memory: 44.58GiB(31.99%) tps: 82,594 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.5715 +[titan] 2025-07-10 02:00:20,295 - root - INFO - lr: 1.3979e-04 gnorm: 0.87 [12:25:19< 9:35:42] +[titan] 2025-07-10 02:00:24,218 - root - INFO - step: 56425 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8258 +[titan] 2025-07-10 02:00:24,218 - root - INFO - lr: 1.3976e-04 gnorm: 0.97 [12:25:23< 9:35:38] +[titan] 2025-07-10 02:00:28,152 - root - INFO - step: 56430 loss: 16.8074 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5861 global_avg_mtp_loss: 14.2213 +[titan] 2025-07-10 02:00:28,153 - root - INFO - lr: 1.3974e-04 gnorm: 0.85 [12:25:27< 9:35:34] +[titan] 2025-07-10 02:00:32,072 - root - INFO - step: 56435 loss: 16.4441 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5390 global_avg_mtp_loss: 13.9051 +[titan] 2025-07-10 02:00:32,073 - root - INFO - lr: 1.3972e-04 gnorm: 0.83 [12:25:31< 9:35:30] +[titan] 2025-07-10 02:00:35,985 - root - INFO - step: 56440 loss: 16.2432 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.7468 +[titan] 2025-07-10 02:00:35,985 - root - INFO - lr: 1.3970e-04 gnorm: 0.80 [12:25:35< 9:35:26] +[titan] 2025-07-10 02:00:39,883 - root - INFO - step: 56445 loss: 16.1447 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.6647 +[titan] 2025-07-10 02:00:39,883 - root - INFO - lr: 1.3968e-04 gnorm: 0.82 [12:25:39< 9:35:22] +[titan] 2025-07-10 02:00:43,005 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:00:43,791 - root - INFO - step: 56450 loss: 16.6341 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0747 +[titan] 2025-07-10 02:00:43,791 - root - INFO - lr: 1.3966e-04 gnorm: 0.85 [12:25:43< 9:35:18] +[titan] 2025-07-10 02:00:47,715 - root - INFO - step: 56455 loss: 16.2009 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7170 +[titan] 2025-07-10 02:00:47,715 - root - INFO - lr: 1.3964e-04 gnorm: 0.85 [12:25:47< 9:35:14] +[titan] 2025-07-10 02:00:51,643 - root - INFO - step: 56460 loss: 16.5297 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9849 +[titan] 2025-07-10 02:00:51,643 - root - INFO - lr: 1.3962e-04 gnorm: 0.90 [12:25:51< 9:35:10] +[titan] 2025-07-10 02:00:55,554 - root - INFO - step: 56465 loss: 16.5265 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9846 +[titan] 2025-07-10 02:00:55,554 - root - INFO - lr: 1.3960e-04 gnorm: 0.89 [12:25:55< 9:35:06] +[titan] 2025-07-10 02:00:59,526 - root - INFO - step: 56470 loss: 16.3290 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.74 mfu: 28.79% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8188 +[titan] 2025-07-10 02:00:59,526 - root - INFO - lr: 1.3958e-04 gnorm: 0.87 [12:25:58< 9:35:02] +[titan] 2025-07-10 02:01:03,423 - root - INFO - step: 56475 loss: 16.2092 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.7123 +[titan] 2025-07-10 02:01:03,423 - root - INFO - lr: 1.3955e-04 gnorm: 0.84 [12:26:02< 9:34:58] +[titan] 2025-07-10 02:01:07,326 - root - INFO - step: 56480 loss: 16.2917 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.7968 +[titan] 2025-07-10 02:01:07,326 - root - INFO - lr: 1.3953e-04 gnorm: 0.82 [12:26:06< 9:34:54] +[titan] 2025-07-10 02:01:11,276 - root - INFO - step: 56485 loss: 16.4733 memory: 44.58GiB(31.99%) tps: 82,965 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9489 +[titan] 2025-07-10 02:01:11,276 - root - INFO - lr: 1.3951e-04 gnorm: 0.82 [12:26:10< 9:34:50] +[titan] 2025-07-10 02:01:15,219 - root - INFO - step: 56490 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.8552 +[titan] 2025-07-10 02:01:15,219 - root - INFO - lr: 1.3949e-04 gnorm: 0.87 [12:26:14< 9:34:46] +[titan] 2025-07-10 02:01:19,136 - root - INFO - step: 56495 loss: 16.5578 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 14.0106 +[titan] 2025-07-10 02:01:19,136 - root - INFO - lr: 1.3947e-04 gnorm: 0.88 [12:26:18< 9:34:42] +[titan] 2025-07-10 02:01:22,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:01:23,035 - root - INFO - step: 56500 loss: 16.6148 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5780 global_avg_mtp_loss: 14.0368 +[titan] 2025-07-10 02:01:23,035 - root - INFO - lr: 1.3945e-04 gnorm: 1.06 [12:26:22< 9:34:38] +[titan] 2025-07-10 02:01:26,947 - root - INFO - step: 56505 loss: 16.4759 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-10 02:01:26,948 - root - INFO - lr: 1.3943e-04 gnorm: 0.86 [12:26:26< 9:34:34] +[titan] 2025-07-10 02:01:30,894 - root - INFO - step: 56510 loss: 16.6210 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0657 +[titan] 2025-07-10 02:01:30,894 - root - INFO - lr: 1.3941e-04 gnorm: 0.86 [12:26:30< 9:34:30] +[titan] 2025-07-10 02:01:34,839 - root - INFO - step: 56515 loss: 16.3837 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8838 +[titan] 2025-07-10 02:01:34,839 - root - INFO - lr: 1.3939e-04 gnorm: 0.83 [12:26:34< 9:34:26] +[titan] 2025-07-10 02:01:38,774 - root - INFO - step: 56520 loss: 16.3616 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-10 02:01:38,774 - root - INFO - lr: 1.3937e-04 gnorm: 0.82 [12:26:38< 9:34:22] +[titan] 2025-07-10 02:01:42,704 - root - INFO - step: 56525 loss: 16.6878 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5674 global_avg_mtp_loss: 14.1204 +[titan] 2025-07-10 02:01:42,704 - root - INFO - lr: 1.3934e-04 gnorm: 0.82 [12:26:42< 9:34:18] +[titan] 2025-07-10 02:01:46,668 - root - INFO - step: 56530 loss: 16.5651 memory: 44.58GiB(31.99%) tps: 82,674 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0176 +[titan] 2025-07-10 02:01:46,668 - root - INFO - lr: 1.3932e-04 gnorm: 0.83 [12:26:46< 9:34:14] +[titan] 2025-07-10 02:01:50,574 - root - INFO - step: 56535 loss: 16.7404 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5736 global_avg_mtp_loss: 14.1667 +[titan] 2025-07-10 02:01:50,575 - root - INFO - lr: 1.3930e-04 gnorm: 0.85 [12:26:50< 9:34:10] +[titan] 2025-07-10 02:01:54,485 - root - INFO - step: 56540 loss: 16.5822 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0285 +[titan] 2025-07-10 02:01:54,485 - root - INFO - lr: 1.3928e-04 gnorm: 0.82 [12:26:53< 9:34:06] +[titan] 2025-07-10 02:01:58,402 - root - INFO - step: 56545 loss: 16.2090 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.7368 +[titan] 2025-07-10 02:01:58,402 - root - INFO - lr: 1.3926e-04 gnorm: 0.84 [12:26:57< 9:34:02] +[titan] 2025-07-10 02:02:01,554 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:02:02,346 - root - INFO - step: 56550 loss: 16.3200 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8181 +[titan] 2025-07-10 02:02:02,347 - root - INFO - lr: 1.3924e-04 gnorm: 0.94 [12:27:01< 9:33:58] +[titan] 2025-07-10 02:02:06,273 - root - INFO - step: 56555 loss: 16.4643 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.9385 +[titan] 2025-07-10 02:02:06,273 - root - INFO - lr: 1.3922e-04 gnorm: 0.83 [12:27:05< 9:33:54] +[titan] 2025-07-10 02:02:10,196 - root - INFO - step: 56560 loss: 16.4785 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 13.9474 +[titan] 2025-07-10 02:02:10,196 - root - INFO - lr: 1.3920e-04 gnorm: 0.82 [12:27:09< 9:33:50] +[titan] 2025-07-10 02:02:14,095 - root - INFO - step: 56565 loss: 16.3985 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8917 +[titan] 2025-07-10 02:02:14,095 - root - INFO - lr: 1.3918e-04 gnorm: 0.86 [12:27:13< 9:33:46] +[titan] 2025-07-10 02:02:18,011 - root - INFO - step: 56570 loss: 16.3286 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 02:02:18,012 - root - INFO - lr: 1.3915e-04 gnorm: 0.82 [12:27:17< 9:33:42] +[titan] 2025-07-10 02:02:21,907 - root - INFO - step: 56575 loss: 16.2414 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7587 +[titan] 2025-07-10 02:02:21,907 - root - INFO - lr: 1.3913e-04 gnorm: 0.84 [12:27:21< 9:33:38] +[titan] 2025-07-10 02:02:25,827 - root - INFO - step: 56580 loss: 16.4902 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 13.9555 +[titan] 2025-07-10 02:02:25,827 - root - INFO - lr: 1.3911e-04 gnorm: 0.82 [12:27:25< 9:33:34] +[titan] 2025-07-10 02:02:29,753 - root - INFO - step: 56585 loss: 16.5930 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0294 +[titan] 2025-07-10 02:02:29,753 - root - INFO - lr: 1.3909e-04 gnorm: 0.78 [12:27:29< 9:33:30] +[titan] 2025-07-10 02:02:33,664 - root - INFO - step: 56590 loss: 16.4993 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.9696 +[titan] 2025-07-10 02:02:33,665 - root - INFO - lr: 1.3907e-04 gnorm: 0.81 [12:27:33< 9:33:26] +[titan] 2025-07-10 02:02:37,553 - root - INFO - step: 56595 loss: 16.5632 memory: 44.58GiB(31.99%) tps: 84,266 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0160 +[titan] 2025-07-10 02:02:37,554 - root - INFO - lr: 1.3905e-04 gnorm: 0.86 [12:27:36< 9:33:22] +[titan] 2025-07-10 02:02:40,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:02:41,470 - root - INFO - step: 56600 loss: 16.6575 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.0860 +[titan] 2025-07-10 02:02:41,470 - root - INFO - lr: 1.3903e-04 gnorm: 0.86 [12:27:40< 9:33:18] +[titan] 2025-07-10 02:02:45,402 - root - INFO - step: 56605 loss: 16.6002 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0402 +[titan] 2025-07-10 02:02:45,402 - root - INFO - lr: 1.3901e-04 gnorm: 0.84 [12:27:44< 9:33:14] +[titan] 2025-07-10 02:02:49,296 - root - INFO - step: 56610 loss: 16.6389 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0731 +[titan] 2025-07-10 02:02:49,296 - root - INFO - lr: 1.3899e-04 gnorm: 0.86 [12:27:48< 9:33:10] +[titan] 2025-07-10 02:02:53,233 - root - INFO - step: 56615 loss: 16.5307 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9914 +[titan] 2025-07-10 02:02:53,233 - root - INFO - lr: 1.3897e-04 gnorm: 0.85 [12:27:52< 9:33:06] +[titan] 2025-07-10 02:02:57,150 - root - INFO - step: 56620 loss: 16.3354 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8331 +[titan] 2025-07-10 02:02:57,151 - root - INFO - lr: 1.3894e-04 gnorm: 0.91 [12:27:56< 9:33:02] +[titan] 2025-07-10 02:03:01,039 - root - INFO - step: 56625 loss: 16.2891 memory: 44.58GiB(31.99%) tps: 84,279 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 02:03:01,039 - root - INFO - lr: 1.3892e-04 gnorm: 0.83 [12:28:00< 9:32:58] +[titan] 2025-07-10 02:03:04,950 - root - INFO - step: 56630 loss: 16.5656 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-10 02:03:04,950 - root - INFO - lr: 1.3890e-04 gnorm: 0.83 [12:28:04< 9:32:54] +[titan] 2025-07-10 02:03:08,854 - root - INFO - step: 56635 loss: 16.5969 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5589 global_avg_mtp_loss: 14.0380 +[titan] 2025-07-10 02:03:08,855 - root - INFO - lr: 1.3888e-04 gnorm: 0.81 [12:28:08< 9:32:50] +[titan] 2025-07-10 02:03:12,774 - root - INFO - step: 56640 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8140 +[titan] 2025-07-10 02:03:12,774 - root - INFO - lr: 1.3886e-04 gnorm: 0.80 [12:28:12< 9:32:46] +[titan] 2025-07-10 02:03:16,680 - root - INFO - step: 56645 loss: 16.2922 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.7891 +[titan] 2025-07-10 02:03:16,680 - root - INFO - lr: 1.3884e-04 gnorm: 0.88 [12:28:16< 9:32:42] +[titan] 2025-07-10 02:03:19,810 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:03:20,596 - root - INFO - step: 56650 loss: 16.4669 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9325 +[titan] 2025-07-10 02:03:20,596 - root - INFO - lr: 1.3882e-04 gnorm: 0.84 [12:28:20< 9:32:38] +[titan] 2025-07-10 02:03:24,547 - root - INFO - step: 56655 loss: 16.4386 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9121 +[titan] 2025-07-10 02:03:24,548 - root - INFO - lr: 1.3880e-04 gnorm: 0.83 [12:28:23< 9:32:34] +[titan] 2025-07-10 02:03:28,482 - root - INFO - step: 56660 loss: 16.5999 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0451 +[titan] 2025-07-10 02:03:28,483 - root - INFO - lr: 1.3878e-04 gnorm: 0.85 [12:28:27< 9:32:30] +[titan] 2025-07-10 02:03:32,406 - root - INFO - step: 56665 loss: 16.3633 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8528 +[titan] 2025-07-10 02:03:32,406 - root - INFO - lr: 1.3876e-04 gnorm: 0.85 [12:28:31< 9:32:26] +[titan] 2025-07-10 02:03:36,310 - root - INFO - step: 56670 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7076 +[titan] 2025-07-10 02:03:36,310 - root - INFO - lr: 1.3873e-04 gnorm: 0.86 [12:28:35< 9:32:22] +[titan] 2025-07-10 02:03:40,228 - root - INFO - step: 56675 loss: 16.5203 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9810 +[titan] 2025-07-10 02:03:40,228 - root - INFO - lr: 1.3871e-04 gnorm: 0.85 [12:28:39< 9:32:18] +[titan] 2025-07-10 02:03:44,128 - root - INFO - step: 56680 loss: 16.3892 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8812 +[titan] 2025-07-10 02:03:44,129 - root - INFO - lr: 1.3869e-04 gnorm: 0.89 [12:28:43< 9:32:14] +[titan] 2025-07-10 02:03:48,034 - root - INFO - step: 56685 loss: 16.5692 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 14.0307 +[titan] 2025-07-10 02:03:48,034 - root - INFO - lr: 1.3867e-04 gnorm: 0.80 [12:28:47< 9:32:10] +[titan] 2025-07-10 02:03:51,963 - root - INFO - step: 56690 loss: 16.4602 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9319 +[titan] 2025-07-10 02:03:51,964 - root - INFO - lr: 1.3865e-04 gnorm: 0.82 [12:28:51< 9:32:06] +[titan] 2025-07-10 02:03:55,904 - root - INFO - step: 56695 loss: 16.7014 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.1267 +[titan] 2025-07-10 02:03:55,904 - root - INFO - lr: 1.3863e-04 gnorm: 0.85 [12:28:55< 9:32:02] +[titan] 2025-07-10 02:03:59,056 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:03:59,845 - root - INFO - step: 56700 loss: 16.5201 memory: 44.58GiB(31.99%) tps: 83,143 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9727 +[titan] 2025-07-10 02:03:59,845 - root - INFO - lr: 1.3861e-04 gnorm: 0.86 [12:28:59< 9:31:58] +[titan] 2025-07-10 02:04:03,753 - root - INFO - step: 56705 loss: 16.5100 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 13.9579 +[titan] 2025-07-10 02:04:03,754 - root - INFO - lr: 1.3859e-04 gnorm: 0.81 [12:29:03< 9:31:54] +[titan] 2025-07-10 02:04:07,689 - root - INFO - step: 56710 loss: 16.4595 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-10 02:04:07,689 - root - INFO - lr: 1.3857e-04 gnorm: 0.87 [12:29:07< 9:31:50] +[titan] 2025-07-10 02:04:11,603 - root - INFO - step: 56715 loss: 16.3078 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8124 +[titan] 2025-07-10 02:04:11,604 - root - INFO - lr: 1.3855e-04 gnorm: 0.84 [12:29:11< 9:31:46] +[titan] 2025-07-10 02:04:15,516 - root - INFO - step: 56720 loss: 16.5509 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 14.0042 +[titan] 2025-07-10 02:04:15,516 - root - INFO - lr: 1.3852e-04 gnorm: 0.81 [12:29:14< 9:31:42] +[titan] 2025-07-10 02:04:19,433 - root - INFO - step: 56725 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8522 +[titan] 2025-07-10 02:04:19,433 - root - INFO - lr: 1.3850e-04 gnorm: 0.86 [12:29:18< 9:31:38] +[titan] 2025-07-10 02:04:23,357 - root - INFO - step: 56730 loss: 16.4002 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.8803 +[titan] 2025-07-10 02:04:23,358 - root - INFO - lr: 1.3848e-04 gnorm: 0.79 [12:29:22< 9:31:34] +[titan] 2025-07-10 02:04:27,298 - root - INFO - step: 56735 loss: 16.3858 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-10 02:04:27,298 - root - INFO - lr: 1.3846e-04 gnorm: 0.80 [12:29:26< 9:31:30] +[titan] 2025-07-10 02:04:31,236 - root - INFO - step: 56740 loss: 16.4487 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9207 +[titan] 2025-07-10 02:04:31,236 - root - INFO - lr: 1.3844e-04 gnorm: 0.81 [12:29:30< 9:31:26] +[titan] 2025-07-10 02:04:35,194 - root - INFO - step: 56745 loss: 16.5268 memory: 44.58GiB(31.99%) tps: 82,785 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9839 +[titan] 2025-07-10 02:04:35,195 - root - INFO - lr: 1.3842e-04 gnorm: 0.83 [12:29:34< 9:31:22] +[titan] 2025-07-10 02:04:38,331 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:04:39,142 - root - INFO - step: 56750 loss: 16.4932 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.9702 +[titan] 2025-07-10 02:04:39,142 - root - INFO - lr: 1.3840e-04 gnorm: 0.82 [12:29:38< 9:31:18] +[titan] 2025-07-10 02:04:43,084 - root - INFO - step: 56755 loss: 16.4471 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9275 +[titan] 2025-07-10 02:04:43,085 - root - INFO - lr: 1.3838e-04 gnorm: 0.84 [12:29:42< 9:31:14] +[titan] 2025-07-10 02:04:46,997 - root - INFO - step: 56760 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 02:04:46,997 - root - INFO - lr: 1.3836e-04 gnorm: 0.84 [12:29:46< 9:31:10] +[titan] 2025-07-10 02:04:50,906 - root - INFO - step: 56765 loss: 16.6940 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.1288 +[titan] 2025-07-10 02:04:50,906 - root - INFO - lr: 1.3834e-04 gnorm: 0.86 [12:29:50< 9:31:06] +[titan] 2025-07-10 02:04:54,814 - root - INFO - step: 56770 loss: 16.7163 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5830 global_avg_mtp_loss: 14.1333 +[titan] 2025-07-10 02:04:54,814 - root - INFO - lr: 1.3831e-04 gnorm: 0.90 [12:29:54< 9:31:02] +[titan] 2025-07-10 02:04:58,726 - root - INFO - step: 56775 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.9196 +[titan] 2025-07-10 02:04:58,726 - root - INFO - lr: 1.3829e-04 gnorm: 0.82 [12:29:58< 9:30:58] +[titan] 2025-07-10 02:05:02,646 - root - INFO - step: 56780 loss: 16.3519 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8485 +[titan] 2025-07-10 02:05:02,646 - root - INFO - lr: 1.3827e-04 gnorm: 0.81 [12:30:02< 9:30:54] +[titan] 2025-07-10 02:05:06,584 - root - INFO - step: 56785 loss: 16.4844 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9404 +[titan] 2025-07-10 02:05:06,585 - root - INFO - lr: 1.3825e-04 gnorm: 0.84 [12:30:06< 9:30:50] +[titan] 2025-07-10 02:05:10,534 - root - INFO - step: 56790 loss: 16.5706 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5469 global_avg_mtp_loss: 14.0238 +[titan] 2025-07-10 02:05:10,534 - root - INFO - lr: 1.3823e-04 gnorm: 0.85 [12:30:09< 9:30:46] +[titan] 2025-07-10 02:05:14,428 - root - INFO - step: 56795 loss: 16.3957 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 02:05:14,428 - root - INFO - lr: 1.3821e-04 gnorm: 0.94 [12:30:13< 9:30:42] +[titan] 2025-07-10 02:05:17,549 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:05:18,356 - root - INFO - step: 56800 loss: 16.5385 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9957 +[titan] 2025-07-10 02:05:18,356 - root - INFO - lr: 1.3819e-04 gnorm: 0.91 [12:30:17< 9:30:38] +[titan] 2025-07-10 02:05:22,257 - root - INFO - step: 56805 loss: 16.7430 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1715 +[titan] 2025-07-10 02:05:22,257 - root - INFO - lr: 1.3817e-04 gnorm: 0.88 [12:30:21< 9:30:34] +[titan] 2025-07-10 02:05:26,165 - root - INFO - step: 56810 loss: 16.5446 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 13.9944 +[titan] 2025-07-10 02:05:26,165 - root - INFO - lr: 1.3815e-04 gnorm: 0.90 [12:30:25< 9:30:30] +[titan] 2025-07-10 02:05:30,108 - root - INFO - step: 56815 loss: 16.5348 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9906 +[titan] 2025-07-10 02:05:30,108 - root - INFO - lr: 1.3813e-04 gnorm: 0.88 [12:30:29< 9:30:26] +[titan] 2025-07-10 02:05:34,056 - root - INFO - step: 56820 loss: 16.3386 memory: 44.58GiB(31.99%) tps: 83,001 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8343 +[titan] 2025-07-10 02:05:34,056 - root - INFO - lr: 1.3810e-04 gnorm: 0.84 [12:30:33< 9:30:22] +[titan] 2025-07-10 02:05:37,976 - root - INFO - step: 56825 loss: 16.6928 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1119 +[titan] 2025-07-10 02:05:37,976 - root - INFO - lr: 1.3808e-04 gnorm: 0.83 [12:30:37< 9:30:18] +[titan] 2025-07-10 02:05:41,995 - root - INFO - step: 56830 loss: 16.3292 memory: 44.58GiB(31.99%) tps: 81,545 tflops: 281.43 mfu: 28.46% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 02:05:41,995 - root - INFO - lr: 1.3806e-04 gnorm: 0.85 [12:30:41< 9:30:14] +[titan] 2025-07-10 02:05:43,710 - root - INFO - Dumping profiler traces at step 56832 +[titan] 2025-07-10 02:05:43,742 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:05:46,117 - root - INFO - step: 56835 loss: 16.4681 memory: 44.58GiB(31.99%) tps: 79,493 tflops: 274.34 mfu: 27.74% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9321 +[titan] 2025-07-10 02:05:46,117 - root - INFO - lr: 1.3804e-04 gnorm: 0.91 [12:30:45< 9:30:11] +[titan] 2025-07-10 02:05:50,024 - root - INFO - step: 56840 loss: 16.3685 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5159 global_avg_mtp_loss: 13.8527 +[titan] 2025-07-10 02:05:50,024 - root - INFO - lr: 1.3802e-04 gnorm: 0.86 [12:30:49< 9:30:07] +[titan] 2025-07-10 02:05:53,934 - root - INFO - step: 56845 loss: 16.5501 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 14.0149 +[titan] 2025-07-10 02:05:53,934 - root - INFO - lr: 1.3800e-04 gnorm: 0.86 [12:30:53< 9:30:03] +[titan] 2025-07-10 02:05:57,042 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:05:57,849 - root - INFO - step: 56850 loss: 16.7351 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1627 +[titan] 2025-07-10 02:05:57,849 - root - INFO - lr: 1.3798e-04 gnorm: 0.83 [12:30:57< 9:29:59] +[titan] 2025-07-10 02:06:01,764 - root - INFO - step: 56855 loss: 16.5052 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9803 +[titan] 2025-07-10 02:06:01,765 - root - INFO - lr: 1.3796e-04 gnorm: 0.82 [12:31:01< 9:29:55] +[titan] 2025-07-10 02:06:05,687 - root - INFO - step: 56860 loss: 16.2587 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-10 02:06:05,687 - root - INFO - lr: 1.3794e-04 gnorm: 0.88 [12:31:05< 9:29:51] +[titan] 2025-07-10 02:06:09,604 - root - INFO - step: 56865 loss: 16.4772 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-10 02:06:09,604 - root - INFO - lr: 1.3792e-04 gnorm: 0.85 [12:31:09< 9:29:47] +[titan] 2025-07-10 02:06:13,526 - root - INFO - step: 56870 loss: 16.3707 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8487 +[titan] 2025-07-10 02:06:13,526 - root - INFO - lr: 1.3789e-04 gnorm: 0.86 [12:31:12< 9:29:43] +[titan] 2025-07-10 02:06:17,413 - root - INFO - step: 56875 loss: 16.3605 memory: 44.58GiB(31.99%) tps: 84,309 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8519 +[titan] 2025-07-10 02:06:17,413 - root - INFO - lr: 1.3787e-04 gnorm: 0.83 [12:31:16< 9:29:39] +[titan] 2025-07-10 02:06:21,329 - root - INFO - step: 56880 loss: 16.5236 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 13.9873 +[titan] 2025-07-10 02:06:21,329 - root - INFO - lr: 1.3785e-04 gnorm: 0.83 [12:31:20< 9:29:35] +[titan] 2025-07-10 02:06:25,248 - root - INFO - step: 56885 loss: 16.3503 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8473 +[titan] 2025-07-10 02:06:25,249 - root - INFO - lr: 1.3783e-04 gnorm: 0.87 [12:31:24< 9:29:31] +[titan] 2025-07-10 02:06:29,165 - root - INFO - step: 56890 loss: 16.4377 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.9167 +[titan] 2025-07-10 02:06:29,165 - root - INFO - lr: 1.3781e-04 gnorm: 0.83 [12:31:28< 9:29:27] +[titan] 2025-07-10 02:06:33,081 - root - INFO - step: 56895 loss: 16.5151 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9768 +[titan] 2025-07-10 02:06:33,082 - root - INFO - lr: 1.3779e-04 gnorm: 0.84 [12:31:32< 9:29:23] +[titan] 2025-07-10 02:06:36,216 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:06:37,004 - root - INFO - step: 56900 loss: 16.7074 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1380 +[titan] 2025-07-10 02:06:37,004 - root - INFO - lr: 1.3777e-04 gnorm: 0.86 [12:31:36< 9:29:19] +[titan] 2025-07-10 02:06:40,919 - root - INFO - step: 56905 loss: 16.3683 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.8496 +[titan] 2025-07-10 02:06:40,919 - root - INFO - lr: 1.3775e-04 gnorm: 0.85 [12:31:40< 9:29:15] +[titan] 2025-07-10 02:06:44,848 - root - INFO - step: 56910 loss: 16.6559 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5871 global_avg_mtp_loss: 14.0688 +[titan] 2025-07-10 02:06:44,848 - root - INFO - lr: 1.3773e-04 gnorm: 0.90 [12:31:44< 9:29:11] +[titan] 2025-07-10 02:06:48,749 - root - INFO - step: 56915 loss: 16.2970 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.7972 +[titan] 2025-07-10 02:06:48,750 - root - INFO - lr: 1.3771e-04 gnorm: 0.82 [12:31:48< 9:29:07] +[titan] 2025-07-10 02:06:52,652 - root - INFO - step: 56920 loss: 16.2798 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.7753 +[titan] 2025-07-10 02:06:52,652 - root - INFO - lr: 1.3768e-04 gnorm: 0.87 [12:31:52< 9:29:03] +[titan] 2025-07-10 02:06:56,572 - root - INFO - step: 56925 loss: 16.4016 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8822 +[titan] 2025-07-10 02:06:56,572 - root - INFO - lr: 1.3766e-04 gnorm: 0.86 [12:31:55< 9:28:59] +[titan] 2025-07-10 02:07:00,497 - root - INFO - step: 56930 loss: 16.5888 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 14.0462 +[titan] 2025-07-10 02:07:00,497 - root - INFO - lr: 1.3764e-04 gnorm: 0.84 [12:31:59< 9:28:55] +[titan] 2025-07-10 02:07:04,421 - root - INFO - step: 56935 loss: 16.6863 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5783 global_avg_mtp_loss: 14.1080 +[titan] 2025-07-10 02:07:04,421 - root - INFO - lr: 1.3762e-04 gnorm: 0.84 [12:32:03< 9:28:51] +[titan] 2025-07-10 02:07:08,365 - root - INFO - step: 56940 loss: 16.3046 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8096 +[titan] 2025-07-10 02:07:08,365 - root - INFO - lr: 1.3760e-04 gnorm: 0.90 [12:32:07< 9:28:47] +[titan] 2025-07-10 02:07:12,287 - root - INFO - step: 56945 loss: 16.4296 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.9188 +[titan] 2025-07-10 02:07:12,287 - root - INFO - lr: 1.3758e-04 gnorm: 0.84 [12:32:11< 9:28:43] +[titan] 2025-07-10 02:07:15,432 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:07:16,223 - root - INFO - step: 56950 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 02:07:16,223 - root - INFO - lr: 1.3756e-04 gnorm: 0.82 [12:32:15< 9:28:39] +[titan] 2025-07-10 02:07:20,128 - root - INFO - step: 56955 loss: 16.1716 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.6971 +[titan] 2025-07-10 02:07:20,129 - root - INFO - lr: 1.3754e-04 gnorm: 0.81 [12:32:19< 9:28:35] +[titan] 2025-07-10 02:07:24,020 - root - INFO - step: 56960 loss: 16.6378 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.0633 +[titan] 2025-07-10 02:07:24,020 - root - INFO - lr: 1.3752e-04 gnorm: 0.82 [12:32:23< 9:28:31] +[titan] 2025-07-10 02:07:27,930 - root - INFO - step: 56965 loss: 16.3740 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8549 +[titan] 2025-07-10 02:07:27,930 - root - INFO - lr: 1.3750e-04 gnorm: 0.90 [12:32:27< 9:28:27] +[titan] 2025-07-10 02:07:31,844 - root - INFO - step: 56970 loss: 16.6045 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5545 global_avg_mtp_loss: 14.0501 +[titan] 2025-07-10 02:07:31,845 - root - INFO - lr: 1.3748e-04 gnorm: 0.82 [12:32:31< 9:28:23] +[titan] 2025-07-10 02:07:35,753 - root - INFO - step: 56975 loss: 16.3858 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8701 +[titan] 2025-07-10 02:07:35,753 - root - INFO - lr: 1.3745e-04 gnorm: 0.87 [12:32:35< 9:28:19] +[titan] 2025-07-10 02:07:39,647 - root - INFO - step: 56980 loss: 16.1767 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.6940 +[titan] 2025-07-10 02:07:39,647 - root - INFO - lr: 1.3743e-04 gnorm: 0.81 [12:32:39< 9:28:15] +[titan] 2025-07-10 02:07:43,559 - root - INFO - step: 56985 loss: 16.6913 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5723 global_avg_mtp_loss: 14.1190 +[titan] 2025-07-10 02:07:43,559 - root - INFO - lr: 1.3741e-04 gnorm: 0.86 [12:32:42< 9:28:11] +[titan] 2025-07-10 02:07:47,482 - root - INFO - step: 56990 loss: 16.1682 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6940 +[titan] 2025-07-10 02:07:47,482 - root - INFO - lr: 1.3739e-04 gnorm: 0.85 [12:32:46< 9:28:07] +[titan] 2025-07-10 02:07:51,373 - root - INFO - step: 56995 loss: 16.2910 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.7883 +[titan] 2025-07-10 02:07:51,374 - root - INFO - lr: 1.3737e-04 gnorm: 0.84 [12:32:50< 9:28:03] +[titan] 2025-07-10 02:07:54,476 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:07:55,262 - root - INFO - step: 57000 loss: 16.5945 memory: 44.58GiB(31.99%) tps: 84,278 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 14.0527 +[titan] 2025-07-10 02:07:55,262 - root - INFO - lr: 1.3735e-04 gnorm: 0.86 [12:32:54< 9:27:59] +[titan] 2025-07-10 02:07:59,165 - root - INFO - step: 57005 loss: 16.4600 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9274 +[titan] 2025-07-10 02:07:59,165 - root - INFO - lr: 1.3733e-04 gnorm: 0.82 [12:32:58< 9:27:55] +[titan] 2025-07-10 02:08:03,073 - root - INFO - step: 57010 loss: 16.3593 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.8310 +[titan] 2025-07-10 02:08:03,073 - root - INFO - lr: 1.3731e-04 gnorm: 0.83 [12:33:02< 9:27:51] +[titan] 2025-07-10 02:08:06,968 - root - INFO - step: 57015 loss: 16.3053 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8064 +[titan] 2025-07-10 02:08:06,968 - root - INFO - lr: 1.3729e-04 gnorm: 0.84 [12:33:06< 9:27:47] +[titan] 2025-07-10 02:08:10,915 - root - INFO - step: 57020 loss: 16.2153 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7361 +[titan] 2025-07-10 02:08:10,915 - root - INFO - lr: 1.3727e-04 gnorm: 0.87 [12:33:10< 9:27:43] +[titan] 2025-07-10 02:08:14,868 - root - INFO - step: 57025 loss: 16.4378 memory: 44.58GiB(31.99%) tps: 82,887 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.5253 global_avg_mtp_loss: 13.9125 +[titan] 2025-07-10 02:08:14,869 - root - INFO - lr: 1.3724e-04 gnorm: 0.89 [12:33:14< 9:27:39] +[titan] 2025-07-10 02:08:18,767 - root - INFO - step: 57030 loss: 16.2811 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.7751 +[titan] 2025-07-10 02:08:18,768 - root - INFO - lr: 1.3722e-04 gnorm: 0.87 [12:33:18< 9:27:35] +[titan] 2025-07-10 02:08:22,661 - root - INFO - step: 57035 loss: 16.3996 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8905 +[titan] 2025-07-10 02:08:22,662 - root - INFO - lr: 1.3720e-04 gnorm: 0.90 [12:33:22< 9:27:31] +[titan] 2025-07-10 02:08:26,584 - root - INFO - step: 57040 loss: 16.3375 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8382 +[titan] 2025-07-10 02:08:26,584 - root - INFO - lr: 1.3718e-04 gnorm: 0.85 [12:33:25< 9:27:27] +[titan] 2025-07-10 02:08:30,521 - root - INFO - step: 57045 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 13.9300 +[titan] 2025-07-10 02:08:30,521 - root - INFO - lr: 1.3716e-04 gnorm: 0.87 [12:33:29< 9:27:23] +[titan] 2025-07-10 02:08:33,634 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:08:34,420 - root - INFO - step: 57050 loss: 16.3779 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8619 +[titan] 2025-07-10 02:08:34,420 - root - INFO - lr: 1.3714e-04 gnorm: 0.86 [12:33:33< 9:27:19] +[titan] 2025-07-10 02:08:38,322 - root - INFO - step: 57055 loss: 16.5271 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-10 02:08:38,322 - root - INFO - lr: 1.3712e-04 gnorm: 0.99 [12:33:37< 9:27:15] +[titan] 2025-07-10 02:08:42,234 - root - INFO - step: 57060 loss: 16.3554 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8450 +[titan] 2025-07-10 02:08:42,235 - root - INFO - lr: 1.3710e-04 gnorm: 0.83 [12:33:41< 9:27:11] +[titan] 2025-07-10 02:08:46,125 - root - INFO - step: 57065 loss: 16.3275 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.8138 +[titan] 2025-07-10 02:08:46,125 - root - INFO - lr: 1.3708e-04 gnorm: 0.84 [12:33:45< 9:27:07] +[titan] 2025-07-10 02:08:50,033 - root - INFO - step: 57070 loss: 16.4779 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9465 +[titan] 2025-07-10 02:08:50,033 - root - INFO - lr: 1.3706e-04 gnorm: 0.81 [12:33:49< 9:27:03] +[titan] 2025-07-10 02:08:53,928 - root - INFO - step: 57075 loss: 16.6086 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5552 global_avg_mtp_loss: 14.0534 +[titan] 2025-07-10 02:08:53,928 - root - INFO - lr: 1.3703e-04 gnorm: 0.89 [12:33:53< 9:26:59] +[titan] 2025-07-10 02:08:57,850 - root - INFO - step: 57080 loss: 16.5230 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9805 +[titan] 2025-07-10 02:08:57,850 - root - INFO - lr: 1.3701e-04 gnorm: 0.85 [12:33:57< 9:26:55] +[titan] 2025-07-10 02:09:01,759 - root - INFO - step: 57085 loss: 16.5849 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 14.0345 +[titan] 2025-07-10 02:09:01,759 - root - INFO - lr: 1.3699e-04 gnorm: 0.85 [12:34:01< 9:26:51] +[titan] 2025-07-10 02:09:05,690 - root - INFO - step: 57090 loss: 16.5919 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 14.0405 +[titan] 2025-07-10 02:09:05,690 - root - INFO - lr: 1.3697e-04 gnorm: 0.82 [12:34:05< 9:26:47] +[titan] 2025-07-10 02:09:09,594 - root - INFO - step: 57095 loss: 16.3362 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8261 +[titan] 2025-07-10 02:09:09,594 - root - INFO - lr: 1.3695e-04 gnorm: 0.87 [12:34:08< 9:26:43] +[titan] 2025-07-10 02:09:12,710 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:09:13,499 - root - INFO - step: 57100 loss: 16.6083 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0506 +[titan] 2025-07-10 02:09:13,499 - root - INFO - lr: 1.3693e-04 gnorm: 0.87 [12:34:12< 9:26:39] +[titan] 2025-07-10 02:09:17,423 - root - INFO - step: 57105 loss: 16.3604 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.8416 +[titan] 2025-07-10 02:09:17,423 - root - INFO - lr: 1.3691e-04 gnorm: 0.88 [12:34:16< 9:26:35] +[titan] 2025-07-10 02:09:21,329 - root - INFO - step: 57110 loss: 16.0582 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.5988 +[titan] 2025-07-10 02:09:21,329 - root - INFO - lr: 1.3689e-04 gnorm: 0.79 [12:34:20< 9:26:31] +[titan] 2025-07-10 02:09:25,261 - root - INFO - step: 57115 loss: 16.6339 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0711 +[titan] 2025-07-10 02:09:25,261 - root - INFO - lr: 1.3687e-04 gnorm: 0.91 [12:34:24< 9:26:27] +[titan] 2025-07-10 02:09:29,193 - root - INFO - step: 57120 loss: 16.4481 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.9199 +[titan] 2025-07-10 02:09:29,194 - root - INFO - lr: 1.3685e-04 gnorm: 0.84 [12:34:28< 9:26:23] +[titan] 2025-07-10 02:09:33,146 - root - INFO - step: 57125 loss: 16.3311 memory: 44.58GiB(31.99%) tps: 82,914 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8226 +[titan] 2025-07-10 02:09:33,146 - root - INFO - lr: 1.3683e-04 gnorm: 0.93 [12:34:32< 9:26:19] +[titan] 2025-07-10 02:09:37,077 - root - INFO - step: 57130 loss: 16.7092 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.1349 +[titan] 2025-07-10 02:09:37,078 - root - INFO - lr: 1.3680e-04 gnorm: 0.83 [12:34:36< 9:26:15] +[titan] 2025-07-10 02:09:40,998 - root - INFO - step: 57135 loss: 16.6234 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 14.0796 +[titan] 2025-07-10 02:09:40,998 - root - INFO - lr: 1.3678e-04 gnorm: 0.83 [12:34:40< 9:26:11] +[titan] 2025-07-10 02:09:44,912 - root - INFO - step: 57140 loss: 16.5350 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-10 02:09:44,912 - root - INFO - lr: 1.3676e-04 gnorm: 0.87 [12:34:44< 9:26:07] +[titan] 2025-07-10 02:09:48,848 - root - INFO - step: 57145 loss: 16.2818 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7891 +[titan] 2025-07-10 02:09:48,848 - root - INFO - lr: 1.3674e-04 gnorm: 0.90 [12:34:48< 9:26:03] +[titan] 2025-07-10 02:09:51,965 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:09:52,760 - root - INFO - step: 57150 loss: 16.4123 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8865 +[titan] 2025-07-10 02:09:52,761 - root - INFO - lr: 1.3672e-04 gnorm: 1.08 [12:34:52< 9:25:59] +[titan] 2025-07-10 02:09:56,682 - root - INFO - step: 57155 loss: 16.3123 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 02:09:56,683 - root - INFO - lr: 1.3670e-04 gnorm: 0.87 [12:34:56< 9:25:55] +[titan] 2025-07-10 02:10:00,592 - root - INFO - step: 57160 loss: 16.4461 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 13.9150 +[titan] 2025-07-10 02:10:00,592 - root - INFO - lr: 1.3668e-04 gnorm: 0.88 [12:34:59< 9:25:51] +[titan] 2025-07-10 02:10:04,493 - root - INFO - step: 57165 loss: 16.4489 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9098 +[titan] 2025-07-10 02:10:04,493 - root - INFO - lr: 1.3666e-04 gnorm: 0.84 [12:35:03< 9:25:47] +[titan] 2025-07-10 02:10:08,410 - root - INFO - step: 57170 loss: 16.5294 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5533 global_avg_mtp_loss: 13.9761 +[titan] 2025-07-10 02:10:08,410 - root - INFO - lr: 1.3664e-04 gnorm: 0.89 [12:35:07< 9:25:43] +[titan] 2025-07-10 02:10:12,347 - root - INFO - step: 57175 loss: 16.3062 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 02:10:12,347 - root - INFO - lr: 1.3662e-04 gnorm: 0.82 [12:35:11< 9:25:39] +[titan] 2025-07-10 02:10:16,304 - root - INFO - step: 57180 loss: 16.4583 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9389 +[titan] 2025-07-10 02:10:16,305 - root - INFO - lr: 1.3659e-04 gnorm: 0.81 [12:35:15< 9:25:35] +[titan] 2025-07-10 02:10:20,211 - root - INFO - step: 57185 loss: 16.5586 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5604 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 02:10:20,212 - root - INFO - lr: 1.3657e-04 gnorm: 0.79 [12:35:19< 9:25:31] +[titan] 2025-07-10 02:10:24,121 - root - INFO - step: 57190 loss: 16.5512 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 14.0091 +[titan] 2025-07-10 02:10:24,121 - root - INFO - lr: 1.3655e-04 gnorm: 0.80 [12:35:23< 9:25:27] +[titan] 2025-07-10 02:10:28,045 - root - INFO - step: 57195 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 14.0098 +[titan] 2025-07-10 02:10:28,045 - root - INFO - lr: 1.3653e-04 gnorm: 0.78 [12:35:27< 9:25:23] +[titan] 2025-07-10 02:10:31,164 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:10:31,960 - root - INFO - step: 57200 loss: 16.6143 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.0462 +[titan] 2025-07-10 02:10:31,960 - root - INFO - lr: 1.3651e-04 gnorm: 0.85 [12:35:31< 9:25:19] +[titan] 2025-07-10 02:10:35,855 - root - INFO - step: 57205 loss: 16.4163 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.8941 +[titan] 2025-07-10 02:10:35,856 - root - INFO - lr: 1.3649e-04 gnorm: 0.85 [12:35:35< 9:25:15] +[titan] 2025-07-10 02:10:39,795 - root - INFO - step: 57210 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.7972 +[titan] 2025-07-10 02:10:39,795 - root - INFO - lr: 1.3647e-04 gnorm: 0.85 [12:35:39< 9:25:11] +[titan] 2025-07-10 02:10:43,715 - root - INFO - step: 57215 loss: 16.5731 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0166 +[titan] 2025-07-10 02:10:43,716 - root - INFO - lr: 1.3645e-04 gnorm: 0.82 [12:35:43< 9:25:07] +[titan] 2025-07-10 02:10:47,635 - root - INFO - step: 57220 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9639 +[titan] 2025-07-10 02:10:47,635 - root - INFO - lr: 1.3643e-04 gnorm: 0.89 [12:35:47< 9:25:03] +[titan] 2025-07-10 02:10:51,584 - root - INFO - step: 57225 loss: 16.4197 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 13.8714 +[titan] 2025-07-10 02:10:51,584 - root - INFO - lr: 1.3641e-04 gnorm: 0.89 [12:35:50< 9:24:59] +[titan] 2025-07-10 02:10:55,508 - root - INFO - step: 57230 loss: 16.2492 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 02:10:55,508 - root - INFO - lr: 1.3639e-04 gnorm: 0.84 [12:35:54< 9:24:55] +[titan] 2025-07-10 02:10:59,427 - root - INFO - step: 57235 loss: 16.4267 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9065 +[titan] 2025-07-10 02:10:59,428 - root - INFO - lr: 1.3636e-04 gnorm: 0.84 [12:35:58< 9:24:51] +[titan] 2025-07-10 02:11:03,367 - root - INFO - step: 57240 loss: 16.4032 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8811 +[titan] 2025-07-10 02:11:03,367 - root - INFO - lr: 1.3634e-04 gnorm: 0.85 [12:36:02< 9:24:47] +[titan] 2025-07-10 02:11:07,278 - root - INFO - step: 57245 loss: 16.5781 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 14.0157 +[titan] 2025-07-10 02:11:07,279 - root - INFO - lr: 1.3632e-04 gnorm: 0.89 [12:36:06< 9:24:43] +[titan] 2025-07-10 02:11:10,417 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:11:11,203 - root - INFO - step: 57250 loss: 16.4167 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.8962 +[titan] 2025-07-10 02:11:11,204 - root - INFO - lr: 1.3630e-04 gnorm: 1.11 [12:36:10< 9:24:39] +[titan] 2025-07-10 02:11:15,112 - root - INFO - step: 57255 loss: 16.5311 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-10 02:11:15,112 - root - INFO - lr: 1.3628e-04 gnorm: 0.89 [12:36:14< 9:24:35] +[titan] 2025-07-10 02:11:19,038 - root - INFO - step: 57260 loss: 16.1683 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6941 +[titan] 2025-07-10 02:11:19,038 - root - INFO - lr: 1.3626e-04 gnorm: 0.88 [12:36:18< 9:24:31] +[titan] 2025-07-10 02:11:22,951 - root - INFO - step: 57265 loss: 16.5130 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9771 +[titan] 2025-07-10 02:11:22,951 - root - INFO - lr: 1.3624e-04 gnorm: 0.90 [12:36:22< 9:24:27] +[titan] 2025-07-10 02:11:26,900 - root - INFO - step: 57270 loss: 16.5168 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9713 +[titan] 2025-07-10 02:11:26,901 - root - INFO - lr: 1.3622e-04 gnorm: 0.83 [12:36:26< 9:24:23] +[titan] 2025-07-10 02:11:30,830 - root - INFO - step: 57275 loss: 16.3851 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.8865 +[titan] 2025-07-10 02:11:30,830 - root - INFO - lr: 1.3620e-04 gnorm: 0.83 [12:36:30< 9:24:19] +[titan] 2025-07-10 02:11:34,738 - root - INFO - step: 57280 loss: 16.3466 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 02:11:34,738 - root - INFO - lr: 1.3618e-04 gnorm: 0.83 [12:36:34< 9:24:15] +[titan] 2025-07-10 02:11:38,661 - root - INFO - step: 57285 loss: 16.1899 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7025 +[titan] 2025-07-10 02:11:38,661 - root - INFO - lr: 1.3616e-04 gnorm: 0.89 [12:36:38< 9:24:11] +[titan] 2025-07-10 02:11:42,597 - root - INFO - step: 57290 loss: 16.3372 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8309 +[titan] 2025-07-10 02:11:42,598 - root - INFO - lr: 1.3613e-04 gnorm: 0.80 [12:36:41< 9:24:07] +[titan] 2025-07-10 02:11:46,505 - root - INFO - step: 57295 loss: 16.6607 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.1001 +[titan] 2025-07-10 02:11:46,505 - root - INFO - lr: 1.3611e-04 gnorm: 0.90 [12:36:45< 9:24:03] +[titan] 2025-07-10 02:11:49,631 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:11:50,432 - root - INFO - step: 57300 loss: 16.4808 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 13.9387 +[titan] 2025-07-10 02:11:50,432 - root - INFO - lr: 1.3609e-04 gnorm: 0.81 [12:36:49< 9:23:59] +[titan] 2025-07-10 02:11:54,401 - root - INFO - step: 57305 loss: 16.2800 memory: 44.58GiB(31.99%) tps: 82,556 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7824 +[titan] 2025-07-10 02:11:54,401 - root - INFO - lr: 1.3607e-04 gnorm: 0.80 [12:36:53< 9:23:55] +[titan] 2025-07-10 02:11:58,333 - root - INFO - step: 57310 loss: 16.6595 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 14.0805 +[titan] 2025-07-10 02:11:58,333 - root - INFO - lr: 1.3605e-04 gnorm: 0.95 [12:36:57< 9:23:51] +[titan] 2025-07-10 02:12:02,248 - root - INFO - step: 57315 loss: 16.3749 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8539 +[titan] 2025-07-10 02:12:02,248 - root - INFO - lr: 1.3603e-04 gnorm: 0.85 [12:37:01< 9:23:47] +[titan] 2025-07-10 02:12:06,194 - root - INFO - step: 57320 loss: 16.5690 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 14.0252 +[titan] 2025-07-10 02:12:06,195 - root - INFO - lr: 1.3601e-04 gnorm: 0.90 [12:37:05< 9:23:43] +[titan] 2025-07-10 02:12:10,124 - root - INFO - step: 57325 loss: 16.3864 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8751 +[titan] 2025-07-10 02:12:10,125 - root - INFO - lr: 1.3599e-04 gnorm: 0.86 [12:37:09< 9:23:39] +[titan] 2025-07-10 02:12:14,034 - root - INFO - step: 57330 loss: 16.4528 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9265 +[titan] 2025-07-10 02:12:14,034 - root - INFO - lr: 1.3597e-04 gnorm: 0.87 [12:37:13< 9:23:35] +[titan] 2025-07-10 02:12:17,969 - root - INFO - step: 57335 loss: 16.0868 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.6404 +[titan] 2025-07-10 02:12:17,969 - root - INFO - lr: 1.3595e-04 gnorm: 0.86 [12:37:17< 9:23:31] +[titan] 2025-07-10 02:12:21,882 - root - INFO - step: 57340 loss: 16.5720 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5591 global_avg_mtp_loss: 14.0129 +[titan] 2025-07-10 02:12:21,882 - root - INFO - lr: 1.3593e-04 gnorm: 0.83 [12:37:21< 9:23:27] +[titan] 2025-07-10 02:12:25,248 - root - INFO - Dumping profiler traces at step 57344 +[titan] 2025-07-10 02:12:25,280 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:12:26,072 - root - INFO - step: 57345 loss: 16.2690 memory: 44.58GiB(31.99%) tps: 78,222 tflops: 269.96 mfu: 27.30% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7893 +[titan] 2025-07-10 02:12:26,072 - root - INFO - lr: 1.3590e-04 gnorm: 0.86 [12:37:25< 9:23:23] +[titan] 2025-07-10 02:12:29,208 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:12:29,996 - root - INFO - step: 57350 loss: 16.5246 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9779 +[titan] 2025-07-10 02:12:29,996 - root - INFO - lr: 1.3588e-04 gnorm: 0.84 [12:37:29< 9:23:19] +[titan] 2025-07-10 02:12:33,934 - root - INFO - step: 57355 loss: 16.7203 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5747 global_avg_mtp_loss: 14.1456 +[titan] 2025-07-10 02:12:33,934 - root - INFO - lr: 1.3586e-04 gnorm: 0.86 [12:37:33< 9:23:15] +[titan] 2025-07-10 02:12:37,860 - root - INFO - step: 57360 loss: 16.2018 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7251 +[titan] 2025-07-10 02:12:37,860 - root - INFO - lr: 1.3584e-04 gnorm: 0.82 [12:37:37< 9:23:11] +[titan] 2025-07-10 02:12:41,770 - root - INFO - step: 57365 loss: 16.4387 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 02:12:41,770 - root - INFO - lr: 1.3582e-04 gnorm: 0.87 [12:37:41< 9:23:07] +[titan] 2025-07-10 02:12:45,694 - root - INFO - step: 57370 loss: 16.4362 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5244 global_avg_mtp_loss: 13.9118 +[titan] 2025-07-10 02:12:45,694 - root - INFO - lr: 1.3580e-04 gnorm: 0.87 [12:37:45< 9:23:03] +[titan] 2025-07-10 02:12:49,618 - root - INFO - step: 57375 loss: 16.3032 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8020 +[titan] 2025-07-10 02:12:49,618 - root - INFO - lr: 1.3578e-04 gnorm: 0.93 [12:37:48< 9:22:59] +[titan] 2025-07-10 02:12:53,544 - root - INFO - step: 57380 loss: 16.3719 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.8496 +[titan] 2025-07-10 02:12:53,544 - root - INFO - lr: 1.3576e-04 gnorm: 0.86 [12:37:52< 9:22:55] +[titan] 2025-07-10 02:12:57,469 - root - INFO - step: 57385 loss: 16.4697 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9376 +[titan] 2025-07-10 02:12:57,469 - root - INFO - lr: 1.3574e-04 gnorm: 0.82 [12:37:56< 9:22:51] +[titan] 2025-07-10 02:13:01,436 - root - INFO - step: 57390 loss: 16.6396 memory: 44.58GiB(31.99%) tps: 82,604 tflops: 285.08 mfu: 28.82% global_avg_ntp_loss: 2.5548 global_avg_mtp_loss: 14.0848 +[titan] 2025-07-10 02:13:01,436 - root - INFO - lr: 1.3572e-04 gnorm: 0.85 [12:38:00< 9:22:47] +[titan] 2025-07-10 02:13:05,371 - root - INFO - step: 57395 loss: 16.3732 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8574 +[titan] 2025-07-10 02:13:05,372 - root - INFO - lr: 1.3570e-04 gnorm: 0.85 [12:38:04< 9:22:43] +[titan] 2025-07-10 02:13:08,485 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:13:09,274 - root - INFO - step: 57400 loss: 16.2047 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7245 +[titan] 2025-07-10 02:13:09,274 - root - INFO - lr: 1.3567e-04 gnorm: 0.90 [12:38:08< 9:22:39] +[titan] 2025-07-10 02:13:13,203 - root - INFO - step: 57405 loss: 16.6250 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.0496 +[titan] 2025-07-10 02:13:13,203 - root - INFO - lr: 1.3565e-04 gnorm: 0.84 [12:38:12< 9:22:35] +[titan] 2025-07-10 02:13:17,115 - root - INFO - step: 57410 loss: 16.3854 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.8666 +[titan] 2025-07-10 02:13:17,116 - root - INFO - lr: 1.3563e-04 gnorm: 0.84 [12:38:16< 9:22:31] +[titan] 2025-07-10 02:13:21,050 - root - INFO - step: 57415 loss: 16.3525 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8439 +[titan] 2025-07-10 02:13:21,051 - root - INFO - lr: 1.3561e-04 gnorm: 0.83 [12:38:20< 9:22:27] +[titan] 2025-07-10 02:13:24,976 - root - INFO - step: 57420 loss: 16.2534 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.7588 +[titan] 2025-07-10 02:13:24,976 - root - INFO - lr: 1.3559e-04 gnorm: 0.81 [12:38:24< 9:22:23] +[titan] 2025-07-10 02:13:28,884 - root - INFO - step: 57425 loss: 16.5709 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 14.0235 +[titan] 2025-07-10 02:13:28,884 - root - INFO - lr: 1.3557e-04 gnorm: 0.84 [12:38:28< 9:22:19] +[titan] 2025-07-10 02:13:32,796 - root - INFO - step: 57430 loss: 16.1790 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.6930 +[titan] 2025-07-10 02:13:32,796 - root - INFO - lr: 1.3555e-04 gnorm: 0.83 [12:38:32< 9:22:15] +[titan] 2025-07-10 02:13:36,699 - root - INFO - step: 57435 loss: 16.6104 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.0454 +[titan] 2025-07-10 02:13:36,699 - root - INFO - lr: 1.3553e-04 gnorm: 0.82 [12:38:36< 9:22:11] +[titan] 2025-07-10 02:13:40,681 - root - INFO - step: 57440 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 82,287 tflops: 283.99 mfu: 28.71% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.7712 +[titan] 2025-07-10 02:13:40,682 - root - INFO - lr: 1.3551e-04 gnorm: 0.81 [12:38:40< 9:22:07] +[titan] 2025-07-10 02:13:44,605 - root - INFO - step: 57445 loss: 16.6724 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5677 global_avg_mtp_loss: 14.1048 +[titan] 2025-07-10 02:13:44,605 - root - INFO - lr: 1.3549e-04 gnorm: 0.83 [12:38:43< 9:22:03] +[titan] 2025-07-10 02:13:47,740 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:13:48,530 - root - INFO - step: 57450 loss: 16.4631 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9295 +[titan] 2025-07-10 02:13:48,530 - root - INFO - lr: 1.3547e-04 gnorm: 0.85 [12:38:47< 9:21:59] +[titan] 2025-07-10 02:13:52,440 - root - INFO - step: 57455 loss: 16.5346 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 13.9723 +[titan] 2025-07-10 02:13:52,440 - root - INFO - lr: 1.3544e-04 gnorm: 0.91 [12:38:51< 9:21:55] +[titan] 2025-07-10 02:13:56,385 - root - INFO - step: 57460 loss: 16.6840 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5778 global_avg_mtp_loss: 14.1062 +[titan] 2025-07-10 02:13:56,385 - root - INFO - lr: 1.3542e-04 gnorm: 0.84 [12:38:55< 9:21:51] +[titan] 2025-07-10 02:14:00,309 - root - INFO - step: 57465 loss: 16.3119 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.8252 +[titan] 2025-07-10 02:14:00,309 - root - INFO - lr: 1.3540e-04 gnorm: 0.85 [12:38:59< 9:21:47] +[titan] 2025-07-10 02:14:04,230 - root - INFO - step: 57470 loss: 16.6026 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 14.0463 +[titan] 2025-07-10 02:14:04,231 - root - INFO - lr: 1.3538e-04 gnorm: 0.85 [12:39:03< 9:21:44] +[titan] 2025-07-10 02:14:08,143 - root - INFO - step: 57475 loss: 16.6968 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.1367 +[titan] 2025-07-10 02:14:08,144 - root - INFO - lr: 1.3536e-04 gnorm: 0.87 [12:39:07< 9:21:40] +[titan] 2025-07-10 02:14:12,059 - root - INFO - step: 57480 loss: 16.7611 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5892 global_avg_mtp_loss: 14.1720 +[titan] 2025-07-10 02:14:12,059 - root - INFO - lr: 1.3534e-04 gnorm: 0.87 [12:39:11< 9:21:36] +[titan] 2025-07-10 02:14:15,974 - root - INFO - step: 57485 loss: 16.5500 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 14.0071 +[titan] 2025-07-10 02:14:15,975 - root - INFO - lr: 1.3532e-04 gnorm: 0.81 [12:39:15< 9:21:32] +[titan] 2025-07-10 02:14:19,892 - root - INFO - step: 57490 loss: 16.3521 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8460 +[titan] 2025-07-10 02:14:19,892 - root - INFO - lr: 1.3530e-04 gnorm: 0.81 [12:39:19< 9:21:28] +[titan] 2025-07-10 02:14:23,798 - root - INFO - step: 57495 loss: 16.4674 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 02:14:23,798 - root - INFO - lr: 1.3528e-04 gnorm: 0.84 [12:39:23< 9:21:24] +[titan] 2025-07-10 02:14:26,925 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:14:27,739 - root - INFO - step: 57500 loss: 15.9098 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4174 global_avg_mtp_loss: 13.4923 +[titan] 2025-07-10 02:14:27,739 - root - INFO - lr: 1.3526e-04 gnorm: 0.85 [12:39:27< 9:21:20] +[titan] 2025-07-10 02:14:31,642 - root - INFO - step: 57505 loss: 16.4066 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.8756 +[titan] 2025-07-10 02:14:31,642 - root - INFO - lr: 1.3524e-04 gnorm: 0.89 [12:39:31< 9:21:16] +[titan] 2025-07-10 02:14:35,600 - root - INFO - step: 57510 loss: 16.1829 memory: 44.58GiB(31.99%) tps: 82,811 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7093 +[titan] 2025-07-10 02:14:35,600 - root - INFO - lr: 1.3521e-04 gnorm: 0.83 [12:39:34< 9:21:12] +[titan] 2025-07-10 02:14:39,505 - root - INFO - step: 57515 loss: 16.6340 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0672 +[titan] 2025-07-10 02:14:39,506 - root - INFO - lr: 1.3519e-04 gnorm: 0.85 [12:39:38< 9:21:08] +[titan] 2025-07-10 02:14:43,469 - root - INFO - step: 57520 loss: 16.3302 memory: 44.58GiB(31.99%) tps: 82,681 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8222 +[titan] 2025-07-10 02:14:43,469 - root - INFO - lr: 1.3517e-04 gnorm: 0.86 [12:39:42< 9:21:04] +[titan] 2025-07-10 02:14:47,372 - root - INFO - step: 57525 loss: 16.5803 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0309 +[titan] 2025-07-10 02:14:47,372 - root - INFO - lr: 1.3515e-04 gnorm: 0.87 [12:39:46< 9:21:00] +[titan] 2025-07-10 02:14:51,275 - root - INFO - step: 57530 loss: 16.7417 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5744 global_avg_mtp_loss: 14.1672 +[titan] 2025-07-10 02:14:51,275 - root - INFO - lr: 1.3513e-04 gnorm: 0.89 [12:39:50< 9:20:56] +[titan] 2025-07-10 02:14:55,193 - root - INFO - step: 57535 loss: 16.1735 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7012 +[titan] 2025-07-10 02:14:55,193 - root - INFO - lr: 1.3511e-04 gnorm: 0.83 [12:39:54< 9:20:52] +[titan] 2025-07-10 02:14:59,113 - root - INFO - step: 57540 loss: 16.5753 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5554 global_avg_mtp_loss: 14.0199 +[titan] 2025-07-10 02:14:59,114 - root - INFO - lr: 1.3509e-04 gnorm: 0.82 [12:39:58< 9:20:48] +[titan] 2025-07-10 02:15:03,039 - root - INFO - step: 57545 loss: 16.5321 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 14.0096 +[titan] 2025-07-10 02:15:03,040 - root - INFO - lr: 1.3507e-04 gnorm: 0.88 [12:40:02< 9:20:44] +[titan] 2025-07-10 02:15:06,177 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:15:06,965 - root - INFO - step: 57550 loss: 16.7697 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5910 global_avg_mtp_loss: 14.1787 +[titan] 2025-07-10 02:15:06,966 - root - INFO - lr: 1.3505e-04 gnorm: 0.84 [12:40:06< 9:20:40] +[titan] 2025-07-10 02:15:10,872 - root - INFO - step: 57555 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.8056 +[titan] 2025-07-10 02:15:10,872 - root - INFO - lr: 1.3503e-04 gnorm: 0.87 [12:40:10< 9:20:36] +[titan] 2025-07-10 02:15:14,777 - root - INFO - step: 57560 loss: 16.0870 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.6076 +[titan] 2025-07-10 02:15:14,777 - root - INFO - lr: 1.3501e-04 gnorm: 0.89 [12:40:14< 9:20:32] +[titan] 2025-07-10 02:15:18,695 - root - INFO - step: 57565 loss: 16.3240 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8158 +[titan] 2025-07-10 02:15:18,695 - root - INFO - lr: 1.3498e-04 gnorm: 0.83 [12:40:18< 9:20:28] +[titan] 2025-07-10 02:15:22,606 - root - INFO - step: 57570 loss: 16.3489 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8390 +[titan] 2025-07-10 02:15:22,606 - root - INFO - lr: 1.3496e-04 gnorm: 0.88 [12:40:21< 9:20:24] +[titan] 2025-07-10 02:15:26,544 - root - INFO - step: 57575 loss: 16.7199 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.1404 +[titan] 2025-07-10 02:15:26,544 - root - INFO - lr: 1.3494e-04 gnorm: 0.89 [12:40:25< 9:20:20] +[titan] 2025-07-10 02:15:30,466 - root - INFO - step: 57580 loss: 16.2933 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7998 +[titan] 2025-07-10 02:15:30,466 - root - INFO - lr: 1.3492e-04 gnorm: 0.87 [12:40:29< 9:20:16] +[titan] 2025-07-10 02:15:34,391 - root - INFO - step: 57585 loss: 16.4577 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 02:15:34,391 - root - INFO - lr: 1.3490e-04 gnorm: 1.62 [12:40:33< 9:20:12] +[titan] 2025-07-10 02:15:38,307 - root - INFO - step: 57590 loss: 16.4557 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9230 +[titan] 2025-07-10 02:15:38,307 - root - INFO - lr: 1.3488e-04 gnorm: 0.92 [12:40:37< 9:20:08] +[titan] 2025-07-10 02:15:42,272 - root - INFO - step: 57595 loss: 16.6943 memory: 44.58GiB(31.99%) tps: 82,643 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.1119 +[titan] 2025-07-10 02:15:42,273 - root - INFO - lr: 1.3486e-04 gnorm: 0.83 [12:40:41< 9:20:04] +[titan] 2025-07-10 02:15:45,382 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:15:46,168 - root - INFO - step: 57600 loss: 16.3460 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8517 +[titan] 2025-07-10 02:15:46,168 - root - INFO - lr: 1.3484e-04 gnorm: 0.85 [12:40:45< 9:20:00] +[titan] 2025-07-10 02:15:50,080 - root - INFO - step: 57605 loss: 16.5391 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 02:15:50,081 - root - INFO - lr: 1.3482e-04 gnorm: 0.84 [12:40:49< 9:19:56] +[titan] 2025-07-10 02:15:54,002 - root - INFO - step: 57610 loss: 16.2801 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.7851 +[titan] 2025-07-10 02:15:54,002 - root - INFO - lr: 1.3480e-04 gnorm: 0.88 [12:40:53< 9:19:52] +[titan] 2025-07-10 02:15:57,924 - root - INFO - step: 57615 loss: 16.4700 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9497 +[titan] 2025-07-10 02:15:57,924 - root - INFO - lr: 1.3478e-04 gnorm: 0.83 [12:40:57< 9:19:48] +[titan] 2025-07-10 02:16:01,829 - root - INFO - step: 57620 loss: 16.2167 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 02:16:01,829 - root - INFO - lr: 1.3476e-04 gnorm: 0.93 [12:41:01< 9:19:44] +[titan] 2025-07-10 02:16:05,794 - root - INFO - step: 57625 loss: 16.3622 memory: 44.58GiB(31.99%) tps: 82,650 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8471 +[titan] 2025-07-10 02:16:05,794 - root - INFO - lr: 1.3473e-04 gnorm: 0.89 [12:41:05< 9:19:40] +[titan] 2025-07-10 02:16:09,728 - root - INFO - step: 57630 loss: 16.3970 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8873 +[titan] 2025-07-10 02:16:09,729 - root - INFO - lr: 1.3471e-04 gnorm: 0.89 [12:41:09< 9:19:36] +[titan] 2025-07-10 02:16:13,635 - root - INFO - step: 57635 loss: 16.3621 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.8498 +[titan] 2025-07-10 02:16:13,636 - root - INFO - lr: 1.3469e-04 gnorm: 0.87 [12:41:12< 9:19:32] +[titan] 2025-07-10 02:16:17,549 - root - INFO - step: 57640 loss: 16.4979 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9676 +[titan] 2025-07-10 02:16:17,550 - root - INFO - lr: 1.3467e-04 gnorm: 0.79 [12:41:16< 9:19:28] +[titan] 2025-07-10 02:16:21,458 - root - INFO - step: 57645 loss: 16.6503 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0894 +[titan] 2025-07-10 02:16:21,459 - root - INFO - lr: 1.3465e-04 gnorm: 0.84 [12:41:20< 9:19:24] +[titan] 2025-07-10 02:16:24,595 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:16:25,384 - root - INFO - step: 57650 loss: 16.5595 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 14.0142 +[titan] 2025-07-10 02:16:25,384 - root - INFO - lr: 1.3463e-04 gnorm: 0.84 [12:41:24< 9:19:20] +[titan] 2025-07-10 02:16:29,318 - root - INFO - step: 57655 loss: 16.3110 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.8055 +[titan] 2025-07-10 02:16:29,318 - root - INFO - lr: 1.3461e-04 gnorm: 0.82 [12:41:28< 9:19:16] +[titan] 2025-07-10 02:16:33,234 - root - INFO - step: 57660 loss: 16.5978 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5655 global_avg_mtp_loss: 14.0323 +[titan] 2025-07-10 02:16:33,235 - root - INFO - lr: 1.3459e-04 gnorm: 0.87 [12:41:32< 9:19:12] +[titan] 2025-07-10 02:16:37,155 - root - INFO - step: 57665 loss: 16.6041 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5555 global_avg_mtp_loss: 14.0486 +[titan] 2025-07-10 02:16:37,155 - root - INFO - lr: 1.3457e-04 gnorm: 0.87 [12:41:36< 9:19:08] +[titan] 2025-07-10 02:16:41,082 - root - INFO - step: 57670 loss: 16.3999 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-10 02:16:41,082 - root - INFO - lr: 1.3455e-04 gnorm: 0.85 [12:41:40< 9:19:04] +[titan] 2025-07-10 02:16:45,003 - root - INFO - step: 57675 loss: 16.3302 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8219 +[titan] 2025-07-10 02:16:45,003 - root - INFO - lr: 1.3453e-04 gnorm: 0.86 [12:41:44< 9:19:00] +[titan] 2025-07-10 02:16:48,901 - root - INFO - step: 57680 loss: 16.4993 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9624 +[titan] 2025-07-10 02:16:48,901 - root - INFO - lr: 1.3450e-04 gnorm: 0.86 [12:41:48< 9:18:56] +[titan] 2025-07-10 02:16:52,823 - root - INFO - step: 57685 loss: 16.7103 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5774 global_avg_mtp_loss: 14.1329 +[titan] 2025-07-10 02:16:52,823 - root - INFO - lr: 1.3448e-04 gnorm: 0.86 [12:41:52< 9:18:52] +[titan] 2025-07-10 02:16:56,754 - root - INFO - step: 57690 loss: 16.4618 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9292 +[titan] 2025-07-10 02:16:56,754 - root - INFO - lr: 1.3446e-04 gnorm: 0.87 [12:41:56< 9:18:48] +[titan] 2025-07-10 02:17:00,711 - root - INFO - step: 57695 loss: 16.2004 memory: 44.58GiB(31.99%) tps: 82,811 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7107 +[titan] 2025-07-10 02:17:00,711 - root - INFO - lr: 1.3444e-04 gnorm: 0.84 [12:42:00< 9:18:44] +[titan] 2025-07-10 02:17:03,876 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:17:04,665 - root - INFO - step: 57700 loss: 16.6320 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0811 +[titan] 2025-07-10 02:17:04,665 - root - INFO - lr: 1.3442e-04 gnorm: 0.87 [12:42:04< 9:18:40] +[titan] 2025-07-10 02:17:08,587 - root - INFO - step: 57705 loss: 16.5792 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 14.0362 +[titan] 2025-07-10 02:17:08,587 - root - INFO - lr: 1.3440e-04 gnorm: 0.86 [12:42:07< 9:18:36] +[titan] 2025-07-10 02:17:12,528 - root - INFO - step: 57710 loss: 16.4406 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9165 +[titan] 2025-07-10 02:17:12,529 - root - INFO - lr: 1.3438e-04 gnorm: 0.83 [12:42:11< 9:18:32] +[titan] 2025-07-10 02:17:16,465 - root - INFO - step: 57715 loss: 16.1880 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7102 +[titan] 2025-07-10 02:17:16,465 - root - INFO - lr: 1.3436e-04 gnorm: 0.87 [12:42:15< 9:18:28] +[titan] 2025-07-10 02:17:20,367 - root - INFO - step: 57720 loss: 16.5443 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 13.9958 +[titan] 2025-07-10 02:17:20,368 - root - INFO - lr: 1.3434e-04 gnorm: 0.85 [12:42:19< 9:18:24] +[titan] 2025-07-10 02:17:24,319 - root - INFO - step: 57725 loss: 16.3358 memory: 44.58GiB(31.99%) tps: 82,925 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8373 +[titan] 2025-07-10 02:17:24,319 - root - INFO - lr: 1.3432e-04 gnorm: 0.87 [12:42:23< 9:18:20] +[titan] 2025-07-10 02:17:28,272 - root - INFO - step: 57730 loss: 16.3773 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 02:17:28,272 - root - INFO - lr: 1.3430e-04 gnorm: 0.78 [12:42:27< 9:18:16] +[titan] 2025-07-10 02:17:32,207 - root - INFO - step: 57735 loss: 16.4693 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.9470 +[titan] 2025-07-10 02:17:32,207 - root - INFO - lr: 1.3428e-04 gnorm: 0.82 [12:42:31< 9:18:12] +[titan] 2025-07-10 02:17:36,103 - root - INFO - step: 57740 loss: 16.3438 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.8190 +[titan] 2025-07-10 02:17:36,103 - root - INFO - lr: 1.3425e-04 gnorm: 0.84 [12:42:35< 9:18:08] +[titan] 2025-07-10 02:17:40,015 - root - INFO - step: 57745 loss: 16.4680 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 13.9332 +[titan] 2025-07-10 02:17:40,015 - root - INFO - lr: 1.3423e-04 gnorm: 0.80 [12:42:39< 9:18:04] +[titan] 2025-07-10 02:17:43,146 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:17:43,935 - root - INFO - step: 57750 loss: 16.7797 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5867 global_avg_mtp_loss: 14.1930 +[titan] 2025-07-10 02:17:43,935 - root - INFO - lr: 1.3421e-04 gnorm: 0.93 [12:42:43< 9:18:00] +[titan] 2025-07-10 02:17:47,848 - root - INFO - step: 57755 loss: 16.4945 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.9560 +[titan] 2025-07-10 02:17:47,848 - root - INFO - lr: 1.3419e-04 gnorm: 0.83 [12:42:47< 9:17:56] +[titan] 2025-07-10 02:17:51,771 - root - INFO - step: 57760 loss: 16.5983 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-10 02:17:51,771 - root - INFO - lr: 1.3417e-04 gnorm: 0.84 [12:42:51< 9:17:52] +[titan] 2025-07-10 02:17:55,687 - root - INFO - step: 57765 loss: 16.2234 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7457 +[titan] 2025-07-10 02:17:55,688 - root - INFO - lr: 1.3415e-04 gnorm: 0.83 [12:42:55< 9:17:48] +[titan] 2025-07-10 02:17:59,605 - root - INFO - step: 57770 loss: 16.2805 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.7804 +[titan] 2025-07-10 02:17:59,606 - root - INFO - lr: 1.3413e-04 gnorm: 1.00 [12:42:58< 9:17:44] +[titan] 2025-07-10 02:18:03,572 - root - INFO - step: 57775 loss: 16.2672 memory: 44.58GiB(31.99%) tps: 82,612 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-10 02:18:03,573 - root - INFO - lr: 1.3411e-04 gnorm: 0.90 [12:43:02< 9:17:40] +[titan] 2025-07-10 02:18:07,498 - root - INFO - step: 57780 loss: 16.3632 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8407 +[titan] 2025-07-10 02:18:07,498 - root - INFO - lr: 1.3409e-04 gnorm: 0.84 [12:43:06< 9:17:36] +[titan] 2025-07-10 02:18:11,425 - root - INFO - step: 57785 loss: 16.6212 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.0585 +[titan] 2025-07-10 02:18:11,425 - root - INFO - lr: 1.3407e-04 gnorm: 0.85 [12:43:10< 9:17:32] +[titan] 2025-07-10 02:18:15,326 - root - INFO - step: 57790 loss: 16.5844 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5573 global_avg_mtp_loss: 14.0271 +[titan] 2025-07-10 02:18:15,327 - root - INFO - lr: 1.3405e-04 gnorm: 0.88 [12:43:14< 9:17:28] +[titan] 2025-07-10 02:18:19,251 - root - INFO - step: 57795 loss: 16.6640 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5664 global_avg_mtp_loss: 14.0975 +[titan] 2025-07-10 02:18:19,252 - root - INFO - lr: 1.3402e-04 gnorm: 0.84 [12:43:18< 9:17:24] +[titan] 2025-07-10 02:18:22,387 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:18:23,174 - root - INFO - step: 57800 loss: 16.6819 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5663 global_avg_mtp_loss: 14.1155 +[titan] 2025-07-10 02:18:23,174 - root - INFO - lr: 1.3400e-04 gnorm: 0.89 [12:43:22< 9:17:20] +[titan] 2025-07-10 02:18:27,076 - root - INFO - step: 57805 loss: 16.3663 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8600 +[titan] 2025-07-10 02:18:27,077 - root - INFO - lr: 1.3398e-04 gnorm: 0.86 [12:43:26< 9:17:16] +[titan] 2025-07-10 02:18:31,047 - root - INFO - step: 57810 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 82,533 tflops: 284.83 mfu: 28.80% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8021 +[titan] 2025-07-10 02:18:31,047 - root - INFO - lr: 1.3396e-04 gnorm: 0.88 [12:43:30< 9:17:12] +[titan] 2025-07-10 02:18:34,972 - root - INFO - step: 57815 loss: 16.0403 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4483 global_avg_mtp_loss: 13.5919 +[titan] 2025-07-10 02:18:34,972 - root - INFO - lr: 1.3394e-04 gnorm: 0.83 [12:43:34< 9:17:08] +[titan] 2025-07-10 02:18:38,878 - root - INFO - step: 57820 loss: 16.2026 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.7131 +[titan] 2025-07-10 02:18:38,878 - root - INFO - lr: 1.3392e-04 gnorm: 0.86 [12:43:38< 9:17:04] +[titan] 2025-07-10 02:18:42,776 - root - INFO - step: 57825 loss: 16.5725 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 14.0325 +[titan] 2025-07-10 02:18:42,776 - root - INFO - lr: 1.3390e-04 gnorm: 0.87 [12:43:42< 9:17:00] +[titan] 2025-07-10 02:18:46,689 - root - INFO - step: 57830 loss: 16.3467 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.8216 +[titan] 2025-07-10 02:18:46,689 - root - INFO - lr: 1.3388e-04 gnorm: 0.88 [12:43:46< 9:16:56] +[titan] 2025-07-10 02:18:50,596 - root - INFO - step: 57835 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9093 +[titan] 2025-07-10 02:18:50,596 - root - INFO - lr: 1.3386e-04 gnorm: 0.94 [12:43:49< 9:16:52] +[titan] 2025-07-10 02:18:54,547 - root - INFO - step: 57840 loss: 16.2041 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7241 +[titan] 2025-07-10 02:18:54,547 - root - INFO - lr: 1.3384e-04 gnorm: 0.88 [12:43:53< 9:16:48] +[titan] 2025-07-10 02:18:58,465 - root - INFO - step: 57845 loss: 16.4990 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 13.9593 +[titan] 2025-07-10 02:18:58,465 - root - INFO - lr: 1.3382e-04 gnorm: 0.88 [12:43:57< 9:16:44] +[titan] 2025-07-10 02:19:01,618 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:19:02,407 - root - INFO - step: 57850 loss: 16.5094 memory: 44.58GiB(31.99%) tps: 83,142 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 13.9494 +[titan] 2025-07-10 02:19:02,407 - root - INFO - lr: 1.3380e-04 gnorm: 0.90 [12:44:01< 9:16:40] +[titan] 2025-07-10 02:19:06,389 - root - INFO - step: 57855 loss: 16.6156 memory: 44.58GiB(31.99%) tps: 82,296 tflops: 284.02 mfu: 28.72% global_avg_ntp_loss: 2.5692 global_avg_mtp_loss: 14.0464 +[titan] 2025-07-10 02:19:06,389 - root - INFO - lr: 1.3377e-04 gnorm: 0.87 [12:44:05< 9:16:36] +[titan] 2025-07-10 02:19:07,329 - root - INFO - Dumping profiler traces at step 57856 +[titan] 2025-07-10 02:19:07,362 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:19:10,514 - root - INFO - step: 57860 loss: 16.3275 memory: 44.58GiB(31.99%) tps: 79,444 tflops: 274.17 mfu: 27.72% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8275 +[titan] 2025-07-10 02:19:10,514 - root - INFO - lr: 1.3375e-04 gnorm: 0.85 [12:44:09< 9:16:32] +[titan] 2025-07-10 02:19:14,454 - root - INFO - step: 57865 loss: 16.4651 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 02:19:14,454 - root - INFO - lr: 1.3373e-04 gnorm: 0.82 [12:44:13< 9:16:28] +[titan] 2025-07-10 02:19:18,361 - root - INFO - step: 57870 loss: 16.4097 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8926 +[titan] 2025-07-10 02:19:18,362 - root - INFO - lr: 1.3371e-04 gnorm: 0.90 [12:44:17< 9:16:24] +[titan] 2025-07-10 02:19:22,266 - root - INFO - step: 57875 loss: 16.6433 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0912 +[titan] 2025-07-10 02:19:22,266 - root - INFO - lr: 1.3369e-04 gnorm: 0.84 [12:44:21< 9:16:20] +[titan] 2025-07-10 02:19:26,170 - root - INFO - step: 57880 loss: 16.6801 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.1179 +[titan] 2025-07-10 02:19:26,170 - root - INFO - lr: 1.3367e-04 gnorm: 0.86 [12:44:25< 9:16:16] +[titan] 2025-07-10 02:19:30,143 - root - INFO - step: 57885 loss: 16.5162 memory: 44.58GiB(31.99%) tps: 82,470 tflops: 284.62 mfu: 28.78% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9763 +[titan] 2025-07-10 02:19:30,144 - root - INFO - lr: 1.3365e-04 gnorm: 0.87 [12:44:29< 9:16:12] +[titan] 2025-07-10 02:19:34,050 - root - INFO - step: 57890 loss: 16.4480 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9237 +[titan] 2025-07-10 02:19:34,050 - root - INFO - lr: 1.3363e-04 gnorm: 0.89 [12:44:33< 9:16:08] +[titan] 2025-07-10 02:19:37,962 - root - INFO - step: 57895 loss: 16.6864 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5751 global_avg_mtp_loss: 14.1114 +[titan] 2025-07-10 02:19:37,963 - root - INFO - lr: 1.3361e-04 gnorm: 0.89 [12:44:37< 9:16:04] +[titan] 2025-07-10 02:19:41,118 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:19:41,910 - root - INFO - step: 57900 loss: 16.6268 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.5607 global_avg_mtp_loss: 14.0662 +[titan] 2025-07-10 02:19:41,910 - root - INFO - lr: 1.3359e-04 gnorm: 0.83 [12:44:41< 9:16:00] +[titan] 2025-07-10 02:19:45,822 - root - INFO - step: 57905 loss: 16.3774 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8634 +[titan] 2025-07-10 02:19:45,823 - root - INFO - lr: 1.3357e-04 gnorm: 0.87 [12:44:45< 9:15:56] +[titan] 2025-07-10 02:19:49,747 - root - INFO - step: 57910 loss: 16.4430 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9262 +[titan] 2025-07-10 02:19:49,747 - root - INFO - lr: 1.3355e-04 gnorm: 0.87 [12:44:49< 9:15:52] +[titan] 2025-07-10 02:19:53,653 - root - INFO - step: 57915 loss: 16.6637 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0912 +[titan] 2025-07-10 02:19:53,653 - root - INFO - lr: 1.3352e-04 gnorm: 0.81 [12:44:52< 9:15:48] +[titan] 2025-07-10 02:19:57,590 - root - INFO - step: 57920 loss: 16.3037 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.7954 +[titan] 2025-07-10 02:19:57,590 - root - INFO - lr: 1.3350e-04 gnorm: 0.82 [12:44:56< 9:15:44] +[titan] 2025-07-10 02:20:01,481 - root - INFO - step: 57925 loss: 16.4469 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9172 +[titan] 2025-07-10 02:20:01,481 - root - INFO - lr: 1.3348e-04 gnorm: 0.83 [12:45:00< 9:15:40] +[titan] 2025-07-10 02:20:05,427 - root - INFO - step: 57930 loss: 16.3651 memory: 44.58GiB(31.99%) tps: 83,042 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8551 +[titan] 2025-07-10 02:20:05,428 - root - INFO - lr: 1.3346e-04 gnorm: 0.82 [12:45:04< 9:15:37] +[titan] 2025-07-10 02:20:09,336 - root - INFO - step: 57935 loss: 16.3590 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5253 global_avg_mtp_loss: 13.8336 +[titan] 2025-07-10 02:20:09,336 - root - INFO - lr: 1.3344e-04 gnorm: 0.91 [12:45:08< 9:15:33] +[titan] 2025-07-10 02:20:13,270 - root - INFO - step: 57940 loss: 16.2724 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7841 +[titan] 2025-07-10 02:20:13,270 - root - INFO - lr: 1.3342e-04 gnorm: 0.88 [12:45:12< 9:15:29] +[titan] 2025-07-10 02:20:17,170 - root - INFO - step: 57945 loss: 16.4598 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9348 +[titan] 2025-07-10 02:20:17,170 - root - INFO - lr: 1.3340e-04 gnorm: 0.87 [12:45:16< 9:15:25] +[titan] 2025-07-10 02:20:20,291 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:20:21,082 - root - INFO - step: 57950 loss: 16.3920 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8730 +[titan] 2025-07-10 02:20:21,083 - root - INFO - lr: 1.3338e-04 gnorm: 0.86 [12:45:20< 9:15:21] +[titan] 2025-07-10 02:20:25,006 - root - INFO - step: 57955 loss: 16.3950 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8904 +[titan] 2025-07-10 02:20:25,006 - root - INFO - lr: 1.3336e-04 gnorm: 0.84 [12:45:24< 9:15:17] +[titan] 2025-07-10 02:20:28,938 - root - INFO - step: 57960 loss: 16.0078 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.5442 +[titan] 2025-07-10 02:20:28,938 - root - INFO - lr: 1.3334e-04 gnorm: 0.94 [12:45:28< 9:15:13] +[titan] 2025-07-10 02:20:32,841 - root - INFO - step: 57965 loss: 16.5844 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 14.0278 +[titan] 2025-07-10 02:20:32,842 - root - INFO - lr: 1.3332e-04 gnorm: 0.86 [12:45:32< 9:15:09] +[titan] 2025-07-10 02:20:36,742 - root - INFO - step: 57970 loss: 16.4064 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8880 +[titan] 2025-07-10 02:20:36,742 - root - INFO - lr: 1.3330e-04 gnorm: 0.91 [12:45:36< 9:15:05] +[titan] 2025-07-10 02:20:40,680 - root - INFO - step: 57975 loss: 16.6361 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.5773 global_avg_mtp_loss: 14.0588 +[titan] 2025-07-10 02:20:40,680 - root - INFO - lr: 1.3327e-04 gnorm: 0.92 [12:45:39< 9:15:01] +[titan] 2025-07-10 02:20:44,628 - root - INFO - step: 57980 loss: 16.3729 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8643 +[titan] 2025-07-10 02:20:44,628 - root - INFO - lr: 1.3325e-04 gnorm: 0.84 [12:45:43< 9:14:57] +[titan] 2025-07-10 02:20:48,602 - root - INFO - step: 57985 loss: 16.6300 memory: 44.58GiB(31.99%) tps: 82,472 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0763 +[titan] 2025-07-10 02:20:48,602 - root - INFO - lr: 1.3323e-04 gnorm: 0.81 [12:45:47< 9:14:53] +[titan] 2025-07-10 02:20:52,533 - root - INFO - step: 57990 loss: 16.6939 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5863 global_avg_mtp_loss: 14.1076 +[titan] 2025-07-10 02:20:52,534 - root - INFO - lr: 1.3321e-04 gnorm: 0.83 [12:45:51< 9:14:49] +[titan] 2025-07-10 02:20:56,456 - root - INFO - step: 57995 loss: 16.5979 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5676 global_avg_mtp_loss: 14.0303 +[titan] 2025-07-10 02:20:56,456 - root - INFO - lr: 1.3319e-04 gnorm: 0.83 [12:45:55< 9:14:45] +[titan] 2025-07-10 02:20:59,602 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:21:00,391 - root - INFO - step: 58000 loss: 16.6155 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 14.0694 +[titan] 2025-07-10 02:21:00,391 - root - INFO - lr: 1.3317e-04 gnorm: 0.89 [12:45:59< 9:14:41] +[titan] 2025-07-10 02:21:04,309 - root - INFO - step: 58005 loss: 16.6387 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0739 +[titan] 2025-07-10 02:21:04,309 - root - INFO - lr: 1.3315e-04 gnorm: 0.90 [12:46:03< 9:14:37] +[titan] 2025-07-10 02:21:08,212 - root - INFO - step: 58010 loss: 16.6300 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5636 global_avg_mtp_loss: 14.0665 +[titan] 2025-07-10 02:21:08,213 - root - INFO - lr: 1.3313e-04 gnorm: 0.84 [12:46:07< 9:14:33] +[titan] 2025-07-10 02:21:12,169 - root - INFO - step: 58015 loss: 16.5678 memory: 44.58GiB(31.99%) tps: 82,829 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 14.0202 +[titan] 2025-07-10 02:21:12,169 - root - INFO - lr: 1.3311e-04 gnorm: 0.86 [12:46:11< 9:14:29] +[titan] 2025-07-10 02:21:16,106 - root - INFO - step: 58020 loss: 16.6372 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0742 +[titan] 2025-07-10 02:21:16,106 - root - INFO - lr: 1.3309e-04 gnorm: 0.83 [12:46:15< 9:14:25] +[titan] 2025-07-10 02:21:20,038 - root - INFO - step: 58025 loss: 16.2516 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7658 +[titan] 2025-07-10 02:21:20,038 - root - INFO - lr: 1.3307e-04 gnorm: 0.83 [12:46:19< 9:14:21] +[titan] 2025-07-10 02:21:23,966 - root - INFO - step: 58030 loss: 16.3150 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8090 +[titan] 2025-07-10 02:21:23,966 - root - INFO - lr: 1.3305e-04 gnorm: 0.82 [12:46:23< 9:14:17] +[titan] 2025-07-10 02:21:27,907 - root - INFO - step: 58035 loss: 16.8141 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5993 global_avg_mtp_loss: 14.2148 +[titan] 2025-07-10 02:21:27,907 - root - INFO - lr: 1.3303e-04 gnorm: 0.85 [12:46:27< 9:14:13] +[titan] 2025-07-10 02:21:31,825 - root - INFO - step: 58040 loss: 16.4128 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.8819 +[titan] 2025-07-10 02:21:31,826 - root - INFO - lr: 1.3300e-04 gnorm: 0.84 [12:46:31< 9:14:09] +[titan] 2025-07-10 02:21:35,763 - root - INFO - step: 58045 loss: 16.5326 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5458 global_avg_mtp_loss: 13.9868 +[titan] 2025-07-10 02:21:35,764 - root - INFO - lr: 1.3298e-04 gnorm: 0.85 [12:46:35< 9:14:05] +[titan] 2025-07-10 02:21:38,882 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:21:39,667 - root - INFO - step: 58050 loss: 16.3607 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8555 +[titan] 2025-07-10 02:21:39,667 - root - INFO - lr: 1.3296e-04 gnorm: 0.83 [12:46:38< 9:14:01] +[titan] 2025-07-10 02:21:43,596 - root - INFO - step: 58055 loss: 16.5206 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9849 +[titan] 2025-07-10 02:21:43,596 - root - INFO - lr: 1.3294e-04 gnorm: 0.82 [12:46:42< 9:13:57] +[titan] 2025-07-10 02:21:47,517 - root - INFO - step: 58060 loss: 16.3139 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8141 +[titan] 2025-07-10 02:21:47,517 - root - INFO - lr: 1.3292e-04 gnorm: 0.83 [12:46:46< 9:13:53] +[titan] 2025-07-10 02:21:51,409 - root - INFO - step: 58065 loss: 16.4713 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9421 +[titan] 2025-07-10 02:21:51,410 - root - INFO - lr: 1.3290e-04 gnorm: 0.79 [12:46:50< 9:13:49] +[titan] 2025-07-10 02:21:55,319 - root - INFO - step: 58070 loss: 16.3425 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8313 +[titan] 2025-07-10 02:21:55,319 - root - INFO - lr: 1.3288e-04 gnorm: 0.86 [12:46:54< 9:13:45] +[titan] 2025-07-10 02:21:59,236 - root - INFO - step: 58075 loss: 16.5236 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9857 +[titan] 2025-07-10 02:21:59,236 - root - INFO - lr: 1.3286e-04 gnorm: 0.86 [12:46:58< 9:13:41] +[titan] 2025-07-10 02:22:03,134 - root - INFO - step: 58080 loss: 16.2494 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4930 global_avg_mtp_loss: 13.7564 +[titan] 2025-07-10 02:22:03,134 - root - INFO - lr: 1.3284e-04 gnorm: 1.14 [12:47:02< 9:13:37] +[titan] 2025-07-10 02:22:07,045 - root - INFO - step: 58085 loss: 16.3027 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.7959 +[titan] 2025-07-10 02:22:07,045 - root - INFO - lr: 1.3282e-04 gnorm: 0.88 [12:47:06< 9:13:33] +[titan] 2025-07-10 02:22:10,945 - root - INFO - step: 58090 loss: 16.6363 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5601 global_avg_mtp_loss: 14.0762 +[titan] 2025-07-10 02:22:10,945 - root - INFO - lr: 1.3280e-04 gnorm: 0.86 [12:47:10< 9:13:29] +[titan] 2025-07-10 02:22:14,883 - root - INFO - step: 58095 loss: 16.1929 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7080 +[titan] 2025-07-10 02:22:14,884 - root - INFO - lr: 1.3278e-04 gnorm: 0.85 [12:47:14< 9:13:25] +[titan] 2025-07-10 02:22:17,993 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:22:18,784 - root - INFO - step: 58100 loss: 16.7567 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5915 global_avg_mtp_loss: 14.1652 +[titan] 2025-07-10 02:22:18,784 - root - INFO - lr: 1.3275e-04 gnorm: 0.86 [12:47:18< 9:13:21] +[titan] 2025-07-10 02:22:22,703 - root - INFO - step: 58105 loss: 16.4309 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.9095 +[titan] 2025-07-10 02:22:22,703 - root - INFO - lr: 1.3273e-04 gnorm: 0.82 [12:47:22< 9:13:17] +[titan] 2025-07-10 02:22:26,614 - root - INFO - step: 58110 loss: 16.2548 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 02:22:26,614 - root - INFO - lr: 1.3271e-04 gnorm: 0.82 [12:47:25< 9:13:13] +[titan] 2025-07-10 02:22:30,545 - root - INFO - step: 58115 loss: 16.2137 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7341 +[titan] 2025-07-10 02:22:30,546 - root - INFO - lr: 1.3269e-04 gnorm: 0.81 [12:47:29< 9:13:09] +[titan] 2025-07-10 02:22:34,483 - root - INFO - step: 58120 loss: 16.4415 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.9193 +[titan] 2025-07-10 02:22:34,483 - root - INFO - lr: 1.3267e-04 gnorm: 0.83 [12:47:33< 9:13:05] +[titan] 2025-07-10 02:22:38,400 - root - INFO - step: 58125 loss: 16.4178 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.9055 +[titan] 2025-07-10 02:22:38,400 - root - INFO - lr: 1.3265e-04 gnorm: 0.85 [12:47:37< 9:13:01] +[titan] 2025-07-10 02:22:42,304 - root - INFO - step: 58130 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.8928 +[titan] 2025-07-10 02:22:42,304 - root - INFO - lr: 1.3263e-04 gnorm: 0.99 [12:47:41< 9:12:57] +[titan] 2025-07-10 02:22:46,264 - root - INFO - step: 58135 loss: 16.4070 memory: 44.58GiB(31.99%) tps: 82,750 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-10 02:22:46,264 - root - INFO - lr: 1.3261e-04 gnorm: 0.83 [12:47:45< 9:12:53] +[titan] 2025-07-10 02:22:50,210 - root - INFO - step: 58140 loss: 16.5713 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5541 global_avg_mtp_loss: 14.0172 +[titan] 2025-07-10 02:22:50,210 - root - INFO - lr: 1.3259e-04 gnorm: 0.83 [12:47:49< 9:12:49] +[titan] 2025-07-10 02:22:54,131 - root - INFO - step: 58145 loss: 16.3340 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8323 +[titan] 2025-07-10 02:22:54,131 - root - INFO - lr: 1.3257e-04 gnorm: 0.90 [12:47:53< 9:12:45] +[titan] 2025-07-10 02:22:57,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:22:58,072 - root - INFO - step: 58150 loss: 16.2581 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7763 +[titan] 2025-07-10 02:22:58,072 - root - INFO - lr: 1.3255e-04 gnorm: 0.84 [12:47:57< 9:12:41] +[titan] 2025-07-10 02:23:01,992 - root - INFO - step: 58155 loss: 16.2419 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7557 +[titan] 2025-07-10 02:23:01,992 - root - INFO - lr: 1.3253e-04 gnorm: 0.86 [12:48:01< 9:12:37] +[titan] 2025-07-10 02:23:05,909 - root - INFO - step: 58160 loss: 16.1579 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.6733 +[titan] 2025-07-10 02:23:05,909 - root - INFO - lr: 1.3251e-04 gnorm: 0.98 [12:48:05< 9:12:33] +[titan] 2025-07-10 02:23:09,842 - root - INFO - step: 58165 loss: 16.5831 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0295 +[titan] 2025-07-10 02:23:09,842 - root - INFO - lr: 1.3248e-04 gnorm: 0.91 [12:48:09< 9:12:29] +[titan] 2025-07-10 02:23:13,778 - root - INFO - step: 58170 loss: 16.5767 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 14.0233 +[titan] 2025-07-10 02:23:13,778 - root - INFO - lr: 1.3246e-04 gnorm: 0.86 [12:48:13< 9:12:25] +[titan] 2025-07-10 02:23:17,673 - root - INFO - step: 58175 loss: 16.5717 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5595 global_avg_mtp_loss: 14.0122 +[titan] 2025-07-10 02:23:17,673 - root - INFO - lr: 1.3244e-04 gnorm: 0.90 [12:48:16< 9:12:21] +[titan] 2025-07-10 02:23:21,587 - root - INFO - step: 58180 loss: 16.4982 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.9731 +[titan] 2025-07-10 02:23:21,587 - root - INFO - lr: 1.3242e-04 gnorm: 0.86 [12:48:20< 9:12:17] +[titan] 2025-07-10 02:23:25,500 - root - INFO - step: 58185 loss: 16.5700 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5555 global_avg_mtp_loss: 14.0145 +[titan] 2025-07-10 02:23:25,500 - root - INFO - lr: 1.3240e-04 gnorm: 0.84 [12:48:24< 9:12:13] +[titan] 2025-07-10 02:23:29,450 - root - INFO - step: 58190 loss: 16.5596 memory: 44.58GiB(31.99%) tps: 82,980 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 14.0208 +[titan] 2025-07-10 02:23:29,450 - root - INFO - lr: 1.3238e-04 gnorm: 0.89 [12:48:28< 9:12:09] +[titan] 2025-07-10 02:23:33,364 - root - INFO - step: 58195 loss: 16.5886 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 14.0453 +[titan] 2025-07-10 02:23:33,365 - root - INFO - lr: 1.3236e-04 gnorm: 0.88 [12:48:32< 9:12:05] +[titan] 2025-07-10 02:23:36,485 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:23:37,273 - root - INFO - step: 58200 loss: 16.3686 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8562 +[titan] 2025-07-10 02:23:37,274 - root - INFO - lr: 1.3234e-04 gnorm: 0.92 [12:48:36< 9:12:01] +[titan] 2025-07-10 02:23:41,164 - root - INFO - step: 58205 loss: 15.8349 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.4332 global_avg_mtp_loss: 13.4017 +[titan] 2025-07-10 02:23:41,165 - root - INFO - lr: 1.3232e-04 gnorm: 0.89 [12:48:40< 9:11:57] +[titan] 2025-07-10 02:23:45,098 - root - INFO - step: 58210 loss: 16.3805 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8712 +[titan] 2025-07-10 02:23:45,099 - root - INFO - lr: 1.3230e-04 gnorm: 0.91 [12:48:44< 9:11:53] +[titan] 2025-07-10 02:23:49,001 - root - INFO - step: 58215 loss: 16.6592 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.0748 +[titan] 2025-07-10 02:23:49,001 - root - INFO - lr: 1.3228e-04 gnorm: 0.88 [12:48:48< 9:11:49] +[titan] 2025-07-10 02:23:52,948 - root - INFO - step: 58220 loss: 16.5333 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9898 +[titan] 2025-07-10 02:23:52,948 - root - INFO - lr: 1.3226e-04 gnorm: 0.86 [12:48:52< 9:11:45] +[titan] 2025-07-10 02:23:56,873 - root - INFO - step: 58225 loss: 16.5810 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0210 +[titan] 2025-07-10 02:23:56,873 - root - INFO - lr: 1.3223e-04 gnorm: 0.81 [12:48:56< 9:11:41] +[titan] 2025-07-10 02:24:00,786 - root - INFO - step: 58230 loss: 16.5384 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 13.9943 +[titan] 2025-07-10 02:24:00,786 - root - INFO - lr: 1.3221e-04 gnorm: 0.82 [12:49:00< 9:11:37] +[titan] 2025-07-10 02:24:04,705 - root - INFO - step: 58235 loss: 16.5740 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0204 +[titan] 2025-07-10 02:24:04,706 - root - INFO - lr: 1.3219e-04 gnorm: 0.84 [12:49:03< 9:11:33] +[titan] 2025-07-10 02:24:08,613 - root - INFO - step: 58240 loss: 16.3174 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8199 +[titan] 2025-07-10 02:24:08,613 - root - INFO - lr: 1.3217e-04 gnorm: 0.80 [12:49:07< 9:11:29] +[titan] 2025-07-10 02:24:12,559 - root - INFO - step: 58245 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9525 +[titan] 2025-07-10 02:24:12,559 - root - INFO - lr: 1.3215e-04 gnorm: 0.90 [12:49:11< 9:11:25] +[titan] 2025-07-10 02:24:15,680 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:24:16,469 - root - INFO - step: 58250 loss: 16.4196 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.9066 +[titan] 2025-07-10 02:24:16,470 - root - INFO - lr: 1.3213e-04 gnorm: 0.83 [12:49:15< 9:11:21] +[titan] 2025-07-10 02:24:20,390 - root - INFO - step: 58255 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6507 +[titan] 2025-07-10 02:24:20,390 - root - INFO - lr: 1.3211e-04 gnorm: 0.81 [12:49:19< 9:11:17] +[titan] 2025-07-10 02:24:24,291 - root - INFO - step: 58260 loss: 16.4276 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.8955 +[titan] 2025-07-10 02:24:24,291 - root - INFO - lr: 1.3209e-04 gnorm: 0.83 [12:49:23< 9:11:13] +[titan] 2025-07-10 02:24:28,212 - root - INFO - step: 58265 loss: 16.3344 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8272 +[titan] 2025-07-10 02:24:28,212 - root - INFO - lr: 1.3207e-04 gnorm: 0.90 [12:49:27< 9:11:09] +[titan] 2025-07-10 02:24:32,130 - root - INFO - step: 58270 loss: 16.4288 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 02:24:32,130 - root - INFO - lr: 1.3205e-04 gnorm: 0.88 [12:49:31< 9:11:05] +[titan] 2025-07-10 02:24:36,049 - root - INFO - step: 58275 loss: 16.5133 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 13.9769 +[titan] 2025-07-10 02:24:36,049 - root - INFO - lr: 1.3203e-04 gnorm: 0.93 [12:49:35< 9:11:01] +[titan] 2025-07-10 02:24:39,959 - root - INFO - step: 58280 loss: 16.4671 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9415 +[titan] 2025-07-10 02:24:39,959 - root - INFO - lr: 1.3201e-04 gnorm: 0.83 [12:49:39< 9:10:57] +[titan] 2025-07-10 02:24:43,879 - root - INFO - step: 58285 loss: 16.3769 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 02:24:43,880 - root - INFO - lr: 1.3199e-04 gnorm: 0.87 [12:49:43< 9:10:53] +[titan] 2025-07-10 02:24:47,790 - root - INFO - step: 58290 loss: 16.4015 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8877 +[titan] 2025-07-10 02:24:47,790 - root - INFO - lr: 1.3196e-04 gnorm: 0.90 [12:49:47< 9:10:49] +[titan] 2025-07-10 02:24:51,726 - root - INFO - step: 58295 loss: 16.4567 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.9411 +[titan] 2025-07-10 02:24:51,726 - root - INFO - lr: 1.3194e-04 gnorm: 0.90 [12:49:51< 9:10:45] +[titan] 2025-07-10 02:24:54,854 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:24:55,647 - root - INFO - step: 58300 loss: 16.2376 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7527 +[titan] 2025-07-10 02:24:55,648 - root - INFO - lr: 1.3192e-04 gnorm: 0.86 [12:49:54< 9:10:41] +[titan] 2025-07-10 02:24:59,564 - root - INFO - step: 58305 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.8399 +[titan] 2025-07-10 02:24:59,564 - root - INFO - lr: 1.3190e-04 gnorm: 0.85 [12:49:58< 9:10:37] +[titan] 2025-07-10 02:25:03,484 - root - INFO - step: 58310 loss: 16.5113 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9791 +[titan] 2025-07-10 02:25:03,484 - root - INFO - lr: 1.3188e-04 gnorm: 0.90 [12:50:02< 9:10:33] +[titan] 2025-07-10 02:25:07,427 - root - INFO - step: 58315 loss: 16.5403 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 14.0070 +[titan] 2025-07-10 02:25:07,427 - root - INFO - lr: 1.3186e-04 gnorm: 0.86 [12:50:06< 9:10:29] +[titan] 2025-07-10 02:25:11,352 - root - INFO - step: 58320 loss: 16.6239 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5789 global_avg_mtp_loss: 14.0449 +[titan] 2025-07-10 02:25:11,352 - root - INFO - lr: 1.3184e-04 gnorm: 0.88 [12:50:10< 9:10:25] +[titan] 2025-07-10 02:25:15,259 - root - INFO - step: 58325 loss: 16.2137 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.7304 +[titan] 2025-07-10 02:25:15,259 - root - INFO - lr: 1.3182e-04 gnorm: 0.84 [12:50:14< 9:10:21] +[titan] 2025-07-10 02:25:19,188 - root - INFO - step: 58330 loss: 16.2353 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7495 +[titan] 2025-07-10 02:25:19,188 - root - INFO - lr: 1.3180e-04 gnorm: 0.83 [12:50:18< 9:10:17] +[titan] 2025-07-10 02:25:23,132 - root - INFO - step: 58335 loss: 16.4776 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9425 +[titan] 2025-07-10 02:25:23,132 - root - INFO - lr: 1.3178e-04 gnorm: 0.83 [12:50:22< 9:10:13] +[titan] 2025-07-10 02:25:27,050 - root - INFO - step: 58340 loss: 16.4435 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9190 +[titan] 2025-07-10 02:25:27,050 - root - INFO - lr: 1.3176e-04 gnorm: 0.85 [12:50:26< 9:10:09] +[titan] 2025-07-10 02:25:30,966 - root - INFO - step: 58345 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8513 +[titan] 2025-07-10 02:25:30,967 - root - INFO - lr: 1.3174e-04 gnorm: 0.88 [12:50:30< 9:10:05] +[titan] 2025-07-10 02:25:34,093 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:25:34,882 - root - INFO - step: 58350 loss: 16.6567 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0895 +[titan] 2025-07-10 02:25:34,883 - root - INFO - lr: 1.3172e-04 gnorm: 0.81 [12:50:34< 9:10:01] +[titan] 2025-07-10 02:25:38,804 - root - INFO - step: 58355 loss: 16.1429 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.6684 +[titan] 2025-07-10 02:25:38,805 - root - INFO - lr: 1.3169e-04 gnorm: 0.85 [12:50:38< 9:09:57] +[titan] 2025-07-10 02:25:42,708 - root - INFO - step: 58360 loss: 16.5020 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9648 +[titan] 2025-07-10 02:25:42,709 - root - INFO - lr: 1.3167e-04 gnorm: 0.84 [12:50:41< 9:09:53] +[titan] 2025-07-10 02:25:46,732 - root - INFO - step: 58365 loss: 16.2773 memory: 44.58GiB(31.99%) tps: 81,455 tflops: 281.11 mfu: 28.42% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7942 +[titan] 2025-07-10 02:25:46,732 - root - INFO - lr: 1.3165e-04 gnorm: 0.85 [12:50:46< 9:09:49] +[titan] 2025-07-10 02:25:49,224 - root - INFO - Dumping profiler traces at step 58368 +[titan] 2025-07-10 02:25:49,256 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:25:50,832 - root - INFO - step: 58370 loss: 16.4242 memory: 44.58GiB(31.99%) tps: 79,915 tflops: 275.80 mfu: 27.89% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.8988 +[titan] 2025-07-10 02:25:50,833 - root - INFO - lr: 1.3163e-04 gnorm: 0.87 [12:50:50< 9:09:45] +[titan] 2025-07-10 02:25:54,761 - root - INFO - step: 58375 loss: 16.5124 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9832 +[titan] 2025-07-10 02:25:54,762 - root - INFO - lr: 1.3161e-04 gnorm: 0.83 [12:50:54< 9:09:42] +[titan] 2025-07-10 02:25:58,700 - root - INFO - step: 58380 loss: 16.3006 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-10 02:25:58,700 - root - INFO - lr: 1.3159e-04 gnorm: 0.88 [12:50:57< 9:09:38] +[titan] 2025-07-10 02:26:02,633 - root - INFO - step: 58385 loss: 16.4135 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8932 +[titan] 2025-07-10 02:26:02,633 - root - INFO - lr: 1.3157e-04 gnorm: 0.83 [12:51:01< 9:09:34] +[titan] 2025-07-10 02:26:06,578 - root - INFO - step: 58390 loss: 16.4506 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9299 +[titan] 2025-07-10 02:26:06,578 - root - INFO - lr: 1.3155e-04 gnorm: 0.80 [12:51:05< 9:09:30] +[titan] 2025-07-10 02:26:10,506 - root - INFO - step: 58395 loss: 16.2772 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.7742 +[titan] 2025-07-10 02:26:10,507 - root - INFO - lr: 1.3153e-04 gnorm: 0.86 [12:51:09< 9:09:26] +[titan] 2025-07-10 02:26:13,617 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:26:14,407 - root - INFO - step: 58400 loss: 16.7156 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1347 +[titan] 2025-07-10 02:26:14,407 - root - INFO - lr: 1.3151e-04 gnorm: 0.86 [12:51:13< 9:09:22] +[titan] 2025-07-10 02:26:18,326 - root - INFO - step: 58405 loss: 16.3734 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8584 +[titan] 2025-07-10 02:26:18,326 - root - INFO - lr: 1.3149e-04 gnorm: 0.81 [12:51:17< 9:09:18] +[titan] 2025-07-10 02:26:22,270 - root - INFO - step: 58410 loss: 16.7317 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1415 +[titan] 2025-07-10 02:26:22,270 - root - INFO - lr: 1.3147e-04 gnorm: 0.81 [12:51:21< 9:09:14] +[titan] 2025-07-10 02:26:26,167 - root - INFO - step: 58415 loss: 16.4578 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9352 +[titan] 2025-07-10 02:26:26,167 - root - INFO - lr: 1.3145e-04 gnorm: 0.86 [12:51:25< 9:09:10] +[titan] 2025-07-10 02:26:30,091 - root - INFO - step: 58420 loss: 16.4274 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.8988 +[titan] 2025-07-10 02:26:30,091 - root - INFO - lr: 1.3143e-04 gnorm: 0.82 [12:51:29< 9:09:06] +[titan] 2025-07-10 02:26:34,000 - root - INFO - step: 58425 loss: 16.0958 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.6257 +[titan] 2025-07-10 02:26:34,001 - root - INFO - lr: 1.3140e-04 gnorm: 0.84 [12:51:33< 9:09:02] +[titan] 2025-07-10 02:26:37,928 - root - INFO - step: 58430 loss: 16.4669 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-10 02:26:37,928 - root - INFO - lr: 1.3138e-04 gnorm: 0.84 [12:51:37< 9:08:58] +[titan] 2025-07-10 02:26:41,841 - root - INFO - step: 58435 loss: 16.5477 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0005 +[titan] 2025-07-10 02:26:41,841 - root - INFO - lr: 1.3136e-04 gnorm: 0.87 [12:51:41< 9:08:54] +[titan] 2025-07-10 02:26:45,744 - root - INFO - step: 58440 loss: 16.1636 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.6883 +[titan] 2025-07-10 02:26:45,745 - root - INFO - lr: 1.3134e-04 gnorm: 0.87 [12:51:45< 9:08:50] +[titan] 2025-07-10 02:26:49,668 - root - INFO - step: 58445 loss: 16.5582 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 14.0174 +[titan] 2025-07-10 02:26:49,669 - root - INFO - lr: 1.3132e-04 gnorm: 0.88 [12:51:48< 9:08:46] +[titan] 2025-07-10 02:26:52,798 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:26:53,596 - root - INFO - step: 58450 loss: 16.4056 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8849 +[titan] 2025-07-10 02:26:53,596 - root - INFO - lr: 1.3130e-04 gnorm: 0.84 [12:51:52< 9:08:42] +[titan] 2025-07-10 02:26:57,519 - root - INFO - step: 58455 loss: 16.6467 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0947 +[titan] 2025-07-10 02:26:57,519 - root - INFO - lr: 1.3128e-04 gnorm: 0.82 [12:51:56< 9:08:38] +[titan] 2025-07-10 02:27:01,431 - root - INFO - step: 58460 loss: 16.6343 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0808 +[titan] 2025-07-10 02:27:01,431 - root - INFO - lr: 1.3126e-04 gnorm: 0.85 [12:52:00< 9:08:34] +[titan] 2025-07-10 02:27:05,332 - root - INFO - step: 58465 loss: 16.5396 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 14.0042 +[titan] 2025-07-10 02:27:05,332 - root - INFO - lr: 1.3124e-04 gnorm: 0.82 [12:52:04< 9:08:30] +[titan] 2025-07-10 02:27:09,248 - root - INFO - step: 58470 loss: 16.3319 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.8348 +[titan] 2025-07-10 02:27:09,248 - root - INFO - lr: 1.3122e-04 gnorm: 0.84 [12:52:08< 9:08:26] +[titan] 2025-07-10 02:27:13,150 - root - INFO - step: 58475 loss: 16.4251 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9026 +[titan] 2025-07-10 02:27:13,150 - root - INFO - lr: 1.3120e-04 gnorm: 0.82 [12:52:12< 9:08:22] +[titan] 2025-07-10 02:27:17,052 - root - INFO - step: 58480 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 02:27:17,052 - root - INFO - lr: 1.3118e-04 gnorm: 0.82 [12:52:16< 9:08:18] +[titan] 2025-07-10 02:27:20,943 - root - INFO - step: 58485 loss: 16.6007 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0487 +[titan] 2025-07-10 02:27:20,943 - root - INFO - lr: 1.3116e-04 gnorm: 0.90 [12:52:20< 9:08:14] +[titan] 2025-07-10 02:27:24,852 - root - INFO - step: 58490 loss: 16.7029 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5836 global_avg_mtp_loss: 14.1193 +[titan] 2025-07-10 02:27:24,852 - root - INFO - lr: 1.3113e-04 gnorm: 0.84 [12:52:24< 9:08:10] +[titan] 2025-07-10 02:27:28,757 - root - INFO - step: 58495 loss: 16.1473 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.6550 +[titan] 2025-07-10 02:27:28,758 - root - INFO - lr: 1.3111e-04 gnorm: 0.93 [12:52:28< 9:08:06] +[titan] 2025-07-10 02:27:31,901 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:27:32,691 - root - INFO - step: 58500 loss: 16.8730 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.6192 global_avg_mtp_loss: 14.2539 +[titan] 2025-07-10 02:27:32,691 - root - INFO - lr: 1.3109e-04 gnorm: 0.83 [12:52:31< 9:08:02] +[titan] 2025-07-10 02:27:36,586 - root - INFO - step: 58505 loss: 16.4191 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.9041 +[titan] 2025-07-10 02:27:36,586 - root - INFO - lr: 1.3107e-04 gnorm: 0.90 [12:52:35< 9:07:58] +[titan] 2025-07-10 02:27:40,513 - root - INFO - step: 58510 loss: 16.6815 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1121 +[titan] 2025-07-10 02:27:40,514 - root - INFO - lr: 1.3105e-04 gnorm: 0.81 [12:52:39< 9:07:54] +[titan] 2025-07-10 02:27:44,436 - root - INFO - step: 58515 loss: 16.4701 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-10 02:27:44,436 - root - INFO - lr: 1.3103e-04 gnorm: 0.81 [12:52:43< 9:07:50] +[titan] 2025-07-10 02:27:48,329 - root - INFO - step: 58520 loss: 16.3615 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8483 +[titan] 2025-07-10 02:27:48,330 - root - INFO - lr: 1.3101e-04 gnorm: 0.87 [12:52:47< 9:07:46] +[titan] 2025-07-10 02:27:52,250 - root - INFO - step: 58525 loss: 16.5538 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5536 global_avg_mtp_loss: 14.0002 +[titan] 2025-07-10 02:27:52,250 - root - INFO - lr: 1.3099e-04 gnorm: 0.83 [12:52:51< 9:07:42] +[titan] 2025-07-10 02:27:56,142 - root - INFO - step: 58530 loss: 16.3029 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8072 +[titan] 2025-07-10 02:27:56,142 - root - INFO - lr: 1.3097e-04 gnorm: 0.90 [12:52:55< 9:07:38] +[titan] 2025-07-10 02:28:00,038 - root - INFO - step: 58535 loss: 16.6661 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.5572 global_avg_mtp_loss: 14.1089 +[titan] 2025-07-10 02:28:00,038 - root - INFO - lr: 1.3095e-04 gnorm: 0.92 [12:52:59< 9:07:34] +[titan] 2025-07-10 02:28:03,952 - root - INFO - step: 58540 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5244 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-10 02:28:03,952 - root - INFO - lr: 1.3093e-04 gnorm: 0.88 [12:53:03< 9:07:30] +[titan] 2025-07-10 02:28:07,873 - root - INFO - step: 58545 loss: 16.3918 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.8781 +[titan] 2025-07-10 02:28:07,873 - root - INFO - lr: 1.3091e-04 gnorm: 0.79 [12:53:07< 9:07:26] +[titan] 2025-07-10 02:28:10,979 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:28:11,776 - root - INFO - step: 58550 loss: 16.6800 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5687 global_avg_mtp_loss: 14.1113 +[titan] 2025-07-10 02:28:11,776 - root - INFO - lr: 1.3089e-04 gnorm: 0.84 [12:53:11< 9:07:22] +[titan] 2025-07-10 02:28:15,731 - root - INFO - step: 58555 loss: 16.3525 memory: 44.58GiB(31.99%) tps: 82,843 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8572 +[titan] 2025-07-10 02:28:15,732 - root - INFO - lr: 1.3087e-04 gnorm: 0.83 [12:53:14< 9:07:18] +[titan] 2025-07-10 02:28:19,658 - root - INFO - step: 58560 loss: 16.5326 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9917 +[titan] 2025-07-10 02:28:19,658 - root - INFO - lr: 1.3084e-04 gnorm: 0.87 [12:53:18< 9:07:14] +[titan] 2025-07-10 02:28:23,561 - root - INFO - step: 58565 loss: 16.4729 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.9455 +[titan] 2025-07-10 02:28:23,561 - root - INFO - lr: 1.3082e-04 gnorm: 0.85 [12:53:22< 9:07:10] +[titan] 2025-07-10 02:28:27,496 - root - INFO - step: 58570 loss: 16.5939 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-10 02:28:27,496 - root - INFO - lr: 1.3080e-04 gnorm: 0.90 [12:53:26< 9:07:06] +[titan] 2025-07-10 02:28:31,408 - root - INFO - step: 58575 loss: 16.0781 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6151 +[titan] 2025-07-10 02:28:31,408 - root - INFO - lr: 1.3078e-04 gnorm: 0.93 [12:53:30< 9:07:02] +[titan] 2025-07-10 02:28:35,343 - root - INFO - step: 58580 loss: 16.2987 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.8151 +[titan] 2025-07-10 02:28:35,343 - root - INFO - lr: 1.3076e-04 gnorm: 0.86 [12:53:34< 9:06:58] +[titan] 2025-07-10 02:28:39,289 - root - INFO - step: 58585 loss: 16.5932 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0350 +[titan] 2025-07-10 02:28:39,289 - root - INFO - lr: 1.3074e-04 gnorm: 0.91 [12:53:38< 9:06:54] +[titan] 2025-07-10 02:28:43,212 - root - INFO - step: 58590 loss: 16.4069 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8842 +[titan] 2025-07-10 02:28:43,213 - root - INFO - lr: 1.3072e-04 gnorm: 0.83 [12:53:42< 9:06:50] +[titan] 2025-07-10 02:28:47,122 - root - INFO - step: 58595 loss: 16.3293 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8227 +[titan] 2025-07-10 02:28:47,122 - root - INFO - lr: 1.3070e-04 gnorm: 0.84 [12:53:46< 9:06:46] +[titan] 2025-07-10 02:28:50,230 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:28:51,054 - root - INFO - step: 58600 loss: 16.4602 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9193 +[titan] 2025-07-10 02:28:51,054 - root - INFO - lr: 1.3068e-04 gnorm: 0.86 [12:53:50< 9:06:42] +[titan] 2025-07-10 02:28:54,953 - root - INFO - step: 58605 loss: 16.5797 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.0174 +[titan] 2025-07-10 02:28:54,954 - root - INFO - lr: 1.3066e-04 gnorm: 0.79 [12:53:54< 9:06:38] +[titan] 2025-07-10 02:28:58,862 - root - INFO - step: 58610 loss: 16.5328 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 13.9904 +[titan] 2025-07-10 02:28:58,863 - root - INFO - lr: 1.3064e-04 gnorm: 0.85 [12:53:58< 9:06:34] +[titan] 2025-07-10 02:29:02,798 - root - INFO - step: 58615 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.6963 +[titan] 2025-07-10 02:29:02,798 - root - INFO - lr: 1.3062e-04 gnorm: 0.83 [12:54:02< 9:06:30] +[titan] 2025-07-10 02:29:06,727 - root - INFO - step: 58620 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7005 +[titan] 2025-07-10 02:29:06,728 - root - INFO - lr: 1.3060e-04 gnorm: 0.84 [12:54:05< 9:06:26] +[titan] 2025-07-10 02:29:10,675 - root - INFO - step: 58625 loss: 16.2312 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7475 +[titan] 2025-07-10 02:29:10,675 - root - INFO - lr: 1.3058e-04 gnorm: 0.82 [12:54:09< 9:06:22] +[titan] 2025-07-10 02:29:14,642 - root - INFO - step: 58630 loss: 16.0174 memory: 44.58GiB(31.99%) tps: 82,618 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.5715 +[titan] 2025-07-10 02:29:14,642 - root - INFO - lr: 1.3055e-04 gnorm: 0.87 [12:54:13< 9:06:18] +[titan] 2025-07-10 02:29:18,524 - root - INFO - step: 58635 loss: 16.4726 memory: 44.58GiB(31.99%) tps: 84,420 tflops: 291.35 mfu: 29.46% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.9562 +[titan] 2025-07-10 02:29:18,524 - root - INFO - lr: 1.3053e-04 gnorm: 0.81 [12:54:17< 9:06:14] +[titan] 2025-07-10 02:29:22,450 - root - INFO - step: 58640 loss: 16.2623 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.7598 +[titan] 2025-07-10 02:29:22,450 - root - INFO - lr: 1.3051e-04 gnorm: 0.87 [12:54:21< 9:06:10] +[titan] 2025-07-10 02:29:26,351 - root - INFO - step: 58645 loss: 16.5715 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 14.0211 +[titan] 2025-07-10 02:29:26,351 - root - INFO - lr: 1.3049e-04 gnorm: 0.88 [12:54:25< 9:06:06] +[titan] 2025-07-10 02:29:29,466 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:29:30,256 - root - INFO - step: 58650 loss: 16.5095 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.9763 +[titan] 2025-07-10 02:29:30,256 - root - INFO - lr: 1.3047e-04 gnorm: 0.82 [12:54:29< 9:06:02] +[titan] 2025-07-10 02:29:34,181 - root - INFO - step: 58655 loss: 16.5815 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 14.0368 +[titan] 2025-07-10 02:29:34,181 - root - INFO - lr: 1.3045e-04 gnorm: 0.85 [12:54:33< 9:05:58] +[titan] 2025-07-10 02:29:38,112 - root - INFO - step: 58660 loss: 16.4910 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9522 +[titan] 2025-07-10 02:29:38,112 - root - INFO - lr: 1.3043e-04 gnorm: 0.84 [12:54:37< 9:05:54] +[titan] 2025-07-10 02:29:42,050 - root - INFO - step: 58665 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.9444 +[titan] 2025-07-10 02:29:42,050 - root - INFO - lr: 1.3041e-04 gnorm: 0.90 [12:54:41< 9:05:50] +[titan] 2025-07-10 02:29:45,958 - root - INFO - step: 58670 loss: 16.4206 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.8890 +[titan] 2025-07-10 02:29:45,959 - root - INFO - lr: 1.3039e-04 gnorm: 0.88 [12:54:45< 9:05:46] +[titan] 2025-07-10 02:29:49,872 - root - INFO - step: 58675 loss: 16.5846 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 14.0429 +[titan] 2025-07-10 02:29:49,872 - root - INFO - lr: 1.3037e-04 gnorm: 0.77 [12:54:49< 9:05:42] +[titan] 2025-07-10 02:29:53,767 - root - INFO - step: 58680 loss: 16.3812 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8723 +[titan] 2025-07-10 02:29:53,767 - root - INFO - lr: 1.3035e-04 gnorm: 0.80 [12:54:53< 9:05:38] +[titan] 2025-07-10 02:29:57,687 - root - INFO - step: 58685 loss: 16.7178 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5720 global_avg_mtp_loss: 14.1458 +[titan] 2025-07-10 02:29:57,687 - root - INFO - lr: 1.3033e-04 gnorm: 0.83 [12:54:56< 9:05:34] +[titan] 2025-07-10 02:30:01,595 - root - INFO - step: 58690 loss: 16.4095 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8951 +[titan] 2025-07-10 02:30:01,596 - root - INFO - lr: 1.3031e-04 gnorm: 0.80 [12:55:00< 9:05:30] +[titan] 2025-07-10 02:30:05,501 - root - INFO - step: 58695 loss: 16.6121 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0519 +[titan] 2025-07-10 02:30:05,501 - root - INFO - lr: 1.3029e-04 gnorm: 0.84 [12:55:04< 9:05:26] +[titan] 2025-07-10 02:30:08,628 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:30:09,424 - root - INFO - step: 58700 loss: 16.4244 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.8850 +[titan] 2025-07-10 02:30:09,425 - root - INFO - lr: 1.3026e-04 gnorm: 0.81 [12:55:08< 9:05:22] +[titan] 2025-07-10 02:30:13,361 - root - INFO - step: 58705 loss: 16.0910 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.4635 global_avg_mtp_loss: 13.6276 +[titan] 2025-07-10 02:30:13,361 - root - INFO - lr: 1.3024e-04 gnorm: 0.83 [12:55:12< 9:05:18] +[titan] 2025-07-10 02:30:17,291 - root - INFO - step: 58710 loss: 16.6828 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.1066 +[titan] 2025-07-10 02:30:17,291 - root - INFO - lr: 1.3022e-04 gnorm: 0.85 [12:55:16< 9:05:14] +[titan] 2025-07-10 02:30:21,213 - root - INFO - step: 58715 loss: 16.4141 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8972 +[titan] 2025-07-10 02:30:21,213 - root - INFO - lr: 1.3020e-04 gnorm: 0.90 [12:55:20< 9:05:10] +[titan] 2025-07-10 02:30:25,124 - root - INFO - step: 58720 loss: 16.2393 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.7426 +[titan] 2025-07-10 02:30:25,124 - root - INFO - lr: 1.3018e-04 gnorm: 0.81 [12:55:24< 9:05:06] +[titan] 2025-07-10 02:30:29,038 - root - INFO - step: 58725 loss: 16.3953 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8867 +[titan] 2025-07-10 02:30:29,038 - root - INFO - lr: 1.3016e-04 gnorm: 0.88 [12:55:28< 9:05:02] +[titan] 2025-07-10 02:30:32,954 - root - INFO - step: 58730 loss: 16.6281 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.0530 +[titan] 2025-07-10 02:30:32,954 - root - INFO - lr: 1.3014e-04 gnorm: 0.86 [12:55:32< 9:04:58] +[titan] 2025-07-10 02:30:36,934 - root - INFO - step: 58735 loss: 16.5424 memory: 44.58GiB(31.99%) tps: 82,340 tflops: 284.17 mfu: 28.73% global_avg_ntp_loss: 2.5312 global_avg_mtp_loss: 14.0112 +[titan] 2025-07-10 02:30:36,934 - root - INFO - lr: 1.3012e-04 gnorm: 0.87 [12:55:36< 9:04:54] +[titan] 2025-07-10 02:30:40,956 - root - INFO - step: 58740 loss: 16.4202 memory: 44.58GiB(31.99%) tps: 81,473 tflops: 281.18 mfu: 28.43% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8975 +[titan] 2025-07-10 02:30:40,957 - root - INFO - lr: 1.3010e-04 gnorm: 0.84 [12:55:40< 9:04:50] +[titan] 2025-07-10 02:30:44,868 - root - INFO - step: 58745 loss: 16.3806 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8673 +[titan] 2025-07-10 02:30:44,869 - root - INFO - lr: 1.3008e-04 gnorm: 0.86 [12:55:44< 9:04:46] +[titan] 2025-07-10 02:30:47,990 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:30:48,773 - root - INFO - step: 58750 loss: 16.3496 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8486 +[titan] 2025-07-10 02:30:48,773 - root - INFO - lr: 1.3006e-04 gnorm: 0.80 [12:55:48< 9:04:42] +[titan] 2025-07-10 02:30:52,701 - root - INFO - step: 58755 loss: 16.6082 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0563 +[titan] 2025-07-10 02:30:52,701 - root - INFO - lr: 1.3004e-04 gnorm: 0.81 [12:55:51< 9:04:38] +[titan] 2025-07-10 02:30:56,635 - root - INFO - step: 58760 loss: 16.3019 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.7871 +[titan] 2025-07-10 02:30:56,635 - root - INFO - lr: 1.3002e-04 gnorm: 0.84 [12:55:55< 9:04:34] +[titan] 2025-07-10 02:31:00,549 - root - INFO - step: 58765 loss: 16.2340 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7511 +[titan] 2025-07-10 02:31:00,549 - root - INFO - lr: 1.3000e-04 gnorm: 0.83 [12:55:59< 9:04:30] +[titan] 2025-07-10 02:31:04,458 - root - INFO - step: 58770 loss: 16.4363 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.9174 +[titan] 2025-07-10 02:31:04,458 - root - INFO - lr: 1.2997e-04 gnorm: 0.89 [12:56:03< 9:04:26] +[titan] 2025-07-10 02:31:08,359 - root - INFO - step: 58775 loss: 16.2233 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.7187 +[titan] 2025-07-10 02:31:08,359 - root - INFO - lr: 1.2995e-04 gnorm: 0.85 [12:56:07< 9:04:22] +[titan] 2025-07-10 02:31:12,286 - root - INFO - step: 58780 loss: 16.5387 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9945 +[titan] 2025-07-10 02:31:12,286 - root - INFO - lr: 1.2993e-04 gnorm: 0.85 [12:56:11< 9:04:18] +[titan] 2025-07-10 02:31:16,195 - root - INFO - step: 58785 loss: 16.4410 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9201 +[titan] 2025-07-10 02:31:16,196 - root - INFO - lr: 1.2991e-04 gnorm: 0.85 [12:56:15< 9:04:14] +[titan] 2025-07-10 02:31:20,121 - root - INFO - step: 58790 loss: 16.6664 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5739 global_avg_mtp_loss: 14.0924 +[titan] 2025-07-10 02:31:20,121 - root - INFO - lr: 1.2989e-04 gnorm: 0.82 [12:56:19< 9:04:10] +[titan] 2025-07-10 02:31:24,054 - root - INFO - step: 58795 loss: 16.2796 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 02:31:24,054 - root - INFO - lr: 1.2987e-04 gnorm: 0.82 [12:56:23< 9:04:06] +[titan] 2025-07-10 02:31:27,214 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:31:28,021 - root - INFO - step: 58800 loss: 16.3052 memory: 44.58GiB(31.99%) tps: 82,603 tflops: 285.08 mfu: 28.82% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.7994 +[titan] 2025-07-10 02:31:28,021 - root - INFO - lr: 1.2985e-04 gnorm: 0.83 [12:56:27< 9:04:02] +[titan] 2025-07-10 02:31:31,937 - root - INFO - step: 58805 loss: 16.7017 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5729 global_avg_mtp_loss: 14.1287 +[titan] 2025-07-10 02:31:31,938 - root - INFO - lr: 1.2983e-04 gnorm: 0.86 [12:56:31< 9:03:58] +[titan] 2025-07-10 02:31:35,842 - root - INFO - step: 58810 loss: 16.3162 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8067 +[titan] 2025-07-10 02:31:35,842 - root - INFO - lr: 1.2981e-04 gnorm: 0.87 [12:56:35< 9:03:54] +[titan] 2025-07-10 02:31:39,745 - root - INFO - step: 58815 loss: 16.2644 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7718 +[titan] 2025-07-10 02:31:39,745 - root - INFO - lr: 1.2979e-04 gnorm: 0.88 [12:56:38< 9:03:50] +[titan] 2025-07-10 02:31:43,654 - root - INFO - step: 58820 loss: 16.6673 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.1067 +[titan] 2025-07-10 02:31:43,654 - root - INFO - lr: 1.2977e-04 gnorm: 0.83 [12:56:42< 9:03:46] +[titan] 2025-07-10 02:31:47,559 - root - INFO - step: 58825 loss: 16.4912 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9562 +[titan] 2025-07-10 02:31:47,559 - root - INFO - lr: 1.2975e-04 gnorm: 0.90 [12:56:46< 9:03:42] +[titan] 2025-07-10 02:31:51,482 - root - INFO - step: 58830 loss: 16.5765 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0233 +[titan] 2025-07-10 02:31:51,483 - root - INFO - lr: 1.2973e-04 gnorm: 0.85 [12:56:50< 9:03:38] +[titan] 2025-07-10 02:31:55,400 - root - INFO - step: 58835 loss: 16.4912 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.9626 +[titan] 2025-07-10 02:31:55,400 - root - INFO - lr: 1.2971e-04 gnorm: 0.84 [12:56:54< 9:03:34] +[titan] 2025-07-10 02:31:59,312 - root - INFO - step: 58840 loss: 16.3353 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.8394 +[titan] 2025-07-10 02:31:59,312 - root - INFO - lr: 1.2969e-04 gnorm: 0.83 [12:56:58< 9:03:30] +[titan] 2025-07-10 02:32:03,250 - root - INFO - step: 58845 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 02:32:03,250 - root - INFO - lr: 1.2966e-04 gnorm: 0.84 [12:57:02< 9:03:26] +[titan] 2025-07-10 02:32:06,367 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:32:07,157 - root - INFO - step: 58850 loss: 16.5208 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5371 global_avg_mtp_loss: 13.9837 +[titan] 2025-07-10 02:32:07,157 - root - INFO - lr: 1.2964e-04 gnorm: 0.89 [12:57:06< 9:03:22] +[titan] 2025-07-10 02:32:11,084 - root - INFO - step: 58855 loss: 16.1969 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.7019 +[titan] 2025-07-10 02:32:11,085 - root - INFO - lr: 1.2962e-04 gnorm: 0.88 [12:57:10< 9:03:18] +[titan] 2025-07-10 02:32:15,000 - root - INFO - step: 58860 loss: 15.9830 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4576 global_avg_mtp_loss: 13.5255 +[titan] 2025-07-10 02:32:15,000 - root - INFO - lr: 1.2960e-04 gnorm: 0.85 [12:57:14< 9:03:14] +[titan] 2025-07-10 02:32:18,950 - root - INFO - step: 58865 loss: 16.7437 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1597 +[titan] 2025-07-10 02:32:18,951 - root - INFO - lr: 1.2958e-04 gnorm: 0.84 [12:57:18< 9:03:10] +[titan] 2025-07-10 02:32:22,906 - root - INFO - step: 58870 loss: 16.5803 memory: 44.58GiB(31.99%) tps: 82,850 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5520 global_avg_mtp_loss: 14.0283 +[titan] 2025-07-10 02:32:22,906 - root - INFO - lr: 1.2956e-04 gnorm: 0.80 [12:57:22< 9:03:06] +[titan] 2025-07-10 02:32:26,809 - root - INFO - step: 58875 loss: 16.6817 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5632 global_avg_mtp_loss: 14.1185 +[titan] 2025-07-10 02:32:26,809 - root - INFO - lr: 1.2954e-04 gnorm: 0.87 [12:57:26< 9:03:02] +[titan] 2025-07-10 02:32:30,919 - root - INFO - step: 58880 loss: 16.5483 memory: 44.58GiB(31.99%) tps: 79,734 tflops: 275.17 mfu: 27.82% global_avg_ntp_loss: 2.5453 global_avg_mtp_loss: 14.0030 +[titan] 2025-07-10 02:32:30,919 - root - INFO - lr: 1.2952e-04 gnorm: 0.85 [12:57:30< 9:02:59] +[titan] 2025-07-10 02:32:31,073 - root - INFO - Dumping profiler traces at step 58880 +[titan] 2025-07-10 02:32:31,105 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:32:35,071 - root - INFO - step: 58885 loss: 16.5077 memory: 44.58GiB(31.99%) tps: 78,920 tflops: 272.37 mfu: 27.54% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 13.9679 +[titan] 2025-07-10 02:32:35,072 - root - INFO - lr: 1.2950e-04 gnorm: 0.83 [12:57:34< 9:02:55] +[titan] 2025-07-10 02:32:39,012 - root - INFO - step: 58890 loss: 16.7537 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5891 global_avg_mtp_loss: 14.1646 +[titan] 2025-07-10 02:32:39,012 - root - INFO - lr: 1.2948e-04 gnorm: 0.96 [12:57:38< 9:02:51] +[titan] 2025-07-10 02:32:42,969 - root - INFO - step: 58895 loss: 16.3150 memory: 44.58GiB(31.99%) tps: 82,815 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.8139 +[titan] 2025-07-10 02:32:42,970 - root - INFO - lr: 1.2946e-04 gnorm: 0.91 [12:57:42< 9:02:47] +[titan] 2025-07-10 02:32:46,079 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:32:46,884 - root - INFO - step: 58900 loss: 16.6574 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5854 global_avg_mtp_loss: 14.0720 +[titan] 2025-07-10 02:32:46,884 - root - INFO - lr: 1.2944e-04 gnorm: 0.82 [12:57:46< 9:02:43] +[titan] 2025-07-10 02:32:50,804 - root - INFO - step: 58905 loss: 16.7758 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5857 global_avg_mtp_loss: 14.1901 +[titan] 2025-07-10 02:32:50,804 - root - INFO - lr: 1.2942e-04 gnorm: 0.81 [12:57:50< 9:02:39] +[titan] 2025-07-10 02:32:54,749 - root - INFO - step: 58910 loss: 16.3371 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8308 +[titan] 2025-07-10 02:32:54,749 - root - INFO - lr: 1.2940e-04 gnorm: 0.96 [12:57:53< 9:02:35] +[titan] 2025-07-10 02:32:58,675 - root - INFO - step: 58915 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9389 +[titan] 2025-07-10 02:32:58,676 - root - INFO - lr: 1.2938e-04 gnorm: 0.82 [12:57:57< 9:02:31] +[titan] 2025-07-10 02:33:02,586 - root - INFO - step: 58920 loss: 16.6185 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0646 +[titan] 2025-07-10 02:33:02,586 - root - INFO - lr: 1.2935e-04 gnorm: 0.88 [12:58:01< 9:02:27] +[titan] 2025-07-10 02:33:06,488 - root - INFO - step: 58925 loss: 16.2102 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7211 +[titan] 2025-07-10 02:33:06,488 - root - INFO - lr: 1.2933e-04 gnorm: 0.84 [12:58:05< 9:02:23] +[titan] 2025-07-10 02:33:10,390 - root - INFO - step: 58930 loss: 16.4582 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9156 +[titan] 2025-07-10 02:33:10,390 - root - INFO - lr: 1.2931e-04 gnorm: 0.91 [12:58:09< 9:02:19] +[titan] 2025-07-10 02:33:14,283 - root - INFO - step: 58935 loss: 16.4987 memory: 44.58GiB(31.99%) tps: 84,177 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9532 +[titan] 2025-07-10 02:33:14,283 - root - INFO - lr: 1.2929e-04 gnorm: 0.91 [12:58:13< 9:02:15] +[titan] 2025-07-10 02:33:18,216 - root - INFO - step: 58940 loss: 16.3573 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-10 02:33:18,217 - root - INFO - lr: 1.2927e-04 gnorm: 0.86 [12:58:17< 9:02:11] +[titan] 2025-07-10 02:33:22,123 - root - INFO - step: 58945 loss: 16.5269 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 13.9835 +[titan] 2025-07-10 02:33:22,123 - root - INFO - lr: 1.2925e-04 gnorm: 0.85 [12:58:21< 9:02:07] +[titan] 2025-07-10 02:33:25,244 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:33:26,070 - root - INFO - step: 58950 loss: 16.1098 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.6369 +[titan] 2025-07-10 02:33:26,070 - root - INFO - lr: 1.2923e-04 gnorm: 0.85 [12:58:25< 9:02:03] +[titan] 2025-07-10 02:33:29,986 - root - INFO - step: 58955 loss: 16.6993 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5650 global_avg_mtp_loss: 14.1344 +[titan] 2025-07-10 02:33:29,986 - root - INFO - lr: 1.2921e-04 gnorm: 0.86 [12:58:29< 9:01:59] +[titan] 2025-07-10 02:33:33,911 - root - INFO - step: 58960 loss: 16.4970 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9598 +[titan] 2025-07-10 02:33:33,911 - root - INFO - lr: 1.2919e-04 gnorm: 0.86 [12:58:33< 9:01:55] +[titan] 2025-07-10 02:33:37,824 - root - INFO - step: 58965 loss: 16.5769 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0182 +[titan] 2025-07-10 02:33:37,824 - root - INFO - lr: 1.2917e-04 gnorm: 0.82 [12:58:37< 9:01:51] +[titan] 2025-07-10 02:33:41,770 - root - INFO - step: 58970 loss: 16.4221 memory: 44.58GiB(31.99%) tps: 83,037 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.8958 +[titan] 2025-07-10 02:33:41,771 - root - INFO - lr: 1.2915e-04 gnorm: 0.83 [12:58:40< 9:01:47] +[titan] 2025-07-10 02:33:45,723 - root - INFO - step: 58975 loss: 16.4192 memory: 44.58GiB(31.99%) tps: 82,919 tflops: 286.17 mfu: 28.93% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9015 +[titan] 2025-07-10 02:33:45,723 - root - INFO - lr: 1.2913e-04 gnorm: 0.87 [12:58:44< 9:01:43] +[titan] 2025-07-10 02:33:49,635 - root - INFO - step: 58980 loss: 16.3174 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8155 +[titan] 2025-07-10 02:33:49,635 - root - INFO - lr: 1.2911e-04 gnorm: 0.84 [12:58:48< 9:01:39] +[titan] 2025-07-10 02:33:53,576 - root - INFO - step: 58985 loss: 16.4593 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 02:33:53,576 - root - INFO - lr: 1.2909e-04 gnorm: 0.84 [12:58:52< 9:01:35] +[titan] 2025-07-10 02:33:57,504 - root - INFO - step: 58990 loss: 16.7491 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5791 global_avg_mtp_loss: 14.1701 +[titan] 2025-07-10 02:33:57,505 - root - INFO - lr: 1.2907e-04 gnorm: 0.82 [12:58:56< 9:01:31] +[titan] 2025-07-10 02:34:01,420 - root - INFO - step: 58995 loss: 16.6027 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0483 +[titan] 2025-07-10 02:34:01,421 - root - INFO - lr: 1.2904e-04 gnorm: 0.81 [12:59:00< 9:01:27] +[titan] 2025-07-10 02:34:04,534 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:34:05,326 - root - INFO - step: 59000 loss: 16.4979 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9600 +[titan] 2025-07-10 02:34:05,326 - root - INFO - lr: 1.2902e-04 gnorm: 0.82 [12:59:04< 9:01:23] +[titan] 2025-07-10 02:34:09,238 - root - INFO - step: 59005 loss: 16.4687 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9416 +[titan] 2025-07-10 02:34:09,239 - root - INFO - lr: 1.2900e-04 gnorm: 0.83 [12:59:08< 9:01:19] +[titan] 2025-07-10 02:34:13,144 - root - INFO - step: 59010 loss: 16.7084 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5705 global_avg_mtp_loss: 14.1379 +[titan] 2025-07-10 02:34:13,144 - root - INFO - lr: 1.2898e-04 gnorm: 0.88 [12:59:12< 9:01:15] +[titan] 2025-07-10 02:34:17,093 - root - INFO - step: 59015 loss: 16.4486 memory: 44.58GiB(31.99%) tps: 82,991 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9245 +[titan] 2025-07-10 02:34:17,093 - root - INFO - lr: 1.2896e-04 gnorm: 0.86 [12:59:16< 9:01:11] +[titan] 2025-07-10 02:34:21,069 - root - INFO - step: 59020 loss: 16.5155 memory: 44.58GiB(31.99%) tps: 82,419 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9786 +[titan] 2025-07-10 02:34:21,069 - root - INFO - lr: 1.2894e-04 gnorm: 0.81 [12:59:20< 9:01:07] +[titan] 2025-07-10 02:34:25,002 - root - INFO - step: 59025 loss: 16.3025 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.7905 +[titan] 2025-07-10 02:34:25,003 - root - INFO - lr: 1.2892e-04 gnorm: 0.85 [12:59:24< 9:01:03] +[titan] 2025-07-10 02:34:28,925 - root - INFO - step: 59030 loss: 16.5431 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5501 global_avg_mtp_loss: 13.9930 +[titan] 2025-07-10 02:34:28,926 - root - INFO - lr: 1.2890e-04 gnorm: 0.90 [12:59:28< 9:00:59] +[titan] 2025-07-10 02:34:32,830 - root - INFO - step: 59035 loss: 16.2805 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.7746 +[titan] 2025-07-10 02:34:32,830 - root - INFO - lr: 1.2888e-04 gnorm: 0.88 [12:59:32< 9:00:55] +[titan] 2025-07-10 02:34:36,791 - root - INFO - step: 59040 loss: 16.2872 memory: 44.58GiB(31.99%) tps: 82,727 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7971 +[titan] 2025-07-10 02:34:36,792 - root - INFO - lr: 1.2886e-04 gnorm: 0.86 [12:59:36< 9:00:51] +[titan] 2025-07-10 02:34:40,745 - root - INFO - step: 59045 loss: 16.3320 memory: 44.58GiB(31.99%) tps: 82,886 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8283 +[titan] 2025-07-10 02:34:40,745 - root - INFO - lr: 1.2884e-04 gnorm: 0.87 [12:59:39< 9:00:47] +[titan] 2025-07-10 02:34:43,852 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:34:44,641 - root - INFO - step: 59050 loss: 16.1511 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6860 +[titan] 2025-07-10 02:34:44,642 - root - INFO - lr: 1.2882e-04 gnorm: 0.96 [12:59:43< 9:00:43] +[titan] 2025-07-10 02:34:48,552 - root - INFO - step: 59055 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 02:34:48,552 - root - INFO - lr: 1.2880e-04 gnorm: 0.85 [12:59:47< 9:00:39] +[titan] 2025-07-10 02:34:52,462 - root - INFO - step: 59060 loss: 16.4601 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9254 +[titan] 2025-07-10 02:34:52,462 - root - INFO - lr: 1.2878e-04 gnorm: 0.86 [12:59:51< 9:00:35] +[titan] 2025-07-10 02:34:56,362 - root - INFO - step: 59065 loss: 16.3466 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.8493 +[titan] 2025-07-10 02:34:56,363 - root - INFO - lr: 1.2876e-04 gnorm: 0.86 [12:59:55< 9:00:31] +[titan] 2025-07-10 02:35:00,307 - root - INFO - step: 59070 loss: 16.5616 memory: 44.58GiB(31.99%) tps: 83,069 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 14.0188 +[titan] 2025-07-10 02:35:00,308 - root - INFO - lr: 1.2874e-04 gnorm: 0.88 [12:59:59< 9:00:27] +[titan] 2025-07-10 02:35:04,243 - root - INFO - step: 59075 loss: 16.2915 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.7882 +[titan] 2025-07-10 02:35:04,244 - root - INFO - lr: 1.2871e-04 gnorm: 0.86 [13:00:03< 9:00:23] +[titan] 2025-07-10 02:35:08,162 - root - INFO - step: 59080 loss: 16.3811 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.8597 +[titan] 2025-07-10 02:35:08,162 - root - INFO - lr: 1.2869e-04 gnorm: 0.85 [13:00:07< 9:00:19] +[titan] 2025-07-10 02:35:12,114 - root - INFO - step: 59085 loss: 16.6952 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.5597 global_avg_mtp_loss: 14.1355 +[titan] 2025-07-10 02:35:12,114 - root - INFO - lr: 1.2867e-04 gnorm: 0.86 [13:00:11< 9:00:15] +[titan] 2025-07-10 02:35:16,028 - root - INFO - step: 59090 loss: 16.5031 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-10 02:35:16,028 - root - INFO - lr: 1.2865e-04 gnorm: 0.84 [13:00:15< 9:00:11] +[titan] 2025-07-10 02:35:19,952 - root - INFO - step: 59095 loss: 16.5302 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9859 +[titan] 2025-07-10 02:35:19,952 - root - INFO - lr: 1.2863e-04 gnorm: 0.85 [13:00:19< 9:00:07] +[titan] 2025-07-10 02:35:23,093 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:35:23,884 - root - INFO - step: 59100 loss: 16.3777 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-10 02:35:23,884 - root - INFO - lr: 1.2861e-04 gnorm: 0.87 [13:00:23< 9:00:03] +[titan] 2025-07-10 02:35:27,798 - root - INFO - step: 59105 loss: 16.2663 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7797 +[titan] 2025-07-10 02:35:27,799 - root - INFO - lr: 1.2859e-04 gnorm: 0.91 [13:00:27< 8:59:59] +[titan] 2025-07-10 02:35:31,738 - root - INFO - step: 59110 loss: 16.3867 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.02% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 02:35:31,739 - root - INFO - lr: 1.2857e-04 gnorm: 0.84 [13:00:30< 8:59:55] +[titan] 2025-07-10 02:35:35,734 - root - INFO - step: 59115 loss: 16.3926 memory: 44.58GiB(31.99%) tps: 82,014 tflops: 283.04 mfu: 28.62% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8795 +[titan] 2025-07-10 02:35:35,734 - root - INFO - lr: 1.2855e-04 gnorm: 0.87 [13:00:34< 8:59:51] +[titan] 2025-07-10 02:35:39,678 - root - INFO - step: 59120 loss: 16.5322 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5334 global_avg_mtp_loss: 13.9987 +[titan] 2025-07-10 02:35:39,678 - root - INFO - lr: 1.2853e-04 gnorm: 0.88 [13:00:38< 8:59:47] +[titan] 2025-07-10 02:35:43,579 - root - INFO - step: 59125 loss: 16.3351 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8278 +[titan] 2025-07-10 02:35:43,579 - root - INFO - lr: 1.2851e-04 gnorm: 0.84 [13:00:42< 8:59:43] +[titan] 2025-07-10 02:35:47,493 - root - INFO - step: 59130 loss: 16.4345 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.9134 +[titan] 2025-07-10 02:35:47,493 - root - INFO - lr: 1.2849e-04 gnorm: 0.94 [13:00:46< 8:59:39] +[titan] 2025-07-10 02:35:51,401 - root - INFO - step: 59135 loss: 16.6960 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5617 global_avg_mtp_loss: 14.1344 +[titan] 2025-07-10 02:35:51,401 - root - INFO - lr: 1.2847e-04 gnorm: 0.87 [13:00:50< 8:59:35] +[titan] 2025-07-10 02:35:55,338 - root - INFO - step: 59140 loss: 16.1450 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.6694 +[titan] 2025-07-10 02:35:55,338 - root - INFO - lr: 1.2845e-04 gnorm: 0.93 [13:00:54< 8:59:31] +[titan] 2025-07-10 02:35:59,261 - root - INFO - step: 59145 loss: 16.3802 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.8586 +[titan] 2025-07-10 02:35:59,261 - root - INFO - lr: 1.2843e-04 gnorm: 0.86 [13:00:58< 8:59:27] +[titan] 2025-07-10 02:36:02,391 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:36:03,189 - root - INFO - step: 59150 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9250 +[titan] 2025-07-10 02:36:03,190 - root - INFO - lr: 1.2841e-04 gnorm: 0.83 [13:01:02< 8:59:23] +[titan] 2025-07-10 02:36:07,106 - root - INFO - step: 59155 loss: 16.2802 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7938 +[titan] 2025-07-10 02:36:07,107 - root - INFO - lr: 1.2838e-04 gnorm: 0.84 [13:01:06< 8:59:19] +[titan] 2025-07-10 02:36:11,018 - root - INFO - step: 59160 loss: 16.5663 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 14.0175 +[titan] 2025-07-10 02:36:11,018 - root - INFO - lr: 1.2836e-04 gnorm: 0.85 [13:01:10< 8:59:15] +[titan] 2025-07-10 02:36:14,992 - root - INFO - step: 59165 loss: 16.5627 memory: 44.58GiB(31.99%) tps: 82,458 tflops: 284.57 mfu: 28.77% global_avg_ntp_loss: 2.5421 global_avg_mtp_loss: 14.0206 +[titan] 2025-07-10 02:36:14,992 - root - INFO - lr: 1.2834e-04 gnorm: 0.87 [13:01:14< 8:59:12] +[titan] 2025-07-10 02:36:18,932 - root - INFO - step: 59170 loss: 16.2108 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7296 +[titan] 2025-07-10 02:36:18,932 - root - INFO - lr: 1.2832e-04 gnorm: 0.92 [13:01:18< 8:59:08] +[titan] 2025-07-10 02:36:22,850 - root - INFO - step: 59175 loss: 16.5291 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9885 +[titan] 2025-07-10 02:36:22,850 - root - INFO - lr: 1.2830e-04 gnorm: 0.88 [13:01:22< 8:59:04] +[titan] 2025-07-10 02:36:26,773 - root - INFO - step: 59180 loss: 16.6282 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-10 02:36:26,774 - root - INFO - lr: 1.2828e-04 gnorm: 0.83 [13:01:25< 8:59:00] +[titan] 2025-07-10 02:36:30,719 - root - INFO - step: 59185 loss: 16.1795 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4562 global_avg_mtp_loss: 13.7233 +[titan] 2025-07-10 02:36:30,720 - root - INFO - lr: 1.2826e-04 gnorm: 0.92 [13:01:29< 8:58:56] +[titan] 2025-07-10 02:36:34,631 - root - INFO - step: 59190 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 14.0183 +[titan] 2025-07-10 02:36:34,631 - root - INFO - lr: 1.2824e-04 gnorm: 0.85 [13:01:33< 8:58:52] +[titan] 2025-07-10 02:36:38,559 - root - INFO - step: 59195 loss: 16.4969 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9713 +[titan] 2025-07-10 02:36:38,559 - root - INFO - lr: 1.2822e-04 gnorm: 0.85 [13:01:37< 8:58:48] +[titan] 2025-07-10 02:36:41,679 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:36:42,476 - root - INFO - step: 59200 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9269 +[titan] 2025-07-10 02:36:42,476 - root - INFO - lr: 1.2820e-04 gnorm: 0.89 [13:01:41< 8:58:44] +[titan] 2025-07-10 02:36:46,385 - root - INFO - step: 59205 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.9497 +[titan] 2025-07-10 02:36:46,385 - root - INFO - lr: 1.2818e-04 gnorm: 0.88 [13:01:45< 8:58:40] +[titan] 2025-07-10 02:36:50,293 - root - INFO - step: 59210 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8701 +[titan] 2025-07-10 02:36:50,293 - root - INFO - lr: 1.2816e-04 gnorm: 0.87 [13:01:49< 8:58:36] +[titan] 2025-07-10 02:36:54,212 - root - INFO - step: 59215 loss: 16.3941 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-10 02:36:54,212 - root - INFO - lr: 1.2814e-04 gnorm: 0.84 [13:01:53< 8:58:32] +[titan] 2025-07-10 02:36:58,139 - root - INFO - step: 59220 loss: 16.3346 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8149 +[titan] 2025-07-10 02:36:58,139 - root - INFO - lr: 1.2812e-04 gnorm: 0.83 [13:01:57< 8:58:28] +[titan] 2025-07-10 02:37:02,056 - root - INFO - step: 59225 loss: 16.6129 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5599 global_avg_mtp_loss: 14.0529 +[titan] 2025-07-10 02:37:02,056 - root - INFO - lr: 1.2810e-04 gnorm: 0.85 [13:02:01< 8:58:24] +[titan] 2025-07-10 02:37:05,982 - root - INFO - step: 59230 loss: 16.3873 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.8506 +[titan] 2025-07-10 02:37:05,982 - root - INFO - lr: 1.2808e-04 gnorm: 0.84 [13:02:05< 8:58:20] +[titan] 2025-07-10 02:37:09,892 - root - INFO - step: 59235 loss: 16.4583 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9218 +[titan] 2025-07-10 02:37:09,892 - root - INFO - lr: 1.2806e-04 gnorm: 0.90 [13:02:09< 8:58:16] +[titan] 2025-07-10 02:37:13,820 - root - INFO - step: 59240 loss: 16.1834 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.6976 +[titan] 2025-07-10 02:37:13,820 - root - INFO - lr: 1.2803e-04 gnorm: 0.82 [13:02:13< 8:58:12] +[titan] 2025-07-10 02:37:17,738 - root - INFO - step: 59245 loss: 16.5600 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-10 02:37:17,739 - root - INFO - lr: 1.2801e-04 gnorm: 0.86 [13:02:16< 8:58:08] +[titan] 2025-07-10 02:37:20,871 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:37:21,668 - root - INFO - step: 59250 loss: 16.4665 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9187 +[titan] 2025-07-10 02:37:21,668 - root - INFO - lr: 1.2799e-04 gnorm: 0.93 [13:02:20< 8:58:04] +[titan] 2025-07-10 02:37:25,614 - root - INFO - step: 59255 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8615 +[titan] 2025-07-10 02:37:25,614 - root - INFO - lr: 1.2797e-04 gnorm: 0.82 [13:02:24< 8:58:00] +[titan] 2025-07-10 02:37:29,534 - root - INFO - step: 59260 loss: 16.7940 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5893 global_avg_mtp_loss: 14.2047 +[titan] 2025-07-10 02:37:29,535 - root - INFO - lr: 1.2795e-04 gnorm: 0.86 [13:02:28< 8:57:56] +[titan] 2025-07-10 02:37:33,451 - root - INFO - step: 59265 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5679 global_avg_mtp_loss: 14.0794 +[titan] 2025-07-10 02:37:33,451 - root - INFO - lr: 1.2793e-04 gnorm: 0.88 [13:02:32< 8:57:52] +[titan] 2025-07-10 02:37:37,371 - root - INFO - step: 59270 loss: 16.2799 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 02:37:37,371 - root - INFO - lr: 1.2791e-04 gnorm: 0.87 [13:02:36< 8:57:48] +[titan] 2025-07-10 02:37:41,276 - root - INFO - step: 59275 loss: 16.6898 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5787 global_avg_mtp_loss: 14.1111 +[titan] 2025-07-10 02:37:41,276 - root - INFO - lr: 1.2789e-04 gnorm: 0.86 [13:02:40< 8:57:44] +[titan] 2025-07-10 02:37:45,191 - root - INFO - step: 59280 loss: 16.4144 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-10 02:37:45,191 - root - INFO - lr: 1.2787e-04 gnorm: 0.80 [13:02:44< 8:57:40] +[titan] 2025-07-10 02:37:49,103 - root - INFO - step: 59285 loss: 16.2079 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7093 +[titan] 2025-07-10 02:37:49,104 - root - INFO - lr: 1.2785e-04 gnorm: 0.83 [13:02:48< 8:57:36] +[titan] 2025-07-10 02:37:53,029 - root - INFO - step: 59290 loss: 16.3418 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8354 +[titan] 2025-07-10 02:37:53,029 - root - INFO - lr: 1.2783e-04 gnorm: 0.84 [13:02:52< 8:57:32] +[titan] 2025-07-10 02:37:56,958 - root - INFO - step: 59295 loss: 16.3917 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.8609 +[titan] 2025-07-10 02:37:56,958 - root - INFO - lr: 1.2781e-04 gnorm: 0.84 [13:02:56< 8:57:28] +[titan] 2025-07-10 02:38:00,072 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:38:00,860 - root - INFO - step: 59300 loss: 16.4435 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9172 +[titan] 2025-07-10 02:38:00,861 - root - INFO - lr: 1.2779e-04 gnorm: 0.86 [13:03:00< 8:57:24] +[titan] 2025-07-10 02:38:04,770 - root - INFO - step: 59305 loss: 16.4962 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 13.9443 +[titan] 2025-07-10 02:38:04,770 - root - INFO - lr: 1.2777e-04 gnorm: 0.87 [13:03:03< 8:57:20] +[titan] 2025-07-10 02:38:08,748 - root - INFO - step: 59310 loss: 16.3743 memory: 44.58GiB(31.99%) tps: 82,393 tflops: 284.35 mfu: 28.75% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-10 02:38:08,748 - root - INFO - lr: 1.2775e-04 gnorm: 0.86 [13:03:07< 8:57:16] +[titan] 2025-07-10 02:38:12,669 - root - INFO - step: 59315 loss: 16.3898 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8738 +[titan] 2025-07-10 02:38:12,669 - root - INFO - lr: 1.2773e-04 gnorm: 0.81 [13:03:11< 8:57:12] +[titan] 2025-07-10 02:38:16,577 - root - INFO - step: 59320 loss: 16.5607 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 13.9978 +[titan] 2025-07-10 02:38:16,577 - root - INFO - lr: 1.2770e-04 gnorm: 0.89 [13:03:15< 8:57:08] +[titan] 2025-07-10 02:38:20,472 - root - INFO - step: 59325 loss: 16.7879 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.6074 global_avg_mtp_loss: 14.1805 +[titan] 2025-07-10 02:38:20,472 - root - INFO - lr: 1.2768e-04 gnorm: 0.92 [13:03:19< 8:57:04] +[titan] 2025-07-10 02:38:24,377 - root - INFO - step: 59330 loss: 16.5324 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9846 +[titan] 2025-07-10 02:38:24,377 - root - INFO - lr: 1.2766e-04 gnorm: 0.87 [13:03:23< 8:57:00] +[titan] 2025-07-10 02:38:28,302 - root - INFO - step: 59335 loss: 16.6279 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5657 global_avg_mtp_loss: 14.0623 +[titan] 2025-07-10 02:38:28,302 - root - INFO - lr: 1.2764e-04 gnorm: 0.86 [13:03:27< 8:56:56] +[titan] 2025-07-10 02:38:32,250 - root - INFO - step: 59340 loss: 16.4715 memory: 44.58GiB(31.99%) tps: 83,004 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9334 +[titan] 2025-07-10 02:38:32,251 - root - INFO - lr: 1.2762e-04 gnorm: 0.85 [13:03:31< 8:56:52] +[titan] 2025-07-10 02:38:36,150 - root - INFO - step: 59345 loss: 16.4453 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9246 +[titan] 2025-07-10 02:38:36,150 - root - INFO - lr: 1.2760e-04 gnorm: 0.84 [13:03:35< 8:56:48] +[titan] 2025-07-10 02:38:39,266 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:38:40,058 - root - INFO - step: 59350 loss: 16.3145 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.8025 +[titan] 2025-07-10 02:38:40,058 - root - INFO - lr: 1.2758e-04 gnorm: 0.90 [13:03:39< 8:56:44] +[titan] 2025-07-10 02:38:44,009 - root - INFO - step: 59355 loss: 16.5143 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9770 +[titan] 2025-07-10 02:38:44,009 - root - INFO - lr: 1.2756e-04 gnorm: 0.85 [13:03:43< 8:56:40] +[titan] 2025-07-10 02:38:47,947 - root - INFO - step: 59360 loss: 16.6100 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0515 +[titan] 2025-07-10 02:38:47,947 - root - INFO - lr: 1.2754e-04 gnorm: 0.91 [13:03:47< 8:56:36] +[titan] 2025-07-10 02:38:51,884 - root - INFO - step: 59365 loss: 16.3356 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 02:38:51,885 - root - INFO - lr: 1.2752e-04 gnorm: 0.83 [13:03:51< 8:56:32] +[titan] 2025-07-10 02:38:55,803 - root - INFO - step: 59370 loss: 16.5572 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0065 +[titan] 2025-07-10 02:38:55,803 - root - INFO - lr: 1.2750e-04 gnorm: 0.85 [13:03:54< 8:56:28] +[titan] 2025-07-10 02:38:59,742 - root - INFO - step: 59375 loss: 16.4758 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9455 +[titan] 2025-07-10 02:38:59,742 - root - INFO - lr: 1.2748e-04 gnorm: 0.86 [13:03:58< 8:56:24] +[titan] 2025-07-10 02:39:03,648 - root - INFO - step: 59380 loss: 16.3872 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 02:39:03,649 - root - INFO - lr: 1.2746e-04 gnorm: 0.87 [13:04:02< 8:56:20] +[titan] 2025-07-10 02:39:07,554 - root - INFO - step: 59385 loss: 16.5071 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 13.9707 +[titan] 2025-07-10 02:39:07,554 - root - INFO - lr: 1.2744e-04 gnorm: 0.86 [13:04:06< 8:56:16] +[titan] 2025-07-10 02:39:11,531 - root - INFO - step: 59390 loss: 16.0797 memory: 44.58GiB(31.99%) tps: 82,408 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.6185 +[titan] 2025-07-10 02:39:11,531 - root - INFO - lr: 1.2742e-04 gnorm: 0.85 [13:04:10< 8:56:12] +[titan] 2025-07-10 02:39:13,249 - root - INFO - Dumping profiler traces at step 59392 +[titan] 2025-07-10 02:39:13,282 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:39:15,652 - root - INFO - step: 59395 loss: 16.2778 memory: 44.58GiB(31.99%) tps: 79,508 tflops: 274.40 mfu: 27.74% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7965 +[titan] 2025-07-10 02:39:15,653 - root - INFO - lr: 1.2740e-04 gnorm: 0.87 [13:04:14< 8:56:08] +[titan] 2025-07-10 02:39:18,778 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:39:19,576 - root - INFO - step: 59400 loss: 16.2202 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 02:39:19,577 - root - INFO - lr: 1.2738e-04 gnorm: 0.88 [13:04:18< 8:56:04] +[titan] 2025-07-10 02:39:23,504 - root - INFO - step: 59405 loss: 16.6031 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 14.0532 +[titan] 2025-07-10 02:39:23,504 - root - INFO - lr: 1.2736e-04 gnorm: 0.93 [13:04:22< 8:56:00] +[titan] 2025-07-10 02:39:27,474 - root - INFO - step: 59410 loss: 16.6182 memory: 44.58GiB(31.99%) tps: 82,551 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.5651 global_avg_mtp_loss: 14.0531 +[titan] 2025-07-10 02:39:27,474 - root - INFO - lr: 1.2733e-04 gnorm: 0.90 [13:04:26< 8:55:56] +[titan] 2025-07-10 02:39:31,408 - root - INFO - step: 59415 loss: 16.3481 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8484 +[titan] 2025-07-10 02:39:31,408 - root - INFO - lr: 1.2731e-04 gnorm: 0.88 [13:04:30< 8:55:52] +[titan] 2025-07-10 02:39:35,390 - root - INFO - step: 59420 loss: 16.2066 memory: 44.58GiB(31.99%) tps: 82,298 tflops: 284.02 mfu: 28.72% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7337 +[titan] 2025-07-10 02:39:35,390 - root - INFO - lr: 1.2729e-04 gnorm: 0.87 [13:04:34< 8:55:48] +[titan] 2025-07-10 02:39:39,307 - root - INFO - step: 59425 loss: 16.7318 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5864 global_avg_mtp_loss: 14.1455 +[titan] 2025-07-10 02:39:39,307 - root - INFO - lr: 1.2727e-04 gnorm: 0.89 [13:04:38< 8:55:44] +[titan] 2025-07-10 02:39:43,228 - root - INFO - step: 59430 loss: 16.4782 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9466 +[titan] 2025-07-10 02:39:43,228 - root - INFO - lr: 1.2725e-04 gnorm: 0.84 [13:04:42< 8:55:40] +[titan] 2025-07-10 02:39:47,149 - root - INFO - step: 59435 loss: 16.3609 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8595 +[titan] 2025-07-10 02:39:47,149 - root - INFO - lr: 1.2723e-04 gnorm: 0.91 [13:04:46< 8:55:36] +[titan] 2025-07-10 02:39:51,081 - root - INFO - step: 59440 loss: 16.2660 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 02:39:51,081 - root - INFO - lr: 1.2721e-04 gnorm: 0.87 [13:04:50< 8:55:32] +[titan] 2025-07-10 02:39:54,999 - root - INFO - step: 59445 loss: 16.0620 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6050 +[titan] 2025-07-10 02:39:55,000 - root - INFO - lr: 1.2719e-04 gnorm: 0.84 [13:04:54< 8:55:28] +[titan] 2025-07-10 02:39:58,172 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:39:58,958 - root - INFO - step: 59450 loss: 16.2684 memory: 44.58GiB(31.99%) tps: 82,780 tflops: 285.69 mfu: 28.89% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.7912 +[titan] 2025-07-10 02:39:58,958 - root - INFO - lr: 1.2717e-04 gnorm: 0.82 [13:04:58< 8:55:24] +[titan] 2025-07-10 02:40:02,869 - root - INFO - step: 59455 loss: 16.4159 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.8977 +[titan] 2025-07-10 02:40:02,869 - root - INFO - lr: 1.2715e-04 gnorm: 0.85 [13:05:02< 8:55:20] +[titan] 2025-07-10 02:40:06,778 - root - INFO - step: 59460 loss: 16.4272 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.9146 +[titan] 2025-07-10 02:40:06,778 - root - INFO - lr: 1.2713e-04 gnorm: 0.84 [13:05:05< 8:55:16] +[titan] 2025-07-10 02:40:10,681 - root - INFO - step: 59465 loss: 16.3942 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5094 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 02:40:10,681 - root - INFO - lr: 1.2711e-04 gnorm: 0.86 [13:05:09< 8:55:12] +[titan] 2025-07-10 02:40:14,589 - root - INFO - step: 59470 loss: 16.3118 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8090 +[titan] 2025-07-10 02:40:14,589 - root - INFO - lr: 1.2709e-04 gnorm: 0.85 [13:05:13< 8:55:08] +[titan] 2025-07-10 02:40:18,490 - root - INFO - step: 59475 loss: 16.3209 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8282 +[titan] 2025-07-10 02:40:18,490 - root - INFO - lr: 1.2707e-04 gnorm: 0.83 [13:05:17< 8:55:04] +[titan] 2025-07-10 02:40:22,400 - root - INFO - step: 59480 loss: 16.4408 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.9241 +[titan] 2025-07-10 02:40:22,400 - root - INFO - lr: 1.2705e-04 gnorm: 0.84 [13:05:21< 8:55:00] +[titan] 2025-07-10 02:40:26,343 - root - INFO - step: 59485 loss: 16.4422 memory: 44.58GiB(31.99%) tps: 83,108 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9036 +[titan] 2025-07-10 02:40:26,344 - root - INFO - lr: 1.2703e-04 gnorm: 0.87 [13:05:25< 8:54:57] +[titan] 2025-07-10 02:40:30,277 - root - INFO - step: 59490 loss: 16.5095 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 13.9698 +[titan] 2025-07-10 02:40:30,277 - root - INFO - lr: 1.2701e-04 gnorm: 0.86 [13:05:29< 8:54:53] +[titan] 2025-07-10 02:40:34,183 - root - INFO - step: 59495 loss: 16.2652 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7751 +[titan] 2025-07-10 02:40:34,184 - root - INFO - lr: 1.2698e-04 gnorm: 0.82 [13:05:33< 8:54:49] +[titan] 2025-07-10 02:40:37,316 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:40:38,114 - root - INFO - step: 59500 loss: 16.3577 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.5015 global_avg_mtp_loss: 13.8562 +[titan] 2025-07-10 02:40:38,115 - root - INFO - lr: 1.2696e-04 gnorm: 0.81 [13:05:37< 8:54:45] +[titan] 2025-07-10 02:40:42,079 - root - INFO - step: 59505 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 82,656 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.9195 +[titan] 2025-07-10 02:40:42,079 - root - INFO - lr: 1.2694e-04 gnorm: 0.84 [13:05:41< 8:54:41] +[titan] 2025-07-10 02:40:45,990 - root - INFO - step: 59510 loss: 16.4686 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9270 +[titan] 2025-07-10 02:40:45,990 - root - INFO - lr: 1.2692e-04 gnorm: 0.89 [13:05:45< 8:54:37] +[titan] 2025-07-10 02:40:50,011 - root - INFO - step: 59515 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 81,504 tflops: 281.28 mfu: 28.44% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.7916 +[titan] 2025-07-10 02:40:50,011 - root - INFO - lr: 1.2690e-04 gnorm: 0.83 [13:05:49< 8:54:33] +[titan] 2025-07-10 02:40:53,942 - root - INFO - step: 59520 loss: 16.7314 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1462 +[titan] 2025-07-10 02:40:53,943 - root - INFO - lr: 1.2688e-04 gnorm: 0.90 [13:05:53< 8:54:29] +[titan] 2025-07-10 02:40:57,866 - root - INFO - step: 59525 loss: 16.4906 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9583 +[titan] 2025-07-10 02:40:57,866 - root - INFO - lr: 1.2686e-04 gnorm: 0.81 [13:05:57< 8:54:25] +[titan] 2025-07-10 02:41:01,759 - root - INFO - step: 59530 loss: 16.3806 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 02:41:01,759 - root - INFO - lr: 1.2684e-04 gnorm: 0.84 [13:06:00< 8:54:21] +[titan] 2025-07-10 02:41:05,704 - root - INFO - step: 59535 loss: 16.3781 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8744 +[titan] 2025-07-10 02:41:05,704 - root - INFO - lr: 1.2682e-04 gnorm: 0.88 [13:06:04< 8:54:17] +[titan] 2025-07-10 02:41:09,630 - root - INFO - step: 59540 loss: 16.2158 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7351 +[titan] 2025-07-10 02:41:09,630 - root - INFO - lr: 1.2680e-04 gnorm: 0.86 [13:06:08< 8:54:13] +[titan] 2025-07-10 02:41:13,547 - root - INFO - step: 59545 loss: 15.9316 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4440 global_avg_mtp_loss: 13.4875 +[titan] 2025-07-10 02:41:13,548 - root - INFO - lr: 1.2678e-04 gnorm: 0.89 [13:06:12< 8:54:09] +[titan] 2025-07-10 02:41:16,662 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:41:17,473 - root - INFO - step: 59550 loss: 16.2731 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7747 +[titan] 2025-07-10 02:41:17,474 - root - INFO - lr: 1.2676e-04 gnorm: 0.90 [13:06:16< 8:54:05] +[titan] 2025-07-10 02:41:21,405 - root - INFO - step: 59555 loss: 16.3399 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-10 02:41:21,405 - root - INFO - lr: 1.2674e-04 gnorm: 0.84 [13:06:20< 8:54:01] +[titan] 2025-07-10 02:41:25,312 - root - INFO - step: 59560 loss: 16.6330 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0731 +[titan] 2025-07-10 02:41:25,313 - root - INFO - lr: 1.2672e-04 gnorm: 0.82 [13:06:24< 8:53:57] +[titan] 2025-07-10 02:41:29,248 - root - INFO - step: 59565 loss: 16.6746 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1052 +[titan] 2025-07-10 02:41:29,249 - root - INFO - lr: 1.2670e-04 gnorm: 0.80 [13:06:28< 8:53:53] +[titan] 2025-07-10 02:41:33,176 - root - INFO - step: 59570 loss: 16.4474 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9209 +[titan] 2025-07-10 02:41:33,176 - root - INFO - lr: 1.2668e-04 gnorm: 0.83 [13:06:32< 8:53:49] +[titan] 2025-07-10 02:41:37,082 - root - INFO - step: 59575 loss: 16.4320 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.9029 +[titan] 2025-07-10 02:41:37,082 - root - INFO - lr: 1.2666e-04 gnorm: 0.85 [13:06:36< 8:53:45] +[titan] 2025-07-10 02:41:41,009 - root - INFO - step: 59580 loss: 16.3185 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.8258 +[titan] 2025-07-10 02:41:41,009 - root - INFO - lr: 1.2664e-04 gnorm: 0.88 [13:06:40< 8:53:41] +[titan] 2025-07-10 02:41:44,946 - root - INFO - step: 59585 loss: 16.2974 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.7960 +[titan] 2025-07-10 02:41:44,946 - root - INFO - lr: 1.2662e-04 gnorm: 0.85 [13:06:44< 8:53:37] +[titan] 2025-07-10 02:41:48,845 - root - INFO - step: 59590 loss: 16.4149 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 02:41:48,845 - root - INFO - lr: 1.2659e-04 gnorm: 0.88 [13:06:48< 8:53:33] +[titan] 2025-07-10 02:41:52,753 - root - INFO - step: 59595 loss: 16.3457 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8412 +[titan] 2025-07-10 02:41:52,753 - root - INFO - lr: 1.2657e-04 gnorm: 0.83 [13:06:51< 8:53:29] +[titan] 2025-07-10 02:41:55,871 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:41:56,659 - root - INFO - step: 59600 loss: 16.3864 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8776 +[titan] 2025-07-10 02:41:56,659 - root - INFO - lr: 1.2655e-04 gnorm: 0.83 [13:06:55< 8:53:25] +[titan] 2025-07-10 02:42:00,555 - root - INFO - step: 59605 loss: 16.5299 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 14.0058 +[titan] 2025-07-10 02:42:00,555 - root - INFO - lr: 1.2653e-04 gnorm: 0.82 [13:06:59< 8:53:21] +[titan] 2025-07-10 02:42:04,491 - root - INFO - step: 59610 loss: 16.3462 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8267 +[titan] 2025-07-10 02:42:04,491 - root - INFO - lr: 1.2651e-04 gnorm: 0.84 [13:07:03< 8:53:17] +[titan] 2025-07-10 02:42:08,433 - root - INFO - step: 59615 loss: 16.3935 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.8779 +[titan] 2025-07-10 02:42:08,433 - root - INFO - lr: 1.2649e-04 gnorm: 0.85 [13:07:07< 8:53:13] +[titan] 2025-07-10 02:42:12,395 - root - INFO - step: 59620 loss: 16.3271 memory: 44.58GiB(31.99%) tps: 82,728 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.8387 +[titan] 2025-07-10 02:42:12,395 - root - INFO - lr: 1.2647e-04 gnorm: 0.81 [13:07:11< 8:53:09] +[titan] 2025-07-10 02:42:16,317 - root - INFO - step: 59625 loss: 16.2648 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 02:42:16,317 - root - INFO - lr: 1.2645e-04 gnorm: 0.84 [13:07:15< 8:53:05] +[titan] 2025-07-10 02:42:20,227 - root - INFO - step: 59630 loss: 16.4628 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-10 02:42:20,227 - root - INFO - lr: 1.2643e-04 gnorm: 0.87 [13:07:19< 8:53:01] +[titan] 2025-07-10 02:42:24,151 - root - INFO - step: 59635 loss: 16.3750 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-10 02:42:24,151 - root - INFO - lr: 1.2641e-04 gnorm: 0.87 [13:07:23< 8:52:57] +[titan] 2025-07-10 02:42:28,127 - root - INFO - step: 59640 loss: 16.4319 memory: 44.58GiB(31.99%) tps: 82,430 tflops: 284.48 mfu: 28.76% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9116 +[titan] 2025-07-10 02:42:28,127 - root - INFO - lr: 1.2639e-04 gnorm: 0.84 [13:07:27< 8:52:53] +[titan] 2025-07-10 02:42:32,112 - root - INFO - step: 59645 loss: 16.5634 memory: 44.58GiB(31.99%) tps: 82,241 tflops: 283.83 mfu: 28.70% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0197 +[titan] 2025-07-10 02:42:32,112 - root - INFO - lr: 1.2637e-04 gnorm: 0.85 [13:07:31< 8:52:49] +[titan] 2025-07-10 02:42:35,238 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:42:36,030 - root - INFO - step: 59650 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 02:42:36,030 - root - INFO - lr: 1.2635e-04 gnorm: 0.85 [13:07:35< 8:52:45] +[titan] 2025-07-10 02:42:39,961 - root - INFO - step: 59655 loss: 16.2398 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7549 +[titan] 2025-07-10 02:42:39,962 - root - INFO - lr: 1.2633e-04 gnorm: 0.85 [13:07:39< 8:52:41] +[titan] 2025-07-10 02:42:43,878 - root - INFO - step: 59660 loss: 16.3081 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4941 global_avg_mtp_loss: 13.8141 +[titan] 2025-07-10 02:42:43,878 - root - INFO - lr: 1.2631e-04 gnorm: 0.89 [13:07:43< 8:52:37] +[titan] 2025-07-10 02:42:47,794 - root - INFO - step: 59665 loss: 16.4254 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-10 02:42:47,795 - root - INFO - lr: 1.2629e-04 gnorm: 0.94 [13:07:46< 8:52:33] +[titan] 2025-07-10 02:42:51,727 - root - INFO - step: 59670 loss: 16.5404 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 13.9955 +[titan] 2025-07-10 02:42:51,727 - root - INFO - lr: 1.2627e-04 gnorm: 0.84 [13:07:50< 8:52:29] +[titan] 2025-07-10 02:42:55,652 - root - INFO - step: 59675 loss: 16.1092 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6503 +[titan] 2025-07-10 02:42:55,652 - root - INFO - lr: 1.2625e-04 gnorm: 0.99 [13:07:54< 8:52:25] +[titan] 2025-07-10 02:42:59,592 - root - INFO - step: 59680 loss: 16.4597 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9365 +[titan] 2025-07-10 02:42:59,593 - root - INFO - lr: 1.2623e-04 gnorm: 0.93 [13:07:58< 8:52:21] +[titan] 2025-07-10 02:43:03,500 - root - INFO - step: 59685 loss: 16.3972 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8786 +[titan] 2025-07-10 02:43:03,500 - root - INFO - lr: 1.2620e-04 gnorm: 0.88 [13:08:02< 8:52:17] +[titan] 2025-07-10 02:43:07,414 - root - INFO - step: 59690 loss: 16.5786 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 14.0249 +[titan] 2025-07-10 02:43:07,414 - root - INFO - lr: 1.2618e-04 gnorm: 0.91 [13:08:06< 8:52:13] +[titan] 2025-07-10 02:43:11,313 - root - INFO - step: 59695 loss: 15.9595 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4456 global_avg_mtp_loss: 13.5138 +[titan] 2025-07-10 02:43:11,313 - root - INFO - lr: 1.2616e-04 gnorm: 0.84 [13:08:10< 8:52:09] +[titan] 2025-07-10 02:43:14,471 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:43:15,262 - root - INFO - step: 59700 loss: 16.4203 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.9042 +[titan] 2025-07-10 02:43:15,262 - root - INFO - lr: 1.2614e-04 gnorm: 0.91 [13:08:14< 8:52:05] +[titan] 2025-07-10 02:43:19,158 - root - INFO - step: 59705 loss: 16.1801 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 02:43:19,158 - root - INFO - lr: 1.2612e-04 gnorm: 0.85 [13:08:18< 8:52:01] +[titan] 2025-07-10 02:43:23,060 - root - INFO - step: 59710 loss: 16.2712 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 02:43:23,060 - root - INFO - lr: 1.2610e-04 gnorm: 0.87 [13:08:22< 8:51:57] +[titan] 2025-07-10 02:43:26,987 - root - INFO - step: 59715 loss: 16.3163 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8138 +[titan] 2025-07-10 02:43:26,988 - root - INFO - lr: 1.2608e-04 gnorm: 0.85 [13:08:26< 8:51:53] +[titan] 2025-07-10 02:43:30,883 - root - INFO - step: 59720 loss: 16.3719 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.8709 +[titan] 2025-07-10 02:43:30,883 - root - INFO - lr: 1.2606e-04 gnorm: 0.85 [13:08:30< 8:51:49] +[titan] 2025-07-10 02:43:34,826 - root - INFO - step: 59725 loss: 16.4875 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.9623 +[titan] 2025-07-10 02:43:34,826 - root - INFO - lr: 1.2604e-04 gnorm: 0.80 [13:08:33< 8:51:45] +[titan] 2025-07-10 02:43:38,719 - root - INFO - step: 59730 loss: 16.5153 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9672 +[titan] 2025-07-10 02:43:38,719 - root - INFO - lr: 1.2602e-04 gnorm: 1.33 [13:08:37< 8:51:41] +[titan] 2025-07-10 02:43:42,620 - root - INFO - step: 59735 loss: 16.5095 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5267 global_avg_mtp_loss: 13.9827 +[titan] 2025-07-10 02:43:42,620 - root - INFO - lr: 1.2600e-04 gnorm: 0.84 [13:08:41< 8:51:37] +[titan] 2025-07-10 02:43:46,528 - root - INFO - step: 59740 loss: 16.4164 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5217 global_avg_mtp_loss: 13.8946 +[titan] 2025-07-10 02:43:46,528 - root - INFO - lr: 1.2598e-04 gnorm: 0.84 [13:08:45< 8:51:33] +[titan] 2025-07-10 02:43:50,459 - root - INFO - step: 59745 loss: 16.2293 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7454 +[titan] 2025-07-10 02:43:50,459 - root - INFO - lr: 1.2596e-04 gnorm: 0.83 [13:08:49< 8:51:29] +[titan] 2025-07-10 02:43:53,582 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:43:54,373 - root - INFO - step: 59750 loss: 16.4839 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9518 +[titan] 2025-07-10 02:43:54,373 - root - INFO - lr: 1.2594e-04 gnorm: 0.83 [13:08:53< 8:51:25] +[titan] 2025-07-10 02:43:58,291 - root - INFO - step: 59755 loss: 16.2283 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7399 +[titan] 2025-07-10 02:43:58,291 - root - INFO - lr: 1.2592e-04 gnorm: 0.86 [13:08:57< 8:51:21] +[titan] 2025-07-10 02:44:02,217 - root - INFO - step: 59760 loss: 16.5728 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5516 global_avg_mtp_loss: 14.0212 +[titan] 2025-07-10 02:44:02,218 - root - INFO - lr: 1.2590e-04 gnorm: 0.92 [13:09:01< 8:51:17] +[titan] 2025-07-10 02:44:06,123 - root - INFO - step: 59765 loss: 16.4694 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9284 +[titan] 2025-07-10 02:44:06,123 - root - INFO - lr: 1.2588e-04 gnorm: 0.85 [13:09:05< 8:51:13] +[titan] 2025-07-10 02:44:10,037 - root - INFO - step: 59770 loss: 16.5006 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9576 +[titan] 2025-07-10 02:44:10,037 - root - INFO - lr: 1.2586e-04 gnorm: 0.87 [13:09:09< 8:51:09] +[titan] 2025-07-10 02:44:13,951 - root - INFO - step: 59775 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.8949 +[titan] 2025-07-10 02:44:13,951 - root - INFO - lr: 1.2584e-04 gnorm: 0.93 [13:09:13< 8:51:05] +[titan] 2025-07-10 02:44:17,877 - root - INFO - step: 59780 loss: 16.6247 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 14.0619 +[titan] 2025-07-10 02:44:17,878 - root - INFO - lr: 1.2582e-04 gnorm: 0.87 [13:09:17< 8:51:01] +[titan] 2025-07-10 02:44:21,836 - root - INFO - step: 59785 loss: 16.2458 memory: 44.58GiB(31.99%) tps: 82,792 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7584 +[titan] 2025-07-10 02:44:21,836 - root - INFO - lr: 1.2579e-04 gnorm: 0.90 [13:09:20< 8:50:57] +[titan] 2025-07-10 02:44:25,768 - root - INFO - step: 59790 loss: 16.3791 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.8645 +[titan] 2025-07-10 02:44:25,768 - root - INFO - lr: 1.2577e-04 gnorm: 0.87 [13:09:24< 8:50:53] +[titan] 2025-07-10 02:44:29,671 - root - INFO - step: 59795 loss: 16.3520 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8328 +[titan] 2025-07-10 02:44:29,671 - root - INFO - lr: 1.2575e-04 gnorm: 0.85 [13:09:28< 8:50:49] +[titan] 2025-07-10 02:44:32,798 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:44:33,598 - root - INFO - step: 59800 loss: 16.5709 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5466 global_avg_mtp_loss: 14.0243 +[titan] 2025-07-10 02:44:33,599 - root - INFO - lr: 1.2573e-04 gnorm: 0.85 [13:09:32< 8:50:45] +[titan] 2025-07-10 02:44:37,577 - root - INFO - step: 59805 loss: 16.5808 memory: 44.58GiB(31.99%) tps: 82,373 tflops: 284.28 mfu: 28.74% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 14.0323 +[titan] 2025-07-10 02:44:37,577 - root - INFO - lr: 1.2571e-04 gnorm: 0.85 [13:09:36< 8:50:41] +[titan] 2025-07-10 02:44:41,479 - root - INFO - step: 59810 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.6853 +[titan] 2025-07-10 02:44:41,479 - root - INFO - lr: 1.2569e-04 gnorm: 0.84 [13:09:40< 8:50:37] +[titan] 2025-07-10 02:44:45,415 - root - INFO - step: 59815 loss: 16.4472 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5307 global_avg_mtp_loss: 13.9165 +[titan] 2025-07-10 02:44:45,415 - root - INFO - lr: 1.2567e-04 gnorm: 0.81 [13:09:44< 8:50:33] +[titan] 2025-07-10 02:44:49,335 - root - INFO - step: 59820 loss: 16.4504 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5362 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 02:44:49,335 - root - INFO - lr: 1.2565e-04 gnorm: 0.78 [13:09:48< 8:50:29] +[titan] 2025-07-10 02:44:53,221 - root - INFO - step: 59825 loss: 16.2862 memory: 44.58GiB(31.99%) tps: 84,315 tflops: 290.98 mfu: 29.42% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.7720 +[titan] 2025-07-10 02:44:53,222 - root - INFO - lr: 1.2563e-04 gnorm: 0.89 [13:09:52< 8:50:25] +[titan] 2025-07-10 02:44:57,157 - root - INFO - step: 59830 loss: 16.2390 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7515 +[titan] 2025-07-10 02:44:57,157 - root - INFO - lr: 1.2561e-04 gnorm: 0.88 [13:09:56< 8:50:21] +[titan] 2025-07-10 02:45:01,092 - root - INFO - step: 59835 loss: 16.3146 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8105 +[titan] 2025-07-10 02:45:01,092 - root - INFO - lr: 1.2559e-04 gnorm: 0.93 [13:10:00< 8:50:18] +[titan] 2025-07-10 02:45:04,994 - root - INFO - step: 59840 loss: 16.4805 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5390 global_avg_mtp_loss: 13.9415 +[titan] 2025-07-10 02:45:04,994 - root - INFO - lr: 1.2557e-04 gnorm: 0.82 [13:10:04< 8:50:14] +[titan] 2025-07-10 02:45:08,896 - root - INFO - step: 59845 loss: 16.3661 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-10 02:45:08,896 - root - INFO - lr: 1.2555e-04 gnorm: 0.83 [13:10:08< 8:50:10] +[titan] 2025-07-10 02:45:12,016 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:45:12,816 - root - INFO - step: 59850 loss: 16.2901 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 02:45:12,816 - root - INFO - lr: 1.2553e-04 gnorm: 0.88 [13:10:11< 8:50:06] +[titan] 2025-07-10 02:45:16,724 - root - INFO - step: 59855 loss: 16.4143 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.8941 +[titan] 2025-07-10 02:45:16,724 - root - INFO - lr: 1.2551e-04 gnorm: 0.80 [13:10:15< 8:50:02] +[titan] 2025-07-10 02:45:20,700 - root - INFO - step: 59860 loss: 16.5425 memory: 44.58GiB(31.99%) tps: 82,408 tflops: 284.40 mfu: 28.76% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 14.0143 +[titan] 2025-07-10 02:45:20,701 - root - INFO - lr: 1.2549e-04 gnorm: 0.88 [13:10:19< 8:49:58] +[titan] 2025-07-10 02:45:24,661 - root - INFO - step: 59865 loss: 16.3518 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.8372 +[titan] 2025-07-10 02:45:24,661 - root - INFO - lr: 1.2547e-04 gnorm: 0.94 [13:10:23< 8:49:54] +[titan] 2025-07-10 02:45:28,571 - root - INFO - step: 59870 loss: 16.2096 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.7096 +[titan] 2025-07-10 02:45:28,571 - root - INFO - lr: 1.2545e-04 gnorm: 0.86 [13:10:27< 8:49:50] +[titan] 2025-07-10 02:45:32,485 - root - INFO - step: 59875 loss: 16.3547 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8525 +[titan] 2025-07-10 02:45:32,485 - root - INFO - lr: 1.2543e-04 gnorm: 0.85 [13:10:31< 8:49:46] +[titan] 2025-07-10 02:45:36,404 - root - INFO - step: 59880 loss: 16.3292 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8250 +[titan] 2025-07-10 02:45:36,404 - root - INFO - lr: 1.2541e-04 gnorm: 0.84 [13:10:35< 8:49:42] +[titan] 2025-07-10 02:45:40,335 - root - INFO - step: 59885 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9685 +[titan] 2025-07-10 02:45:40,335 - root - INFO - lr: 1.2538e-04 gnorm: 0.87 [13:10:39< 8:49:38] +[titan] 2025-07-10 02:45:44,243 - root - INFO - step: 59890 loss: 16.4941 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9622 +[titan] 2025-07-10 02:45:44,243 - root - INFO - lr: 1.2536e-04 gnorm: 0.92 [13:10:43< 8:49:34] +[titan] 2025-07-10 02:45:48,137 - root - INFO - step: 59895 loss: 16.4232 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.9119 +[titan] 2025-07-10 02:45:48,138 - root - INFO - lr: 1.2534e-04 gnorm: 0.85 [13:10:47< 8:49:30] +[titan] 2025-07-10 02:45:51,259 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:45:52,049 - root - INFO - step: 59900 loss: 16.4366 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.9140 +[titan] 2025-07-10 02:45:52,049 - root - INFO - lr: 1.2532e-04 gnorm: 0.87 [13:10:51< 8:49:26] +[titan] 2025-07-10 02:45:55,411 - root - INFO - Dumping profiler traces at step 59904 +[titan] 2025-07-10 02:45:55,443 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:45:56,251 - root - INFO - step: 59905 loss: 16.3353 memory: 44.58GiB(31.99%) tps: 77,996 tflops: 269.18 mfu: 27.22% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.8097 +[titan] 2025-07-10 02:45:56,251 - root - INFO - lr: 1.2530e-04 gnorm: 0.85 [13:10:55< 8:49:22] +[titan] 2025-07-10 02:46:00,203 - root - INFO - step: 59910 loss: 16.4363 memory: 44.58GiB(31.99%) tps: 82,915 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.9127 +[titan] 2025-07-10 02:46:00,203 - root - INFO - lr: 1.2528e-04 gnorm: 0.84 [13:10:59< 8:49:18] +[titan] 2025-07-10 02:46:04,114 - root - INFO - step: 59915 loss: 16.5097 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.9787 +[titan] 2025-07-10 02:46:04,114 - root - INFO - lr: 1.2526e-04 gnorm: 0.83 [13:11:03< 8:49:14] +[titan] 2025-07-10 02:46:08,024 - root - INFO - step: 59920 loss: 16.5554 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5490 global_avg_mtp_loss: 14.0064 +[titan] 2025-07-10 02:46:08,024 - root - INFO - lr: 1.2524e-04 gnorm: 0.88 [13:11:07< 8:49:10] +[titan] 2025-07-10 02:46:11,954 - root - INFO - step: 59925 loss: 16.6047 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5576 global_avg_mtp_loss: 14.0471 +[titan] 2025-07-10 02:46:11,954 - root - INFO - lr: 1.2522e-04 gnorm: 0.89 [13:11:11< 8:49:06] +[titan] 2025-07-10 02:46:15,860 - root - INFO - step: 59930 loss: 16.5597 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5552 global_avg_mtp_loss: 14.0045 +[titan] 2025-07-10 02:46:15,860 - root - INFO - lr: 1.2520e-04 gnorm: 0.86 [13:11:14< 8:49:02] +[titan] 2025-07-10 02:46:19,812 - root - INFO - step: 59935 loss: 16.5202 memory: 44.58GiB(31.99%) tps: 82,930 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 13.9806 +[titan] 2025-07-10 02:46:19,812 - root - INFO - lr: 1.2518e-04 gnorm: 0.87 [13:11:18< 8:48:58] +[titan] 2025-07-10 02:46:23,742 - root - INFO - step: 59940 loss: 16.3474 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8367 +[titan] 2025-07-10 02:46:23,742 - root - INFO - lr: 1.2516e-04 gnorm: 0.86 [13:11:22< 8:48:54] +[titan] 2025-07-10 02:46:27,717 - root - INFO - step: 59945 loss: 16.4231 memory: 44.58GiB(31.99%) tps: 82,448 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.8962 +[titan] 2025-07-10 02:46:27,717 - root - INFO - lr: 1.2514e-04 gnorm: 0.87 [13:11:26< 8:48:50] +[titan] 2025-07-10 02:46:30,824 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:46:31,610 - root - INFO - step: 59950 loss: 16.5709 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-10 02:46:31,611 - root - INFO - lr: 1.2512e-04 gnorm: 0.91 [13:11:30< 8:48:46] +[titan] 2025-07-10 02:46:35,523 - root - INFO - step: 59955 loss: 16.3265 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.7984 +[titan] 2025-07-10 02:46:35,524 - root - INFO - lr: 1.2510e-04 gnorm: 0.94 [13:11:34< 8:48:42] +[titan] 2025-07-10 02:46:39,447 - root - INFO - step: 59960 loss: 16.3661 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 02:46:39,447 - root - INFO - lr: 1.2508e-04 gnorm: 0.85 [13:11:38< 8:48:38] +[titan] 2025-07-10 02:46:43,357 - root - INFO - step: 59965 loss: 16.2861 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7946 +[titan] 2025-07-10 02:46:43,358 - root - INFO - lr: 1.2506e-04 gnorm: 0.84 [13:11:42< 8:48:34] +[titan] 2025-07-10 02:46:47,309 - root - INFO - step: 59970 loss: 16.4863 memory: 44.58GiB(31.99%) tps: 82,927 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9587 +[titan] 2025-07-10 02:46:47,309 - root - INFO - lr: 1.2504e-04 gnorm: 0.89 [13:11:46< 8:48:30] +[titan] 2025-07-10 02:46:51,216 - root - INFO - step: 59975 loss: 16.2245 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7378 +[titan] 2025-07-10 02:46:51,216 - root - INFO - lr: 1.2502e-04 gnorm: 0.90 [13:11:50< 8:48:26] +[titan] 2025-07-10 02:46:55,152 - root - INFO - step: 59980 loss: 16.5456 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.06% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 14.0021 +[titan] 2025-07-10 02:46:55,152 - root - INFO - lr: 1.2500e-04 gnorm: 0.83 [13:11:54< 8:48:22] +[titan] 2025-07-10 02:46:59,094 - root - INFO - step: 59985 loss: 16.5862 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0380 +[titan] 2025-07-10 02:46:59,094 - root - INFO - lr: 1.2498e-04 gnorm: 0.86 [13:11:58< 8:48:18] +[titan] 2025-07-10 02:47:03,025 - root - INFO - step: 59990 loss: 16.2235 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.7276 +[titan] 2025-07-10 02:47:03,026 - root - INFO - lr: 1.2496e-04 gnorm: 0.82 [13:12:02< 8:48:14] +[titan] 2025-07-10 02:47:06,936 - root - INFO - step: 59995 loss: 16.4425 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9147 +[titan] 2025-07-10 02:47:06,936 - root - INFO - lr: 1.2493e-04 gnorm: 0.85 [13:12:06< 8:48:10] +[titan] 2025-07-10 02:47:10,049 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:47:10,837 - root - INFO - step: 60000 loss: 16.5080 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9675 +[titan] 2025-07-10 02:47:10,838 - root - INFO - lr: 1.2491e-04 gnorm: 0.81 [13:12:09< 8:48:06] +[titan] 2025-07-10 02:47:10,838 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 02:47:13,324 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 02:47:13,324 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.49 seconds. +[titan] 2025-07-10 02:48:11,753 - root - INFO - step: 60005 loss: 16.4558 memory: 44.58GiB(31.99%) tps: 5,379 tflops: 18.56 mfu: 1.88% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9265 +[titan] 2025-07-10 02:48:11,754 - root - INFO - lr: 1.2489e-04 gnorm: 0.84 [13:13:10< 8:48:40] +[titan] 2025-07-10 02:48:15,715 - root - INFO - step: 60010 loss: 16.6389 memory: 44.58GiB(31.99%) tps: 82,725 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.5846 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-10 02:48:15,715 - root - INFO - lr: 1.2487e-04 gnorm: 0.87 [13:13:14< 8:48:36] +[titan] 2025-07-10 02:48:19,685 - root - INFO - step: 60015 loss: 16.6237 memory: 44.58GiB(31.99%) tps: 82,539 tflops: 284.86 mfu: 28.80% global_avg_ntp_loss: 2.5748 global_avg_mtp_loss: 14.0489 +[titan] 2025-07-10 02:48:19,686 - root - INFO - lr: 1.2485e-04 gnorm: 0.82 [13:13:18< 8:48:32] +[titan] 2025-07-10 02:48:23,648 - root - INFO - step: 60020 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 82,702 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.9652 +[titan] 2025-07-10 02:48:23,648 - root - INFO - lr: 1.2483e-04 gnorm: 0.81 [13:13:22< 8:48:28] +[titan] 2025-07-10 02:48:27,573 - root - INFO - step: 60025 loss: 16.4137 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 02:48:27,573 - root - INFO - lr: 1.2481e-04 gnorm: 0.86 [13:13:26< 8:48:24] +[titan] 2025-07-10 02:48:31,504 - root - INFO - step: 60030 loss: 16.2345 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7436 +[titan] 2025-07-10 02:48:31,504 - root - INFO - lr: 1.2479e-04 gnorm: 0.82 [13:13:30< 8:48:20] +[titan] 2025-07-10 02:48:35,478 - root - INFO - step: 60035 loss: 16.4251 memory: 44.58GiB(31.99%) tps: 82,452 tflops: 284.56 mfu: 28.77% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9012 +[titan] 2025-07-10 02:48:35,479 - root - INFO - lr: 1.2477e-04 gnorm: 0.85 [13:13:34< 8:48:16] +[titan] 2025-07-10 02:48:39,533 - root - INFO - step: 60040 loss: 16.3805 memory: 44.58GiB(31.99%) tps: 80,817 tflops: 278.91 mfu: 28.20% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8657 +[titan] 2025-07-10 02:48:39,534 - root - INFO - lr: 1.2475e-04 gnorm: 0.84 [13:13:38< 8:48:12] +[titan] 2025-07-10 02:48:43,492 - root - INFO - step: 60045 loss: 16.1810 memory: 44.58GiB(31.99%) tps: 82,779 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7032 +[titan] 2025-07-10 02:48:43,492 - root - INFO - lr: 1.2473e-04 gnorm: 0.81 [13:13:42< 8:48:08] +[titan] 2025-07-10 02:48:46,645 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:48:47,437 - root - INFO - step: 60050 loss: 16.4473 memory: 44.58GiB(31.99%) tps: 83,068 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9252 +[titan] 2025-07-10 02:48:47,438 - root - INFO - lr: 1.2471e-04 gnorm: 0.85 [13:13:46< 8:48:04] +[titan] 2025-07-10 02:48:51,368 - root - INFO - step: 60055 loss: 16.3823 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8722 +[titan] 2025-07-10 02:48:51,368 - root - INFO - lr: 1.2469e-04 gnorm: 0.85 [13:13:50< 8:48:00] +[titan] 2025-07-10 02:48:55,308 - root - INFO - step: 60060 loss: 16.6122 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0480 +[titan] 2025-07-10 02:48:55,309 - root - INFO - lr: 1.2467e-04 gnorm: 0.88 [13:13:54< 8:47:56] +[titan] 2025-07-10 02:48:59,284 - root - INFO - step: 60065 loss: 16.3873 memory: 44.58GiB(31.99%) tps: 82,434 tflops: 284.49 mfu: 28.77% global_avg_ntp_loss: 2.5183 global_avg_mtp_loss: 13.8690 +[titan] 2025-07-10 02:48:59,284 - root - INFO - lr: 1.2465e-04 gnorm: 0.92 [13:13:58< 8:47:53] +[titan] 2025-07-10 02:49:03,250 - root - INFO - step: 60070 loss: 16.3880 memory: 44.58GiB(31.99%) tps: 82,626 tflops: 285.16 mfu: 28.83% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8829 +[titan] 2025-07-10 02:49:03,250 - root - INFO - lr: 1.2463e-04 gnorm: 0.92 [13:14:02< 8:47:49] +[titan] 2025-07-10 02:49:07,173 - root - INFO - step: 60075 loss: 16.3301 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8280 +[titan] 2025-07-10 02:49:07,173 - root - INFO - lr: 1.2461e-04 gnorm: 0.91 [13:14:06< 8:47:45] +[titan] 2025-07-10 02:49:11,144 - root - INFO - step: 60080 loss: 16.3464 memory: 44.58GiB(31.99%) tps: 82,527 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8298 +[titan] 2025-07-10 02:49:11,144 - root - INFO - lr: 1.2459e-04 gnorm: 0.97 [13:14:10< 8:47:41] +[titan] 2025-07-10 02:49:15,083 - root - INFO - step: 60085 loss: 16.2125 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7297 +[titan] 2025-07-10 02:49:15,084 - root - INFO - lr: 1.2457e-04 gnorm: 0.83 [13:14:14< 8:47:37] +[titan] 2025-07-10 02:49:19,014 - root - INFO - step: 60090 loss: 16.3952 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.8696 +[titan] 2025-07-10 02:49:19,014 - root - INFO - lr: 1.2455e-04 gnorm: 0.87 [13:14:18< 8:47:33] +[titan] 2025-07-10 02:49:23,021 - root - INFO - step: 60095 loss: 16.3288 memory: 44.58GiB(31.99%) tps: 81,782 tflops: 282.24 mfu: 28.54% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8343 +[titan] 2025-07-10 02:49:23,021 - root - INFO - lr: 1.2453e-04 gnorm: 0.83 [13:14:22< 8:47:29] +[titan] 2025-07-10 02:49:26,178 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:49:26,978 - root - INFO - step: 60100 loss: 16.1420 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.6673 +[titan] 2025-07-10 02:49:26,979 - root - INFO - lr: 1.2451e-04 gnorm: 0.88 [13:14:26< 8:47:25] +[titan] 2025-07-10 02:49:30,931 - root - INFO - step: 60105 loss: 16.2446 memory: 44.58GiB(31.99%) tps: 82,915 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7654 +[titan] 2025-07-10 02:49:30,931 - root - INFO - lr: 1.2448e-04 gnorm: 0.86 [13:14:30< 8:47:21] +[titan] 2025-07-10 02:49:34,866 - root - INFO - step: 60110 loss: 16.3944 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8957 +[titan] 2025-07-10 02:49:34,866 - root - INFO - lr: 1.2446e-04 gnorm: 0.82 [13:14:33< 8:47:17] +[titan] 2025-07-10 02:49:38,782 - root - INFO - step: 60115 loss: 16.3320 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5007 global_avg_mtp_loss: 13.8312 +[titan] 2025-07-10 02:49:38,782 - root - INFO - lr: 1.2444e-04 gnorm: 0.84 [13:14:37< 8:47:13] +[titan] 2025-07-10 02:49:42,706 - root - INFO - step: 60120 loss: 16.5701 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0180 +[titan] 2025-07-10 02:49:42,706 - root - INFO - lr: 1.2442e-04 gnorm: 0.85 [13:14:41< 8:47:09] +[titan] 2025-07-10 02:49:46,657 - root - INFO - step: 60125 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 82,957 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9141 +[titan] 2025-07-10 02:49:46,657 - root - INFO - lr: 1.2440e-04 gnorm: 0.86 [13:14:45< 8:47:05] +[titan] 2025-07-10 02:49:50,596 - root - INFO - step: 60130 loss: 16.4704 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 13.9300 +[titan] 2025-07-10 02:49:50,597 - root - INFO - lr: 1.2438e-04 gnorm: 0.95 [13:14:49< 8:47:01] +[titan] 2025-07-10 02:49:54,529 - root - INFO - step: 60135 loss: 16.3339 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 02:49:54,529 - root - INFO - lr: 1.2436e-04 gnorm: 0.96 [13:14:53< 8:46:57] +[titan] 2025-07-10 02:49:58,469 - root - INFO - step: 60140 loss: 16.4931 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 02:49:58,470 - root - INFO - lr: 1.2434e-04 gnorm: 0.87 [13:14:57< 8:46:53] +[titan] 2025-07-10 02:50:02,375 - root - INFO - step: 60145 loss: 16.5567 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 14.0227 +[titan] 2025-07-10 02:50:02,375 - root - INFO - lr: 1.2432e-04 gnorm: 0.88 [13:15:01< 8:46:49] +[titan] 2025-07-10 02:50:05,516 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:50:06,315 - root - INFO - step: 60150 loss: 16.4422 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9143 +[titan] 2025-07-10 02:50:06,315 - root - INFO - lr: 1.2430e-04 gnorm: 0.83 [13:15:05< 8:46:45] +[titan] 2025-07-10 02:50:10,217 - root - INFO - step: 60155 loss: 16.5701 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 14.0144 +[titan] 2025-07-10 02:50:10,217 - root - INFO - lr: 1.2428e-04 gnorm: 0.86 [13:15:09< 8:46:41] +[titan] 2025-07-10 02:50:14,130 - root - INFO - step: 60160 loss: 16.2582 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.7545 +[titan] 2025-07-10 02:50:14,131 - root - INFO - lr: 1.2426e-04 gnorm: 0.88 [13:15:13< 8:46:37] +[titan] 2025-07-10 02:50:18,052 - root - INFO - step: 60165 loss: 16.2332 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7467 +[titan] 2025-07-10 02:50:18,052 - root - INFO - lr: 1.2424e-04 gnorm: 0.88 [13:15:17< 8:46:33] +[titan] 2025-07-10 02:50:21,964 - root - INFO - step: 60170 loss: 16.5208 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9946 +[titan] 2025-07-10 02:50:21,964 - root - INFO - lr: 1.2422e-04 gnorm: 0.98 [13:15:21< 8:46:29] +[titan] 2025-07-10 02:50:25,875 - root - INFO - step: 60175 loss: 16.5022 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9649 +[titan] 2025-07-10 02:50:25,875 - root - INFO - lr: 1.2420e-04 gnorm: 0.90 [13:15:24< 8:46:25] +[titan] 2025-07-10 02:50:29,786 - root - INFO - step: 60180 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5535 global_avg_mtp_loss: 13.9845 +[titan] 2025-07-10 02:50:29,786 - root - INFO - lr: 1.2418e-04 gnorm: 0.86 [13:15:28< 8:46:21] +[titan] 2025-07-10 02:50:33,720 - root - INFO - step: 60185 loss: 16.3183 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8182 +[titan] 2025-07-10 02:50:33,720 - root - INFO - lr: 1.2416e-04 gnorm: 0.88 [13:15:32< 8:46:17] +[titan] 2025-07-10 02:50:37,681 - root - INFO - step: 60190 loss: 16.3762 memory: 44.58GiB(31.99%) tps: 82,734 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 02:50:37,681 - root - INFO - lr: 1.2414e-04 gnorm: 0.84 [13:15:36< 8:46:13] +[titan] 2025-07-10 02:50:41,599 - root - INFO - step: 60195 loss: 16.5101 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-10 02:50:41,599 - root - INFO - lr: 1.2412e-04 gnorm: 0.87 [13:15:40< 8:46:09] +[titan] 2025-07-10 02:50:44,721 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:50:45,511 - root - INFO - step: 60200 loss: 16.4423 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.9275 +[titan] 2025-07-10 02:50:45,511 - root - INFO - lr: 1.2410e-04 gnorm: 0.86 [13:15:44< 8:46:05] +[titan] 2025-07-10 02:50:49,449 - root - INFO - step: 60205 loss: 16.4090 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.8823 +[titan] 2025-07-10 02:50:49,449 - root - INFO - lr: 1.2408e-04 gnorm: 0.85 [13:15:48< 8:46:01] +[titan] 2025-07-10 02:50:53,360 - root - INFO - step: 60210 loss: 16.3564 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 02:50:53,360 - root - INFO - lr: 1.2406e-04 gnorm: 0.88 [13:15:52< 8:45:57] +[titan] 2025-07-10 02:50:57,262 - root - INFO - step: 60215 loss: 16.5769 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 14.0372 +[titan] 2025-07-10 02:50:57,263 - root - INFO - lr: 1.2404e-04 gnorm: 0.91 [13:15:56< 8:45:53] +[titan] 2025-07-10 02:51:01,173 - root - INFO - step: 60220 loss: 16.2412 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7573 +[titan] 2025-07-10 02:51:01,174 - root - INFO - lr: 1.2402e-04 gnorm: 0.91 [13:16:00< 8:45:49] +[titan] 2025-07-10 02:51:05,089 - root - INFO - step: 60225 loss: 16.4996 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 13.9579 +[titan] 2025-07-10 02:51:05,089 - root - INFO - lr: 1.2399e-04 gnorm: 0.97 [13:16:04< 8:45:45] +[titan] 2025-07-10 02:51:08,993 - root - INFO - step: 60230 loss: 16.4670 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9241 +[titan] 2025-07-10 02:51:08,993 - root - INFO - lr: 1.2397e-04 gnorm: 0.87 [13:16:08< 8:45:41] +[titan] 2025-07-10 02:51:12,952 - root - INFO - step: 60235 loss: 16.5259 memory: 44.58GiB(31.99%) tps: 82,772 tflops: 285.66 mfu: 28.88% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9938 +[titan] 2025-07-10 02:51:12,953 - root - INFO - lr: 1.2395e-04 gnorm: 0.86 [13:16:12< 8:45:37] +[titan] 2025-07-10 02:51:16,868 - root - INFO - step: 60240 loss: 16.6195 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5563 global_avg_mtp_loss: 14.0632 +[titan] 2025-07-10 02:51:16,868 - root - INFO - lr: 1.2393e-04 gnorm: 0.83 [13:16:15< 8:45:33] +[titan] 2025-07-10 02:51:20,782 - root - INFO - step: 60245 loss: 16.2120 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7286 +[titan] 2025-07-10 02:51:20,782 - root - INFO - lr: 1.2391e-04 gnorm: 0.90 [13:16:19< 8:45:29] +[titan] 2025-07-10 02:51:23,931 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:51:24,723 - root - INFO - step: 60250 loss: 16.6049 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5513 global_avg_mtp_loss: 14.0535 +[titan] 2025-07-10 02:51:24,723 - root - INFO - lr: 1.2389e-04 gnorm: 0.87 [13:16:23< 8:45:25] +[titan] 2025-07-10 02:51:28,641 - root - INFO - step: 60255 loss: 16.6635 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0910 +[titan] 2025-07-10 02:51:28,641 - root - INFO - lr: 1.2387e-04 gnorm: 0.86 [13:16:27< 8:45:21] +[titan] 2025-07-10 02:51:32,572 - root - INFO - step: 60260 loss: 16.3889 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 02:51:32,573 - root - INFO - lr: 1.2385e-04 gnorm: 0.86 [13:16:31< 8:45:17] +[titan] 2025-07-10 02:51:36,485 - root - INFO - step: 60265 loss: 16.5443 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5285 global_avg_mtp_loss: 14.0158 +[titan] 2025-07-10 02:51:36,486 - root - INFO - lr: 1.2383e-04 gnorm: 0.86 [13:16:35< 8:45:13] +[titan] 2025-07-10 02:51:40,404 - root - INFO - step: 60270 loss: 16.4442 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9196 +[titan] 2025-07-10 02:51:40,404 - root - INFO - lr: 1.2381e-04 gnorm: 0.94 [13:16:39< 8:45:09] +[titan] 2025-07-10 02:51:44,327 - root - INFO - step: 60275 loss: 16.2707 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.7663 +[titan] 2025-07-10 02:51:44,327 - root - INFO - lr: 1.2379e-04 gnorm: 0.85 [13:16:43< 8:45:05] +[titan] 2025-07-10 02:51:48,278 - root - INFO - step: 60280 loss: 16.3969 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.8695 +[titan] 2025-07-10 02:51:48,278 - root - INFO - lr: 1.2377e-04 gnorm: 0.93 [13:16:47< 8:45:01] +[titan] 2025-07-10 02:51:52,209 - root - INFO - step: 60285 loss: 16.2194 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7385 +[titan] 2025-07-10 02:51:52,209 - root - INFO - lr: 1.2375e-04 gnorm: 0.86 [13:16:51< 8:44:57] +[titan] 2025-07-10 02:51:56,123 - root - INFO - step: 60290 loss: 16.5134 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9788 +[titan] 2025-07-10 02:51:56,124 - root - INFO - lr: 1.2373e-04 gnorm: 0.86 [13:16:55< 8:44:53] +[titan] 2025-07-10 02:52:00,057 - root - INFO - step: 60295 loss: 16.7160 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5681 global_avg_mtp_loss: 14.1479 +[titan] 2025-07-10 02:52:00,057 - root - INFO - lr: 1.2371e-04 gnorm: 0.85 [13:16:59< 8:44:49] +[titan] 2025-07-10 02:52:03,183 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:52:03,974 - root - INFO - step: 60300 loss: 16.4266 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9032 +[titan] 2025-07-10 02:52:03,974 - root - INFO - lr: 1.2369e-04 gnorm: 0.88 [13:17:03< 8:44:45] +[titan] 2025-07-10 02:52:07,894 - root - INFO - step: 60305 loss: 16.5183 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9881 +[titan] 2025-07-10 02:52:07,894 - root - INFO - lr: 1.2367e-04 gnorm: 0.88 [13:17:06< 8:44:41] +[titan] 2025-07-10 02:52:11,822 - root - INFO - step: 60310 loss: 16.1910 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.6853 +[titan] 2025-07-10 02:52:11,823 - root - INFO - lr: 1.2365e-04 gnorm: 0.95 [13:17:10< 8:44:37] +[titan] 2025-07-10 02:52:15,757 - root - INFO - step: 60315 loss: 16.3057 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8078 +[titan] 2025-07-10 02:52:15,757 - root - INFO - lr: 1.2363e-04 gnorm: 0.87 [13:17:14< 8:44:33] +[titan] 2025-07-10 02:52:19,676 - root - INFO - step: 60320 loss: 16.2066 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.7277 +[titan] 2025-07-10 02:52:19,676 - root - INFO - lr: 1.2361e-04 gnorm: 0.86 [13:17:18< 8:44:29] +[titan] 2025-07-10 02:52:23,586 - root - INFO - step: 60325 loss: 16.4018 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.8803 +[titan] 2025-07-10 02:52:23,586 - root - INFO - lr: 1.2359e-04 gnorm: 0.86 [13:17:22< 8:44:25] +[titan] 2025-07-10 02:52:27,523 - root - INFO - step: 60330 loss: 16.2641 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.7672 +[titan] 2025-07-10 02:52:27,523 - root - INFO - lr: 1.2357e-04 gnorm: 0.86 [13:17:26< 8:44:21] +[titan] 2025-07-10 02:52:31,447 - root - INFO - step: 60335 loss: 16.4480 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.9290 +[titan] 2025-07-10 02:52:31,447 - root - INFO - lr: 1.2355e-04 gnorm: 0.92 [13:17:30< 8:44:17] +[titan] 2025-07-10 02:52:35,379 - root - INFO - step: 60340 loss: 16.5907 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 14.0488 +[titan] 2025-07-10 02:52:35,379 - root - INFO - lr: 1.2353e-04 gnorm: 0.90 [13:17:34< 8:44:13] +[titan] 2025-07-10 02:52:39,292 - root - INFO - step: 60345 loss: 16.3026 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5057 global_avg_mtp_loss: 13.7970 +[titan] 2025-07-10 02:52:39,292 - root - INFO - lr: 1.2351e-04 gnorm: 0.88 [13:17:38< 8:44:09] +[titan] 2025-07-10 02:52:42,413 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:52:43,200 - root - INFO - step: 60350 loss: 16.5707 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0196 +[titan] 2025-07-10 02:52:43,200 - root - INFO - lr: 1.2348e-04 gnorm: 0.91 [13:17:42< 8:44:05] +[titan] 2025-07-10 02:52:47,103 - root - INFO - step: 60355 loss: 16.2233 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.7371 +[titan] 2025-07-10 02:52:47,103 - root - INFO - lr: 1.2346e-04 gnorm: 0.89 [13:17:46< 8:44:01] +[titan] 2025-07-10 02:52:51,021 - root - INFO - step: 60360 loss: 16.5089 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9787 +[titan] 2025-07-10 02:52:51,022 - root - INFO - lr: 1.2344e-04 gnorm: 0.86 [13:17:50< 8:43:57] +[titan] 2025-07-10 02:52:54,932 - root - INFO - step: 60365 loss: 16.1159 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6470 +[titan] 2025-07-10 02:52:54,932 - root - INFO - lr: 1.2342e-04 gnorm: 0.85 [13:17:54< 8:43:53] +[titan] 2025-07-10 02:52:58,866 - root - INFO - step: 60370 loss: 16.2228 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.7251 +[titan] 2025-07-10 02:52:58,866 - root - INFO - lr: 1.2340e-04 gnorm: 0.85 [13:17:57< 8:43:49] +[titan] 2025-07-10 02:53:02,774 - root - INFO - step: 60375 loss: 16.6443 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5572 global_avg_mtp_loss: 14.0871 +[titan] 2025-07-10 02:53:02,774 - root - INFO - lr: 1.2338e-04 gnorm: 0.88 [13:18:01< 8:43:45] +[titan] 2025-07-10 02:53:06,719 - root - INFO - step: 60380 loss: 16.5087 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9744 +[titan] 2025-07-10 02:53:06,719 - root - INFO - lr: 1.2336e-04 gnorm: 0.88 [13:18:05< 8:43:41] +[titan] 2025-07-10 02:53:10,627 - root - INFO - step: 60385 loss: 16.5859 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 14.0349 +[titan] 2025-07-10 02:53:10,628 - root - INFO - lr: 1.2334e-04 gnorm: 0.90 [13:18:09< 8:43:37] +[titan] 2025-07-10 02:53:14,538 - root - INFO - step: 60390 loss: 16.6231 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5732 global_avg_mtp_loss: 14.0498 +[titan] 2025-07-10 02:53:14,539 - root - INFO - lr: 1.2332e-04 gnorm: 0.83 [13:18:13< 8:43:33] +[titan] 2025-07-10 02:53:18,498 - root - INFO - step: 60395 loss: 16.5467 memory: 44.58GiB(31.99%) tps: 82,767 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 13.9888 +[titan] 2025-07-10 02:53:18,498 - root - INFO - lr: 1.2330e-04 gnorm: 0.84 [13:18:17< 8:43:29] +[titan] 2025-07-10 02:53:21,611 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:53:22,401 - root - INFO - step: 60400 loss: 16.6975 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5756 global_avg_mtp_loss: 14.1219 +[titan] 2025-07-10 02:53:22,401 - root - INFO - lr: 1.2328e-04 gnorm: 0.83 [13:18:21< 8:43:25] +[titan] 2025-07-10 02:53:26,293 - root - INFO - step: 60405 loss: 16.3040 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.7977 +[titan] 2025-07-10 02:53:26,293 - root - INFO - lr: 1.2326e-04 gnorm: 0.87 [13:18:25< 8:43:21] +[titan] 2025-07-10 02:53:30,205 - root - INFO - step: 60410 loss: 16.2744 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.7788 +[titan] 2025-07-10 02:53:30,205 - root - INFO - lr: 1.2324e-04 gnorm: 0.85 [13:18:29< 8:43:17] +[titan] 2025-07-10 02:53:34,192 - root - INFO - step: 60415 loss: 16.2949 memory: 44.58GiB(31.99%) tps: 82,198 tflops: 283.68 mfu: 28.68% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8004 +[titan] 2025-07-10 02:53:34,192 - root - INFO - lr: 1.2322e-04 gnorm: 0.83 [13:18:33< 8:43:13] +[titan] 2025-07-10 02:53:35,132 - root - INFO - Dumping profiler traces at step 60416 +[titan] 2025-07-10 02:53:35,165 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 02:53:38,298 - root - INFO - step: 60420 loss: 16.6006 memory: 44.58GiB(31.99%) tps: 79,798 tflops: 275.40 mfu: 27.85% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0470 +[titan] 2025-07-10 02:53:38,299 - root - INFO - lr: 1.2320e-04 gnorm: 0.88 [13:18:37< 8:43:09] +[titan] 2025-07-10 02:53:42,205 - root - INFO - step: 60425 loss: 16.5543 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 13.9955 +[titan] 2025-07-10 02:53:42,205 - root - INFO - lr: 1.2318e-04 gnorm: 0.85 [13:18:41< 8:43:05] +[titan] 2025-07-10 02:53:46,125 - root - INFO - step: 60430 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 02:53:46,126 - root - INFO - lr: 1.2316e-04 gnorm: 0.98 [13:18:45< 8:43:01] +[titan] 2025-07-10 02:53:50,056 - root - INFO - step: 60435 loss: 16.6070 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0565 +[titan] 2025-07-10 02:53:50,056 - root - INFO - lr: 1.2314e-04 gnorm: 0.83 [13:18:49< 8:42:57] +[titan] 2025-07-10 02:53:53,972 - root - INFO - step: 60440 loss: 16.6277 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0696 +[titan] 2025-07-10 02:53:53,972 - root - INFO - lr: 1.2312e-04 gnorm: 0.84 [13:18:53< 8:42:53] +[titan] 2025-07-10 02:53:57,878 - root - INFO - step: 60445 loss: 16.4020 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 02:53:57,879 - root - INFO - lr: 1.2310e-04 gnorm: 0.87 [13:18:56< 8:42:49] +[titan] 2025-07-10 02:54:01,025 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:54:01,812 - root - INFO - step: 60450 loss: 16.3992 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.8750 +[titan] 2025-07-10 02:54:01,813 - root - INFO - lr: 1.2308e-04 gnorm: 0.87 [13:19:00< 8:42:45] +[titan] 2025-07-10 02:54:05,776 - root - INFO - step: 60455 loss: 16.4573 memory: 44.58GiB(31.99%) tps: 82,680 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9297 +[titan] 2025-07-10 02:54:05,776 - root - INFO - lr: 1.2306e-04 gnorm: 0.82 [13:19:04< 8:42:41] +[titan] 2025-07-10 02:54:09,706 - root - INFO - step: 60460 loss: 16.5586 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 14.0082 +[titan] 2025-07-10 02:54:09,706 - root - INFO - lr: 1.2304e-04 gnorm: 0.85 [13:19:08< 8:42:37] +[titan] 2025-07-10 02:54:13,613 - root - INFO - step: 60465 loss: 16.4449 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9212 +[titan] 2025-07-10 02:54:13,613 - root - INFO - lr: 1.2302e-04 gnorm: 0.90 [13:19:12< 8:42:33] +[titan] 2025-07-10 02:54:17,514 - root - INFO - step: 60470 loss: 16.4503 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.9268 +[titan] 2025-07-10 02:54:17,515 - root - INFO - lr: 1.2300e-04 gnorm: 0.85 [13:19:16< 8:42:29] +[titan] 2025-07-10 02:54:21,416 - root - INFO - step: 60475 loss: 16.4417 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.9271 +[titan] 2025-07-10 02:54:21,416 - root - INFO - lr: 1.2298e-04 gnorm: 0.96 [13:19:20< 8:42:25] +[titan] 2025-07-10 02:54:25,354 - root - INFO - step: 60480 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8675 +[titan] 2025-07-10 02:54:25,354 - root - INFO - lr: 1.2296e-04 gnorm: 0.88 [13:19:24< 8:42:21] +[titan] 2025-07-10 02:54:29,277 - root - INFO - step: 60485 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.7754 +[titan] 2025-07-10 02:54:29,277 - root - INFO - lr: 1.2293e-04 gnorm: 1.06 [13:19:28< 8:42:17] +[titan] 2025-07-10 02:54:33,193 - root - INFO - step: 60490 loss: 15.9402 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4455 global_avg_mtp_loss: 13.4947 +[titan] 2025-07-10 02:54:33,193 - root - INFO - lr: 1.2291e-04 gnorm: 0.98 [13:19:32< 8:42:13] +[titan] 2025-07-10 02:54:37,111 - root - INFO - step: 60495 loss: 16.5360 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9944 +[titan] 2025-07-10 02:54:37,111 - root - INFO - lr: 1.2289e-04 gnorm: 0.87 [13:19:36< 8:42:09] +[titan] 2025-07-10 02:54:40,245 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:54:41,042 - root - INFO - step: 60500 loss: 16.5862 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0341 +[titan] 2025-07-10 02:54:41,042 - root - INFO - lr: 1.2287e-04 gnorm: 0.88 [13:19:40< 8:42:05] +[titan] 2025-07-10 02:54:44,944 - root - INFO - step: 60505 loss: 16.4213 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.8942 +[titan] 2025-07-10 02:54:44,945 - root - INFO - lr: 1.2285e-04 gnorm: 0.86 [13:19:44< 8:42:01] +[titan] 2025-07-10 02:54:48,858 - root - INFO - step: 60510 loss: 16.1834 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.6954 +[titan] 2025-07-10 02:54:48,858 - root - INFO - lr: 1.2283e-04 gnorm: 0.90 [13:19:47< 8:41:57] +[titan] 2025-07-10 02:54:52,763 - root - INFO - step: 60515 loss: 16.5708 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0187 +[titan] 2025-07-10 02:54:52,763 - root - INFO - lr: 1.2281e-04 gnorm: 0.81 [13:19:51< 8:41:53] +[titan] 2025-07-10 02:54:56,668 - root - INFO - step: 60520 loss: 16.4527 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9213 +[titan] 2025-07-10 02:54:56,669 - root - INFO - lr: 1.2279e-04 gnorm: 0.85 [13:19:55< 8:41:49] +[titan] 2025-07-10 02:55:00,594 - root - INFO - step: 60525 loss: 16.5229 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9849 +[titan] 2025-07-10 02:55:00,594 - root - INFO - lr: 1.2277e-04 gnorm: 0.85 [13:19:59< 8:41:45] +[titan] 2025-07-10 02:55:04,501 - root - INFO - step: 60530 loss: 16.4530 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9185 +[titan] 2025-07-10 02:55:04,501 - root - INFO - lr: 1.2275e-04 gnorm: 0.89 [13:20:03< 8:41:41] +[titan] 2025-07-10 02:55:08,415 - root - INFO - step: 60535 loss: 16.2336 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.7405 +[titan] 2025-07-10 02:55:08,415 - root - INFO - lr: 1.2273e-04 gnorm: 0.85 [13:20:07< 8:41:37] +[titan] 2025-07-10 02:55:12,314 - root - INFO - step: 60540 loss: 16.3246 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 02:55:12,314 - root - INFO - lr: 1.2271e-04 gnorm: 0.87 [13:20:11< 8:41:33] +[titan] 2025-07-10 02:55:16,250 - root - INFO - step: 60545 loss: 16.6087 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5557 global_avg_mtp_loss: 14.0530 +[titan] 2025-07-10 02:55:16,250 - root - INFO - lr: 1.2269e-04 gnorm: 0.91 [13:20:15< 8:41:29] +[titan] 2025-07-10 02:55:19,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:55:20,160 - root - INFO - step: 60550 loss: 16.1113 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.6380 +[titan] 2025-07-10 02:55:20,161 - root - INFO - lr: 1.2267e-04 gnorm: 0.88 [13:20:19< 8:41:25] +[titan] 2025-07-10 02:55:24,130 - root - INFO - step: 60555 loss: 16.8101 memory: 44.58GiB(31.99%) tps: 82,548 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.6191 global_avg_mtp_loss: 14.1910 +[titan] 2025-07-10 02:55:24,131 - root - INFO - lr: 1.2265e-04 gnorm: 0.83 [13:20:23< 8:41:21] +[titan] 2025-07-10 02:55:28,071 - root - INFO - step: 60560 loss: 16.5118 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9666 +[titan] 2025-07-10 02:55:28,071 - root - INFO - lr: 1.2263e-04 gnorm: 0.84 [13:20:27< 8:41:17] +[titan] 2025-07-10 02:55:31,980 - root - INFO - step: 60565 loss: 16.5936 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5525 global_avg_mtp_loss: 14.0411 +[titan] 2025-07-10 02:55:31,980 - root - INFO - lr: 1.2261e-04 gnorm: 0.81 [13:20:31< 8:41:13] +[titan] 2025-07-10 02:55:35,899 - root - INFO - step: 60570 loss: 16.7594 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5853 global_avg_mtp_loss: 14.1741 +[titan] 2025-07-10 02:55:35,899 - root - INFO - lr: 1.2259e-04 gnorm: 0.86 [13:20:34< 8:41:09] +[titan] 2025-07-10 02:55:39,870 - root - INFO - step: 60575 loss: 16.1757 memory: 44.58GiB(31.99%) tps: 82,518 tflops: 284.78 mfu: 28.80% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7018 +[titan] 2025-07-10 02:55:39,871 - root - INFO - lr: 1.2257e-04 gnorm: 0.91 [13:20:38< 8:41:05] +[titan] 2025-07-10 02:55:43,793 - root - INFO - step: 60580 loss: 16.3355 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8424 +[titan] 2025-07-10 02:55:43,793 - root - INFO - lr: 1.2255e-04 gnorm: 0.83 [13:20:42< 8:41:01] +[titan] 2025-07-10 02:55:47,692 - root - INFO - step: 60585 loss: 16.2092 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7364 +[titan] 2025-07-10 02:55:47,693 - root - INFO - lr: 1.2253e-04 gnorm: 0.83 [13:20:46< 8:40:57] +[titan] 2025-07-10 02:55:51,625 - root - INFO - step: 60590 loss: 16.5670 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 14.0232 +[titan] 2025-07-10 02:55:51,625 - root - INFO - lr: 1.2251e-04 gnorm: 0.87 [13:20:50< 8:40:53] +[titan] 2025-07-10 02:55:55,540 - root - INFO - step: 60595 loss: 16.3196 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8124 +[titan] 2025-07-10 02:55:55,540 - root - INFO - lr: 1.2249e-04 gnorm: 0.86 [13:20:54< 8:40:49] +[titan] 2025-07-10 02:55:58,648 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:55:59,442 - root - INFO - step: 60600 loss: 16.3276 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8374 +[titan] 2025-07-10 02:55:59,442 - root - INFO - lr: 1.2247e-04 gnorm: 1.14 [13:20:58< 8:40:45] +[titan] 2025-07-10 02:56:03,412 - root - INFO - step: 60605 loss: 16.5338 memory: 44.58GiB(31.99%) tps: 82,535 tflops: 284.84 mfu: 28.80% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 14.0014 +[titan] 2025-07-10 02:56:03,413 - root - INFO - lr: 1.2245e-04 gnorm: 0.84 [13:21:02< 8:40:42] +[titan] 2025-07-10 02:56:07,331 - root - INFO - step: 60610 loss: 16.2782 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.7741 +[titan] 2025-07-10 02:56:07,331 - root - INFO - lr: 1.2243e-04 gnorm: 0.85 [13:21:06< 8:40:38] +[titan] 2025-07-10 02:56:11,244 - root - INFO - step: 60615 loss: 16.4245 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 02:56:11,244 - root - INFO - lr: 1.2241e-04 gnorm: 0.83 [13:21:10< 8:40:34] +[titan] 2025-07-10 02:56:15,147 - root - INFO - step: 60620 loss: 15.8843 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4249 global_avg_mtp_loss: 13.4593 +[titan] 2025-07-10 02:56:15,147 - root - INFO - lr: 1.2239e-04 gnorm: 1.00 [13:21:14< 8:40:30] +[titan] 2025-07-10 02:56:19,047 - root - INFO - step: 60625 loss: 16.5190 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 13.9789 +[titan] 2025-07-10 02:56:19,047 - root - INFO - lr: 1.2237e-04 gnorm: 0.92 [13:21:18< 8:40:25] +[titan] 2025-07-10 02:56:22,962 - root - INFO - step: 60630 loss: 16.3736 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.8630 +[titan] 2025-07-10 02:56:22,963 - root - INFO - lr: 1.2235e-04 gnorm: 0.88 [13:21:22< 8:40:21] +[titan] 2025-07-10 02:56:26,864 - root - INFO - step: 60635 loss: 16.2549 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7595 +[titan] 2025-07-10 02:56:26,865 - root - INFO - lr: 1.2232e-04 gnorm: 0.87 [13:21:25< 8:40:17] +[titan] 2025-07-10 02:56:30,819 - root - INFO - step: 60640 loss: 16.1960 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.7028 +[titan] 2025-07-10 02:56:30,819 - root - INFO - lr: 1.2230e-04 gnorm: 0.97 [13:21:29< 8:40:14] +[titan] 2025-07-10 02:56:34,761 - root - INFO - step: 60645 loss: 16.2773 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.7828 +[titan] 2025-07-10 02:56:34,761 - root - INFO - lr: 1.2228e-04 gnorm: 0.84 [13:21:33< 8:40:10] +[titan] 2025-07-10 02:56:37,903 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:56:38,715 - root - INFO - step: 60650 loss: 16.3556 memory: 44.58GiB(31.99%) tps: 82,870 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.8346 +[titan] 2025-07-10 02:56:38,716 - root - INFO - lr: 1.2226e-04 gnorm: 0.85 [13:21:37< 8:40:06] +[titan] 2025-07-10 02:56:42,636 - root - INFO - step: 60655 loss: 16.4318 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-10 02:56:42,636 - root - INFO - lr: 1.2224e-04 gnorm: 0.80 [13:21:41< 8:40:02] +[titan] 2025-07-10 02:56:46,551 - root - INFO - step: 60660 loss: 16.3596 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.8586 +[titan] 2025-07-10 02:56:46,551 - root - INFO - lr: 1.2222e-04 gnorm: 0.90 [13:21:45< 8:39:58] +[titan] 2025-07-10 02:56:50,464 - root - INFO - step: 60665 loss: 16.2516 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7777 +[titan] 2025-07-10 02:56:50,464 - root - INFO - lr: 1.2220e-04 gnorm: 0.84 [13:21:49< 8:39:54] +[titan] 2025-07-10 02:56:54,395 - root - INFO - step: 60670 loss: 16.5231 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9908 +[titan] 2025-07-10 02:56:54,396 - root - INFO - lr: 1.2218e-04 gnorm: 0.86 [13:21:53< 8:39:50] +[titan] 2025-07-10 02:56:58,382 - root - INFO - step: 60675 loss: 16.3018 memory: 44.58GiB(31.99%) tps: 82,209 tflops: 283.72 mfu: 28.69% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8023 +[titan] 2025-07-10 02:56:58,382 - root - INFO - lr: 1.2216e-04 gnorm: 0.85 [13:21:57< 8:39:46] +[titan] 2025-07-10 02:57:02,309 - root - INFO - step: 60680 loss: 16.4452 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9259 +[titan] 2025-07-10 02:57:02,309 - root - INFO - lr: 1.2214e-04 gnorm: 0.86 [13:22:01< 8:39:42] +[titan] 2025-07-10 02:57:06,227 - root - INFO - step: 60685 loss: 16.2403 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7566 +[titan] 2025-07-10 02:57:06,227 - root - INFO - lr: 1.2212e-04 gnorm: 0.88 [13:22:05< 8:39:38] +[titan] 2025-07-10 02:57:10,127 - root - INFO - step: 60690 loss: 16.3257 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8141 +[titan] 2025-07-10 02:57:10,127 - root - INFO - lr: 1.2210e-04 gnorm: 0.87 [13:22:09< 8:39:34] +[titan] 2025-07-10 02:57:14,059 - root - INFO - step: 60695 loss: 16.3390 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.8238 +[titan] 2025-07-10 02:57:14,059 - root - INFO - lr: 1.2208e-04 gnorm: 0.87 [13:22:13< 8:39:30] +[titan] 2025-07-10 02:57:17,193 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:57:17,978 - root - INFO - step: 60700 loss: 16.3994 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 02:57:17,978 - root - INFO - lr: 1.2206e-04 gnorm: 0.86 [13:22:17< 8:39:26] +[titan] 2025-07-10 02:57:21,895 - root - INFO - step: 60705 loss: 16.4730 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9361 +[titan] 2025-07-10 02:57:21,895 - root - INFO - lr: 1.2204e-04 gnorm: 0.85 [13:22:20< 8:39:22] +[titan] 2025-07-10 02:57:25,849 - root - INFO - step: 60710 loss: 16.1797 memory: 44.58GiB(31.99%) tps: 82,881 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.6972 +[titan] 2025-07-10 02:57:25,849 - root - INFO - lr: 1.2202e-04 gnorm: 0.86 [13:22:24< 8:39:18] +[titan] 2025-07-10 02:57:29,803 - root - INFO - step: 60715 loss: 16.5604 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 14.0102 +[titan] 2025-07-10 02:57:29,804 - root - INFO - lr: 1.2200e-04 gnorm: 0.87 [13:22:28< 8:39:14] +[titan] 2025-07-10 02:57:33,703 - root - INFO - step: 60720 loss: 16.1639 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6998 +[titan] 2025-07-10 02:57:33,703 - root - INFO - lr: 1.2198e-04 gnorm: 0.97 [13:22:32< 8:39:10] +[titan] 2025-07-10 02:57:37,641 - root - INFO - step: 60725 loss: 16.6222 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0574 +[titan] 2025-07-10 02:57:37,641 - root - INFO - lr: 1.2196e-04 gnorm: 0.92 [13:22:36< 8:39:06] +[titan] 2025-07-10 02:57:41,557 - root - INFO - step: 60730 loss: 16.4131 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5183 global_avg_mtp_loss: 13.8949 +[titan] 2025-07-10 02:57:41,557 - root - INFO - lr: 1.2194e-04 gnorm: 0.84 [13:22:40< 8:39:02] +[titan] 2025-07-10 02:57:45,477 - root - INFO - step: 60735 loss: 16.2867 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.7955 +[titan] 2025-07-10 02:57:45,477 - root - INFO - lr: 1.2192e-04 gnorm: 0.90 [13:22:44< 8:38:58] +[titan] 2025-07-10 02:57:49,377 - root - INFO - step: 60740 loss: 16.3057 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8124 +[titan] 2025-07-10 02:57:49,377 - root - INFO - lr: 1.2190e-04 gnorm: 0.88 [13:22:48< 8:38:54] +[titan] 2025-07-10 02:57:53,354 - root - INFO - step: 60745 loss: 16.5174 memory: 44.58GiB(31.99%) tps: 82,395 tflops: 284.36 mfu: 28.75% global_avg_ntp_loss: 2.5427 global_avg_mtp_loss: 13.9748 +[titan] 2025-07-10 02:57:53,355 - root - INFO - lr: 1.2188e-04 gnorm: 0.84 [13:22:52< 8:38:50] +[titan] 2025-07-10 02:57:56,486 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:57:57,278 - root - INFO - step: 60750 loss: 16.2663 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 02:57:57,278 - root - INFO - lr: 1.2186e-04 gnorm: 0.86 [13:22:56< 8:38:46] +[titan] 2025-07-10 02:58:01,195 - root - INFO - step: 60755 loss: 16.3920 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.8590 +[titan] 2025-07-10 02:58:01,195 - root - INFO - lr: 1.2184e-04 gnorm: 0.83 [13:23:00< 8:38:42] +[titan] 2025-07-10 02:58:05,117 - root - INFO - step: 60760 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8052 +[titan] 2025-07-10 02:58:05,117 - root - INFO - lr: 1.2182e-04 gnorm: 0.89 [13:23:04< 8:38:38] +[titan] 2025-07-10 02:58:09,014 - root - INFO - step: 60765 loss: 16.3566 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8502 +[titan] 2025-07-10 02:58:09,014 - root - INFO - lr: 1.2180e-04 gnorm: 0.85 [13:23:08< 8:38:34] +[titan] 2025-07-10 02:58:12,925 - root - INFO - step: 60770 loss: 16.2033 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.6986 +[titan] 2025-07-10 02:58:12,925 - root - INFO - lr: 1.2178e-04 gnorm: 0.93 [13:23:11< 8:38:30] +[titan] 2025-07-10 02:58:16,844 - root - INFO - step: 60775 loss: 16.4061 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8911 +[titan] 2025-07-10 02:58:16,844 - root - INFO - lr: 1.2176e-04 gnorm: 0.89 [13:23:15< 8:38:26] +[titan] 2025-07-10 02:58:20,763 - root - INFO - step: 60780 loss: 16.5085 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9671 +[titan] 2025-07-10 02:58:20,764 - root - INFO - lr: 1.2174e-04 gnorm: 0.97 [13:23:19< 8:38:22] +[titan] 2025-07-10 02:58:24,691 - root - INFO - step: 60785 loss: 16.3533 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-10 02:58:24,691 - root - INFO - lr: 1.2172e-04 gnorm: 0.87 [13:23:23< 8:38:18] +[titan] 2025-07-10 02:58:28,595 - root - INFO - step: 60790 loss: 16.1982 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7128 +[titan] 2025-07-10 02:58:28,595 - root - INFO - lr: 1.2170e-04 gnorm: 0.89 [13:23:27< 8:38:14] +[titan] 2025-07-10 02:58:32,500 - root - INFO - step: 60795 loss: 16.3715 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.8474 +[titan] 2025-07-10 02:58:32,500 - root - INFO - lr: 1.2168e-04 gnorm: 0.89 [13:23:31< 8:38:10] +[titan] 2025-07-10 02:58:35,624 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:58:36,419 - root - INFO - step: 60800 loss: 16.5755 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 14.0263 +[titan] 2025-07-10 02:58:36,419 - root - INFO - lr: 1.2165e-04 gnorm: 0.90 [13:23:35< 8:38:06] +[titan] 2025-07-10 02:58:40,376 - root - INFO - step: 60805 loss: 16.4486 memory: 44.58GiB(31.99%) tps: 82,820 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 13.9182 +[titan] 2025-07-10 02:58:40,376 - root - INFO - lr: 1.2163e-04 gnorm: 0.87 [13:23:39< 8:38:02] +[titan] 2025-07-10 02:58:44,296 - root - INFO - step: 60810 loss: 16.2197 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.7198 +[titan] 2025-07-10 02:58:44,297 - root - INFO - lr: 1.2161e-04 gnorm: 0.92 [13:23:43< 8:37:58] +[titan] 2025-07-10 02:58:48,227 - root - INFO - step: 60815 loss: 16.2091 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7219 +[titan] 2025-07-10 02:58:48,228 - root - INFO - lr: 1.2159e-04 gnorm: 0.88 [13:23:47< 8:37:54] +[titan] 2025-07-10 02:58:52,140 - root - INFO - step: 60820 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8407 +[titan] 2025-07-10 02:58:52,140 - root - INFO - lr: 1.2157e-04 gnorm: 0.86 [13:23:51< 8:37:50] +[titan] 2025-07-10 02:58:56,085 - root - INFO - step: 60825 loss: 16.0919 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6375 +[titan] 2025-07-10 02:58:56,085 - root - INFO - lr: 1.2155e-04 gnorm: 0.89 [13:23:55< 8:37:46] +[titan] 2025-07-10 02:58:59,996 - root - INFO - step: 60830 loss: 16.0904 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6317 +[titan] 2025-07-10 02:58:59,997 - root - INFO - lr: 1.2153e-04 gnorm: 0.86 [13:23:59< 8:37:42] +[titan] 2025-07-10 02:59:03,931 - root - INFO - step: 60835 loss: 16.1972 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.7323 +[titan] 2025-07-10 02:59:03,932 - root - INFO - lr: 1.2151e-04 gnorm: 0.84 [13:24:02< 8:37:38] +[titan] 2025-07-10 02:59:07,873 - root - INFO - step: 60840 loss: 16.7206 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.6109 global_avg_mtp_loss: 14.1097 +[titan] 2025-07-10 02:59:07,873 - root - INFO - lr: 1.2149e-04 gnorm: 0.93 [13:24:06< 8:37:34] +[titan] 2025-07-10 02:59:11,791 - root - INFO - step: 60845 loss: 16.5007 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9619 +[titan] 2025-07-10 02:59:11,791 - root - INFO - lr: 1.2147e-04 gnorm: 0.86 [13:24:10< 8:37:30] +[titan] 2025-07-10 02:59:14,914 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:59:15,702 - root - INFO - step: 60850 loss: 16.3090 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8044 +[titan] 2025-07-10 02:59:15,702 - root - INFO - lr: 1.2145e-04 gnorm: 0.88 [13:24:14< 8:37:26] +[titan] 2025-07-10 02:59:19,630 - root - INFO - step: 60855 loss: 16.3265 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8220 +[titan] 2025-07-10 02:59:19,630 - root - INFO - lr: 1.2143e-04 gnorm: 0.91 [13:24:18< 8:37:22] +[titan] 2025-07-10 02:59:23,552 - root - INFO - step: 60860 loss: 16.4189 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.9075 +[titan] 2025-07-10 02:59:23,552 - root - INFO - lr: 1.2141e-04 gnorm: 0.96 [13:24:22< 8:37:18] +[titan] 2025-07-10 02:59:27,468 - root - INFO - step: 60865 loss: 16.3712 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 02:59:27,468 - root - INFO - lr: 1.2139e-04 gnorm: 0.84 [13:24:26< 8:37:14] +[titan] 2025-07-10 02:59:31,411 - root - INFO - step: 60870 loss: 16.3320 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8331 +[titan] 2025-07-10 02:59:31,412 - root - INFO - lr: 1.2137e-04 gnorm: 0.84 [13:24:30< 8:37:10] +[titan] 2025-07-10 02:59:35,343 - root - INFO - step: 60875 loss: 16.6560 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5654 global_avg_mtp_loss: 14.0906 +[titan] 2025-07-10 02:59:35,343 - root - INFO - lr: 1.2135e-04 gnorm: 0.84 [13:24:34< 8:37:06] +[titan] 2025-07-10 02:59:39,267 - root - INFO - step: 60880 loss: 16.4519 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-10 02:59:39,267 - root - INFO - lr: 1.2133e-04 gnorm: 0.84 [13:24:38< 8:37:02] +[titan] 2025-07-10 02:59:43,173 - root - INFO - step: 60885 loss: 16.3769 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8806 +[titan] 2025-07-10 02:59:43,173 - root - INFO - lr: 1.2131e-04 gnorm: 0.87 [13:24:42< 8:36:58] +[titan] 2025-07-10 02:59:47,074 - root - INFO - step: 60890 loss: 16.3463 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8328 +[titan] 2025-07-10 02:59:47,074 - root - INFO - lr: 1.2129e-04 gnorm: 0.85 [13:24:46< 8:36:54] +[titan] 2025-07-10 02:59:51,034 - root - INFO - step: 60895 loss: 16.5914 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0326 +[titan] 2025-07-10 02:59:51,034 - root - INFO - lr: 1.2127e-04 gnorm: 0.85 [13:24:50< 8:36:50] +[titan] 2025-07-10 02:59:54,160 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 02:59:54,952 - root - INFO - step: 60900 loss: 16.4142 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8976 +[titan] 2025-07-10 02:59:54,952 - root - INFO - lr: 1.2125e-04 gnorm: 0.83 [13:24:53< 8:36:46] +[titan] 2025-07-10 02:59:58,860 - root - INFO - step: 60905 loss: 16.5697 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 14.0278 +[titan] 2025-07-10 02:59:58,860 - root - INFO - lr: 1.2123e-04 gnorm: 0.84 [13:24:57< 8:36:42] +[titan] 2025-07-10 03:00:02,783 - root - INFO - step: 60910 loss: 16.2464 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.7516 +[titan] 2025-07-10 03:00:02,783 - root - INFO - lr: 1.2121e-04 gnorm: 0.85 [13:25:01< 8:36:38] +[titan] 2025-07-10 03:00:06,694 - root - INFO - step: 60915 loss: 16.2098 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.7096 +[titan] 2025-07-10 03:00:06,694 - root - INFO - lr: 1.2119e-04 gnorm: 0.82 [13:25:05< 8:36:34] +[titan] 2025-07-10 03:00:10,608 - root - INFO - step: 60920 loss: 16.5479 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5684 global_avg_mtp_loss: 13.9796 +[titan] 2025-07-10 03:00:10,608 - root - INFO - lr: 1.2117e-04 gnorm: 0.94 [13:25:09< 8:36:30] +[titan] 2025-07-10 03:00:14,614 - root - INFO - step: 60925 loss: 16.1715 memory: 44.58GiB(31.99%) tps: 81,812 tflops: 282.35 mfu: 28.55% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.6975 +[titan] 2025-07-10 03:00:14,614 - root - INFO - lr: 1.2115e-04 gnorm: 0.91 [13:25:13< 8:36:26] +[titan] 2025-07-10 03:00:17,121 - root - INFO - Dumping profiler traces at step 60928 +[titan] 2025-07-10 03:00:17,163 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 03:00:18,731 - root - INFO - step: 60930 loss: 16.2116 memory: 44.58GiB(31.99%) tps: 79,580 tflops: 274.64 mfu: 27.77% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7250 +[titan] 2025-07-10 03:00:18,732 - root - INFO - lr: 1.2113e-04 gnorm: 0.88 [13:25:17< 8:36:22] +[titan] 2025-07-10 03:00:22,659 - root - INFO - step: 60935 loss: 16.3807 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8669 +[titan] 2025-07-10 03:00:22,659 - root - INFO - lr: 1.2111e-04 gnorm: 0.87 [13:25:21< 8:36:18] +[titan] 2025-07-10 03:00:26,568 - root - INFO - step: 60940 loss: 16.5883 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 14.0263 +[titan] 2025-07-10 03:00:26,568 - root - INFO - lr: 1.2109e-04 gnorm: 0.90 [13:25:25< 8:36:14] +[titan] 2025-07-10 03:00:30,465 - root - INFO - step: 60945 loss: 16.5282 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5322 global_avg_mtp_loss: 13.9960 +[titan] 2025-07-10 03:00:30,465 - root - INFO - lr: 1.2107e-04 gnorm: 0.91 [13:25:29< 8:36:10] +[titan] 2025-07-10 03:00:33,619 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:00:34,419 - root - INFO - step: 60950 loss: 16.2367 memory: 44.58GiB(31.99%) tps: 82,864 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7588 +[titan] 2025-07-10 03:00:34,420 - root - INFO - lr: 1.2105e-04 gnorm: 0.96 [13:25:33< 8:36:06] +[titan] 2025-07-10 03:00:38,313 - root - INFO - step: 60955 loss: 16.5970 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 14.0440 +[titan] 2025-07-10 03:00:38,313 - root - INFO - lr: 1.2103e-04 gnorm: 0.89 [13:25:37< 8:36:02] +[titan] 2025-07-10 03:00:42,245 - root - INFO - step: 60960 loss: 16.5402 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 14.0105 +[titan] 2025-07-10 03:00:42,245 - root - INFO - lr: 1.2101e-04 gnorm: 0.83 [13:25:41< 8:35:58] +[titan] 2025-07-10 03:00:46,158 - root - INFO - step: 60965 loss: 16.4622 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9366 +[titan] 2025-07-10 03:00:46,159 - root - INFO - lr: 1.2099e-04 gnorm: 0.86 [13:25:45< 8:35:54] +[titan] 2025-07-10 03:00:50,082 - root - INFO - step: 60970 loss: 16.5441 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5446 global_avg_mtp_loss: 13.9995 +[titan] 2025-07-10 03:00:50,082 - root - INFO - lr: 1.2097e-04 gnorm: 0.87 [13:25:49< 8:35:50] +[titan] 2025-07-10 03:00:53,980 - root - INFO - step: 60975 loss: 16.3818 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-10 03:00:53,981 - root - INFO - lr: 1.2095e-04 gnorm: 0.83 [13:25:53< 8:35:46] +[titan] 2025-07-10 03:00:57,887 - root - INFO - step: 60980 loss: 16.4083 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.8855 +[titan] 2025-07-10 03:00:57,888 - root - INFO - lr: 1.2093e-04 gnorm: 0.86 [13:25:56< 8:35:42] +[titan] 2025-07-10 03:01:01,815 - root - INFO - step: 60985 loss: 16.6180 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0660 +[titan] 2025-07-10 03:01:01,815 - root - INFO - lr: 1.2090e-04 gnorm: 0.85 [13:26:00< 8:35:38] +[titan] 2025-07-10 03:01:05,739 - root - INFO - step: 60990 loss: 16.6139 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5755 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 03:01:05,739 - root - INFO - lr: 1.2088e-04 gnorm: 0.87 [13:26:04< 8:35:34] +[titan] 2025-07-10 03:01:09,653 - root - INFO - step: 60995 loss: 16.0787 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6191 +[titan] 2025-07-10 03:01:09,653 - root - INFO - lr: 1.2086e-04 gnorm: 0.85 [13:26:08< 8:35:30] +[titan] 2025-07-10 03:01:12,778 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:01:13,568 - root - INFO - step: 61000 loss: 16.4098 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.8856 +[titan] 2025-07-10 03:01:13,568 - root - INFO - lr: 1.2084e-04 gnorm: 0.89 [13:26:12< 8:35:26] +[titan] 2025-07-10 03:01:17,472 - root - INFO - step: 61005 loss: 16.6172 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5730 global_avg_mtp_loss: 14.0442 +[titan] 2025-07-10 03:01:17,473 - root - INFO - lr: 1.2082e-04 gnorm: 0.93 [13:26:16< 8:35:22] +[titan] 2025-07-10 03:01:21,377 - root - INFO - step: 61010 loss: 16.1360 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.6721 +[titan] 2025-07-10 03:01:21,377 - root - INFO - lr: 1.2080e-04 gnorm: 0.88 [13:26:20< 8:35:18] +[titan] 2025-07-10 03:01:25,287 - root - INFO - step: 61015 loss: 16.2326 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7449 +[titan] 2025-07-10 03:01:25,287 - root - INFO - lr: 1.2078e-04 gnorm: 0.86 [13:26:24< 8:35:14] +[titan] 2025-07-10 03:01:29,210 - root - INFO - step: 61020 loss: 16.5452 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-10 03:01:29,211 - root - INFO - lr: 1.2076e-04 gnorm: 0.82 [13:26:28< 8:35:10] +[titan] 2025-07-10 03:01:33,147 - root - INFO - step: 61025 loss: 16.5396 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 03:01:33,148 - root - INFO - lr: 1.2074e-04 gnorm: 0.89 [13:26:32< 8:35:06] +[titan] 2025-07-10 03:01:37,060 - root - INFO - step: 61030 loss: 16.5123 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5392 global_avg_mtp_loss: 13.9731 +[titan] 2025-07-10 03:01:37,061 - root - INFO - lr: 1.2072e-04 gnorm: 0.91 [13:26:36< 8:35:02] +[titan] 2025-07-10 03:01:40,982 - root - INFO - step: 61035 loss: 16.6396 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5537 global_avg_mtp_loss: 14.0859 +[titan] 2025-07-10 03:01:40,982 - root - INFO - lr: 1.2070e-04 gnorm: 0.85 [13:26:40< 8:34:58] +[titan] 2025-07-10 03:01:44,892 - root - INFO - step: 61040 loss: 16.3342 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8216 +[titan] 2025-07-10 03:01:44,892 - root - INFO - lr: 1.2068e-04 gnorm: 0.87 [13:26:43< 8:34:54] +[titan] 2025-07-10 03:01:48,793 - root - INFO - step: 61045 loss: 16.4081 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8915 +[titan] 2025-07-10 03:01:48,794 - root - INFO - lr: 1.2066e-04 gnorm: 0.84 [13:26:47< 8:34:50] +[titan] 2025-07-10 03:01:51,973 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:01:52,758 - root - INFO - step: 61050 loss: 16.4927 memory: 44.58GiB(31.99%) tps: 82,651 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9661 +[titan] 2025-07-10 03:01:52,759 - root - INFO - lr: 1.2064e-04 gnorm: 0.85 [13:26:51< 8:34:46] +[titan] 2025-07-10 03:01:56,659 - root - INFO - step: 61055 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9238 +[titan] 2025-07-10 03:01:56,659 - root - INFO - lr: 1.2062e-04 gnorm: 0.83 [13:26:55< 8:34:42] +[titan] 2025-07-10 03:02:00,592 - root - INFO - step: 61060 loss: 16.4087 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8884 +[titan] 2025-07-10 03:02:00,592 - root - INFO - lr: 1.2060e-04 gnorm: 0.90 [13:26:59< 8:34:38] +[titan] 2025-07-10 03:02:04,527 - root - INFO - step: 61065 loss: 16.3516 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8342 +[titan] 2025-07-10 03:02:04,527 - root - INFO - lr: 1.2058e-04 gnorm: 0.87 [13:27:03< 8:34:34] +[titan] 2025-07-10 03:02:08,454 - root - INFO - step: 61070 loss: 16.5696 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0164 +[titan] 2025-07-10 03:02:08,454 - root - INFO - lr: 1.2056e-04 gnorm: 0.96 [13:27:07< 8:34:30] +[titan] 2025-07-10 03:02:12,381 - root - INFO - step: 61075 loss: 16.4173 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.9046 +[titan] 2025-07-10 03:02:12,382 - root - INFO - lr: 1.2054e-04 gnorm: 0.89 [13:27:11< 8:34:26] +[titan] 2025-07-10 03:02:16,350 - root - INFO - step: 61080 loss: 16.5525 memory: 44.58GiB(31.99%) tps: 82,572 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 14.0171 +[titan] 2025-07-10 03:02:16,350 - root - INFO - lr: 1.2052e-04 gnorm: 0.83 [13:27:15< 8:34:22] +[titan] 2025-07-10 03:02:20,275 - root - INFO - step: 61085 loss: 16.5251 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5360 global_avg_mtp_loss: 13.9892 +[titan] 2025-07-10 03:02:20,276 - root - INFO - lr: 1.2050e-04 gnorm: 0.87 [13:27:19< 8:34:18] +[titan] 2025-07-10 03:02:24,187 - root - INFO - step: 61090 loss: 16.4691 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9351 +[titan] 2025-07-10 03:02:24,187 - root - INFO - lr: 1.2048e-04 gnorm: 0.91 [13:27:23< 8:34:14] +[titan] 2025-07-10 03:02:28,112 - root - INFO - step: 61095 loss: 16.2904 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.8072 +[titan] 2025-07-10 03:02:28,112 - root - INFO - lr: 1.2046e-04 gnorm: 0.89 [13:27:27< 8:34:10] +[titan] 2025-07-10 03:02:31,238 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:02:32,049 - root - INFO - step: 61100 loss: 16.3535 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8538 +[titan] 2025-07-10 03:02:32,049 - root - INFO - lr: 1.2044e-04 gnorm: 0.90 [13:27:31< 8:34:06] +[titan] 2025-07-10 03:02:35,957 - root - INFO - step: 61105 loss: 16.6444 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0866 +[titan] 2025-07-10 03:02:35,957 - root - INFO - lr: 1.2042e-04 gnorm: 0.96 [13:27:34< 8:34:02] +[titan] 2025-07-10 03:02:39,883 - root - INFO - step: 61110 loss: 16.4639 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.9391 +[titan] 2025-07-10 03:02:39,883 - root - INFO - lr: 1.2040e-04 gnorm: 0.87 [13:27:38< 8:33:58] +[titan] 2025-07-10 03:02:43,810 - root - INFO - step: 61115 loss: 16.3309 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 03:02:43,810 - root - INFO - lr: 1.2038e-04 gnorm: 0.82 [13:27:42< 8:33:54] +[titan] 2025-07-10 03:02:47,728 - root - INFO - step: 61120 loss: 16.3384 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8368 +[titan] 2025-07-10 03:02:47,728 - root - INFO - lr: 1.2036e-04 gnorm: 0.85 [13:27:46< 8:33:50] +[titan] 2025-07-10 03:02:51,652 - root - INFO - step: 61125 loss: 16.3607 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8437 +[titan] 2025-07-10 03:02:51,652 - root - INFO - lr: 1.2034e-04 gnorm: 0.87 [13:27:50< 8:33:46] +[titan] 2025-07-10 03:02:55,579 - root - INFO - step: 61130 loss: 16.3065 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.8030 +[titan] 2025-07-10 03:02:55,579 - root - INFO - lr: 1.2032e-04 gnorm: 0.95 [13:27:54< 8:33:42] +[titan] 2025-07-10 03:02:59,499 - root - INFO - step: 61135 loss: 16.5050 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.9755 +[titan] 2025-07-10 03:02:59,499 - root - INFO - lr: 1.2030e-04 gnorm: 0.87 [13:27:58< 8:33:38] +[titan] 2025-07-10 03:03:03,409 - root - INFO - step: 61140 loss: 16.2940 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.7953 +[titan] 2025-07-10 03:03:03,409 - root - INFO - lr: 1.2028e-04 gnorm: 0.85 [13:28:02< 8:33:34] +[titan] 2025-07-10 03:03:07,323 - root - INFO - step: 61145 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8489 +[titan] 2025-07-10 03:03:07,323 - root - INFO - lr: 1.2026e-04 gnorm: 0.88 [13:28:06< 8:33:30] +[titan] 2025-07-10 03:03:10,445 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:03:11,259 - root - INFO - step: 61150 loss: 16.4139 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8985 +[titan] 2025-07-10 03:03:11,259 - root - INFO - lr: 1.2024e-04 gnorm: 0.86 [13:28:10< 8:33:26] +[titan] 2025-07-10 03:03:15,174 - root - INFO - step: 61155 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 03:03:15,175 - root - INFO - lr: 1.2022e-04 gnorm: 0.91 [13:28:14< 8:33:22] +[titan] 2025-07-10 03:03:19,107 - root - INFO - step: 61160 loss: 16.4777 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9461 +[titan] 2025-07-10 03:03:19,108 - root - INFO - lr: 1.2020e-04 gnorm: 0.91 [13:28:18< 8:33:19] +[titan] 2025-07-10 03:03:23,059 - root - INFO - step: 61165 loss: 16.6331 memory: 44.58GiB(31.99%) tps: 82,936 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0826 +[titan] 2025-07-10 03:03:23,059 - root - INFO - lr: 1.2018e-04 gnorm: 0.86 [13:28:22< 8:33:15] +[titan] 2025-07-10 03:03:26,997 - root - INFO - step: 61170 loss: 16.5220 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9903 +[titan] 2025-07-10 03:03:26,997 - root - INFO - lr: 1.2016e-04 gnorm: 0.87 [13:28:26< 8:33:11] +[titan] 2025-07-10 03:03:30,896 - root - INFO - step: 61175 loss: 16.3059 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8046 +[titan] 2025-07-10 03:03:30,897 - root - INFO - lr: 1.2014e-04 gnorm: 0.86 [13:28:29< 8:33:07] +[titan] 2025-07-10 03:03:34,796 - root - INFO - step: 61180 loss: 16.5060 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9716 +[titan] 2025-07-10 03:03:34,797 - root - INFO - lr: 1.2012e-04 gnorm: 0.84 [13:28:33< 8:33:03] +[titan] 2025-07-10 03:03:38,692 - root - INFO - step: 61185 loss: 16.5270 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-10 03:03:38,692 - root - INFO - lr: 1.2010e-04 gnorm: 0.86 [13:28:37< 8:32:59] +[titan] 2025-07-10 03:03:42,633 - root - INFO - step: 61190 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 14.0058 +[titan] 2025-07-10 03:03:42,633 - root - INFO - lr: 1.2008e-04 gnorm: 0.88 [13:28:41< 8:32:55] +[titan] 2025-07-10 03:03:46,557 - root - INFO - step: 61195 loss: 16.3099 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8069 +[titan] 2025-07-10 03:03:46,557 - root - INFO - lr: 1.2006e-04 gnorm: 0.85 [13:28:45< 8:32:51] +[titan] 2025-07-10 03:03:49,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:03:50,460 - root - INFO - step: 61200 loss: 16.4196 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.8860 +[titan] 2025-07-10 03:03:50,460 - root - INFO - lr: 1.2004e-04 gnorm: 0.87 [13:28:49< 8:32:47] +[titan] 2025-07-10 03:03:54,385 - root - INFO - step: 61205 loss: 16.1911 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.7245 +[titan] 2025-07-10 03:03:54,385 - root - INFO - lr: 1.2002e-04 gnorm: 0.87 [13:28:53< 8:32:43] +[titan] 2025-07-10 03:03:58,339 - root - INFO - step: 61210 loss: 16.4633 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9395 +[titan] 2025-07-10 03:03:58,339 - root - INFO - lr: 1.2000e-04 gnorm: 0.91 [13:28:57< 8:32:39] +[titan] 2025-07-10 03:04:02,245 - root - INFO - step: 61215 loss: 16.3801 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-10 03:04:02,246 - root - INFO - lr: 1.1997e-04 gnorm: 0.86 [13:29:01< 8:32:35] +[titan] 2025-07-10 03:04:06,193 - root - INFO - step: 61220 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9721 +[titan] 2025-07-10 03:04:06,193 - root - INFO - lr: 1.1995e-04 gnorm: 0.94 [13:29:05< 8:32:31] +[titan] 2025-07-10 03:04:10,095 - root - INFO - step: 61225 loss: 16.2311 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7492 +[titan] 2025-07-10 03:04:10,095 - root - INFO - lr: 1.1993e-04 gnorm: 0.90 [13:29:09< 8:32:27] +[titan] 2025-07-10 03:04:14,006 - root - INFO - step: 61230 loss: 16.3485 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8418 +[titan] 2025-07-10 03:04:14,006 - root - INFO - lr: 1.1991e-04 gnorm: 0.87 [13:29:13< 8:32:23] +[titan] 2025-07-10 03:04:17,959 - root - INFO - step: 61235 loss: 16.3004 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 03:04:17,959 - root - INFO - lr: 1.1989e-04 gnorm: 0.90 [13:29:16< 8:32:19] +[titan] 2025-07-10 03:04:21,850 - root - INFO - step: 61240 loss: 16.0812 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.6216 +[titan] 2025-07-10 03:04:21,851 - root - INFO - lr: 1.1987e-04 gnorm: 1.00 [13:29:20< 8:32:15] +[titan] 2025-07-10 03:04:25,763 - root - INFO - step: 61245 loss: 16.5723 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0113 +[titan] 2025-07-10 03:04:25,763 - root - INFO - lr: 1.1985e-04 gnorm: 0.82 [13:29:24< 8:32:11] +[titan] 2025-07-10 03:04:28,892 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:04:29,699 - root - INFO - step: 61250 loss: 16.5463 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0008 +[titan] 2025-07-10 03:04:29,699 - root - INFO - lr: 1.1983e-04 gnorm: 0.93 [13:29:28< 8:32:07] +[titan] 2025-07-10 03:04:33,618 - root - INFO - step: 61255 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9302 +[titan] 2025-07-10 03:04:33,618 - root - INFO - lr: 1.1981e-04 gnorm: 0.85 [13:29:32< 8:32:03] +[titan] 2025-07-10 03:04:37,576 - root - INFO - step: 61260 loss: 16.6210 memory: 44.58GiB(31.99%) tps: 82,786 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.5659 global_avg_mtp_loss: 14.0551 +[titan] 2025-07-10 03:04:37,576 - root - INFO - lr: 1.1979e-04 gnorm: 0.91 [13:29:36< 8:31:59] +[titan] 2025-07-10 03:04:41,495 - root - INFO - step: 61265 loss: 16.5230 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9863 +[titan] 2025-07-10 03:04:41,495 - root - INFO - lr: 1.1977e-04 gnorm: 0.87 [13:29:40< 8:31:55] +[titan] 2025-07-10 03:04:45,415 - root - INFO - step: 61270 loss: 16.3895 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.8708 +[titan] 2025-07-10 03:04:45,415 - root - INFO - lr: 1.1975e-04 gnorm: 0.89 [13:29:44< 8:31:51] +[titan] 2025-07-10 03:04:49,349 - root - INFO - step: 61275 loss: 16.2576 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7648 +[titan] 2025-07-10 03:04:49,349 - root - INFO - lr: 1.1973e-04 gnorm: 0.87 [13:29:48< 8:31:47] +[titan] 2025-07-10 03:04:53,296 - root - INFO - step: 61280 loss: 16.6469 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5612 global_avg_mtp_loss: 14.0857 +[titan] 2025-07-10 03:04:53,296 - root - INFO - lr: 1.1971e-04 gnorm: 0.91 [13:29:52< 8:31:43] +[titan] 2025-07-10 03:04:57,228 - root - INFO - step: 61285 loss: 16.4133 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.9009 +[titan] 2025-07-10 03:04:57,228 - root - INFO - lr: 1.1969e-04 gnorm: 0.84 [13:29:56< 8:31:39] +[titan] 2025-07-10 03:05:01,128 - root - INFO - step: 61290 loss: 16.9440 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.6210 global_avg_mtp_loss: 14.3230 +[titan] 2025-07-10 03:05:01,129 - root - INFO - lr: 1.1967e-04 gnorm: 0.85 [13:30:00< 8:31:35] +[titan] 2025-07-10 03:05:05,035 - root - INFO - step: 61295 loss: 16.3860 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.8662 +[titan] 2025-07-10 03:05:05,035 - root - INFO - lr: 1.1965e-04 gnorm: 0.85 [13:30:04< 8:31:31] +[titan] 2025-07-10 03:05:08,142 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:05:08,931 - root - INFO - step: 61300 loss: 16.6572 memory: 44.58GiB(31.99%) tps: 84,104 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5824 global_avg_mtp_loss: 14.0748 +[titan] 2025-07-10 03:05:08,931 - root - INFO - lr: 1.1963e-04 gnorm: 0.88 [13:30:07< 8:31:27] +[titan] 2025-07-10 03:05:12,842 - root - INFO - step: 61305 loss: 16.4029 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.8946 +[titan] 2025-07-10 03:05:12,843 - root - INFO - lr: 1.1961e-04 gnorm: 0.86 [13:30:11< 8:31:23] +[titan] 2025-07-10 03:05:16,829 - root - INFO - step: 61310 loss: 16.5552 memory: 44.58GiB(31.99%) tps: 82,195 tflops: 283.67 mfu: 28.68% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 13.9933 +[titan] 2025-07-10 03:05:16,830 - root - INFO - lr: 1.1959e-04 gnorm: 1.00 [13:30:15< 8:31:19] +[titan] 2025-07-10 03:05:20,750 - root - INFO - step: 61315 loss: 16.3025 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8065 +[titan] 2025-07-10 03:05:20,750 - root - INFO - lr: 1.1957e-04 gnorm: 1.37 [13:30:19< 8:31:15] +[titan] 2025-07-10 03:05:24,668 - root - INFO - step: 61320 loss: 16.3743 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8665 +[titan] 2025-07-10 03:05:24,669 - root - INFO - lr: 1.1955e-04 gnorm: 0.88 [13:30:23< 8:31:11] +[titan] 2025-07-10 03:05:28,621 - root - INFO - step: 61325 loss: 16.4811 memory: 44.58GiB(31.99%) tps: 82,914 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9499 +[titan] 2025-07-10 03:05:28,621 - root - INFO - lr: 1.1953e-04 gnorm: 0.93 [13:30:27< 8:31:07] +[titan] 2025-07-10 03:05:32,513 - root - INFO - step: 61330 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9989 +[titan] 2025-07-10 03:05:32,514 - root - INFO - lr: 1.1951e-04 gnorm: 0.83 [13:30:31< 8:31:03] +[titan] 2025-07-10 03:05:36,444 - root - INFO - step: 61335 loss: 16.4564 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.9345 +[titan] 2025-07-10 03:05:36,445 - root - INFO - lr: 1.1949e-04 gnorm: 0.99 [13:30:35< 8:30:59] +[titan] 2025-07-10 03:05:40,362 - root - INFO - step: 61340 loss: 16.2437 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.7538 +[titan] 2025-07-10 03:05:40,363 - root - INFO - lr: 1.1947e-04 gnorm: 0.89 [13:30:39< 8:30:55] +[titan] 2025-07-10 03:05:44,302 - root - INFO - step: 61345 loss: 16.3383 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8284 +[titan] 2025-07-10 03:05:44,303 - root - INFO - lr: 1.1945e-04 gnorm: 0.87 [13:30:43< 8:30:51] +[titan] 2025-07-10 03:05:47,438 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:05:48,226 - root - INFO - step: 61350 loss: 16.3447 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-10 03:05:48,226 - root - INFO - lr: 1.1943e-04 gnorm: 0.85 [13:30:47< 8:30:47] +[titan] 2025-07-10 03:05:52,138 - root - INFO - step: 61355 loss: 16.4111 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.8754 +[titan] 2025-07-10 03:05:52,138 - root - INFO - lr: 1.1941e-04 gnorm: 0.83 [13:30:51< 8:30:43] +[titan] 2025-07-10 03:05:56,039 - root - INFO - step: 61360 loss: 16.7431 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5947 global_avg_mtp_loss: 14.1484 +[titan] 2025-07-10 03:05:56,040 - root - INFO - lr: 1.1939e-04 gnorm: 0.90 [13:30:55< 8:30:39] +[titan] 2025-07-10 03:05:59,955 - root - INFO - step: 61365 loss: 16.2549 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7695 +[titan] 2025-07-10 03:05:59,956 - root - INFO - lr: 1.1937e-04 gnorm: 0.83 [13:30:58< 8:30:35] +[titan] 2025-07-10 03:06:03,877 - root - INFO - step: 61370 loss: 16.4763 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9500 +[titan] 2025-07-10 03:06:03,877 - root - INFO - lr: 1.1935e-04 gnorm: 0.85 [13:31:02< 8:30:31] +[titan] 2025-07-10 03:06:07,784 - root - INFO - step: 61375 loss: 16.6244 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 14.0721 +[titan] 2025-07-10 03:06:07,784 - root - INFO - lr: 1.1933e-04 gnorm: 0.81 [13:31:06< 8:30:27] +[titan] 2025-07-10 03:06:11,717 - root - INFO - step: 61380 loss: 16.4646 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9401 +[titan] 2025-07-10 03:06:11,717 - root - INFO - lr: 1.1931e-04 gnorm: 0.89 [13:31:10< 8:30:23] +[titan] 2025-07-10 03:06:15,629 - root - INFO - step: 61385 loss: 16.4174 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5180 global_avg_mtp_loss: 13.8993 +[titan] 2025-07-10 03:06:15,629 - root - INFO - lr: 1.1929e-04 gnorm: 0.94 [13:31:14< 8:30:19] +[titan] 2025-07-10 03:06:19,567 - root - INFO - step: 61390 loss: 16.2393 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7553 +[titan] 2025-07-10 03:06:19,568 - root - INFO - lr: 1.1927e-04 gnorm: 0.87 [13:31:18< 8:30:15] +[titan] 2025-07-10 03:06:23,512 - root - INFO - step: 61395 loss: 16.3752 memory: 44.58GiB(31.99%) tps: 83,069 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 03:06:23,513 - root - INFO - lr: 1.1925e-04 gnorm: 0.89 [13:31:22< 8:30:11] +[titan] 2025-07-10 03:06:26,674 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:06:27,468 - root - INFO - step: 61400 loss: 16.6224 memory: 44.58GiB(31.99%) tps: 82,841 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0664 +[titan] 2025-07-10 03:06:27,469 - root - INFO - lr: 1.1923e-04 gnorm: 0.87 [13:31:26< 8:30:07] +[titan] 2025-07-10 03:06:31,359 - root - INFO - step: 61405 loss: 16.4352 memory: 44.58GiB(31.99%) tps: 84,235 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.9091 +[titan] 2025-07-10 03:06:31,359 - root - INFO - lr: 1.1921e-04 gnorm: 0.86 [13:31:30< 8:30:03] +[titan] 2025-07-10 03:06:35,267 - root - INFO - step: 61410 loss: 16.5103 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5433 global_avg_mtp_loss: 13.9670 +[titan] 2025-07-10 03:06:35,268 - root - INFO - lr: 1.1919e-04 gnorm: 0.85 [13:31:34< 8:29:59] +[titan] 2025-07-10 03:06:39,167 - root - INFO - step: 61415 loss: 16.6713 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.1041 +[titan] 2025-07-10 03:06:39,167 - root - INFO - lr: 1.1917e-04 gnorm: 0.88 [13:31:38< 8:29:55] +[titan] 2025-07-10 03:06:43,117 - root - INFO - step: 61420 loss: 16.6816 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.1156 +[titan] 2025-07-10 03:06:43,118 - root - INFO - lr: 1.1915e-04 gnorm: 0.82 [13:31:42< 8:29:51] +[titan] 2025-07-10 03:06:47,054 - root - INFO - step: 61425 loss: 16.2524 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.7558 +[titan] 2025-07-10 03:06:47,055 - root - INFO - lr: 1.1913e-04 gnorm: 0.82 [13:31:46< 8:29:47] +[titan] 2025-07-10 03:06:50,989 - root - INFO - step: 61430 loss: 16.6315 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5607 global_avg_mtp_loss: 14.0708 +[titan] 2025-07-10 03:06:50,989 - root - INFO - lr: 1.1911e-04 gnorm: 0.86 [13:31:49< 8:29:43] +[titan] 2025-07-10 03:06:54,898 - root - INFO - step: 61435 loss: 16.5118 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.9836 +[titan] 2025-07-10 03:06:54,898 - root - INFO - lr: 1.1909e-04 gnorm: 0.88 [13:31:53< 8:29:39] +[titan] 2025-07-10 03:06:58,902 - root - INFO - step: 61440 loss: 16.4325 memory: 44.58GiB(31.99%) tps: 81,843 tflops: 282.45 mfu: 28.56% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9047 +[titan] 2025-07-10 03:06:58,902 - root - INFO - lr: 1.1907e-04 gnorm: 0.91 [13:31:57< 8:29:35] +[titan] 2025-07-10 03:06:59,059 - root - INFO - Dumping profiler traces at step 61440 +[titan] 2025-07-10 03:06:59,094 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:07:03,031 - root - INFO - step: 61445 loss: 16.8094 memory: 44.58GiB(31.99%) tps: 79,361 tflops: 273.89 mfu: 27.69% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.2250 +[titan] 2025-07-10 03:07:03,031 - root - INFO - lr: 1.1905e-04 gnorm: 0.87 [13:32:02< 8:29:31] +[titan] 2025-07-10 03:07:06,172 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:07:06,960 - root - INFO - step: 61450 loss: 16.3080 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.8155 +[titan] 2025-07-10 03:07:06,960 - root - INFO - lr: 1.1903e-04 gnorm: 0.87 [13:32:05< 8:29:27] +[titan] 2025-07-10 03:07:10,871 - root - INFO - step: 61455 loss: 16.7082 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5881 global_avg_mtp_loss: 14.1201 +[titan] 2025-07-10 03:07:10,872 - root - INFO - lr: 1.1901e-04 gnorm: 0.95 [13:32:09< 8:29:23] +[titan] 2025-07-10 03:07:14,796 - root - INFO - step: 61460 loss: 16.5778 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 14.0298 +[titan] 2025-07-10 03:07:14,796 - root - INFO - lr: 1.1899e-04 gnorm: 0.90 [13:32:13< 8:29:19] +[titan] 2025-07-10 03:07:18,718 - root - INFO - step: 61465 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5165 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 03:07:18,718 - root - INFO - lr: 1.1897e-04 gnorm: 0.88 [13:32:17< 8:29:15] +[titan] 2025-07-10 03:07:22,627 - root - INFO - step: 61470 loss: 16.4990 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9730 +[titan] 2025-07-10 03:07:22,628 - root - INFO - lr: 1.1895e-04 gnorm: 0.89 [13:32:21< 8:29:11] +[titan] 2025-07-10 03:07:26,543 - root - INFO - step: 61475 loss: 16.2435 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7505 +[titan] 2025-07-10 03:07:26,544 - root - INFO - lr: 1.1893e-04 gnorm: 0.88 [13:32:25< 8:29:07] +[titan] 2025-07-10 03:07:30,463 - root - INFO - step: 61480 loss: 16.1686 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.6804 +[titan] 2025-07-10 03:07:30,463 - root - INFO - lr: 1.1891e-04 gnorm: 0.84 [13:32:29< 8:29:03] +[titan] 2025-07-10 03:07:34,386 - root - INFO - step: 61485 loss: 16.4451 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9138 +[titan] 2025-07-10 03:07:34,386 - root - INFO - lr: 1.1889e-04 gnorm: 0.84 [13:32:33< 8:28:59] +[titan] 2025-07-10 03:07:38,317 - root - INFO - step: 61490 loss: 16.5977 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5488 global_avg_mtp_loss: 14.0488 +[titan] 2025-07-10 03:07:38,318 - root - INFO - lr: 1.1887e-04 gnorm: 0.91 [13:32:37< 8:28:55] +[titan] 2025-07-10 03:07:42,217 - root - INFO - step: 61495 loss: 16.6558 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0980 +[titan] 2025-07-10 03:07:42,217 - root - INFO - lr: 1.1885e-04 gnorm: 0.91 [13:32:41< 8:28:51] +[titan] 2025-07-10 03:07:45,324 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:07:46,112 - root - INFO - step: 61500 loss: 16.5182 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9845 +[titan] 2025-07-10 03:07:46,112 - root - INFO - lr: 1.1883e-04 gnorm: 0.89 [13:32:45< 8:28:47] +[titan] 2025-07-10 03:07:50,048 - root - INFO - step: 61505 loss: 16.3160 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 03:07:50,048 - root - INFO - lr: 1.1881e-04 gnorm: 0.89 [13:32:49< 8:28:43] +[titan] 2025-07-10 03:07:53,980 - root - INFO - step: 61510 loss: 16.2456 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7592 +[titan] 2025-07-10 03:07:53,980 - root - INFO - lr: 1.1879e-04 gnorm: 0.88 [13:32:52< 8:28:39] +[titan] 2025-07-10 03:07:57,892 - root - INFO - step: 61515 loss: 16.2384 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7414 +[titan] 2025-07-10 03:07:57,892 - root - INFO - lr: 1.1877e-04 gnorm: 0.84 [13:32:56< 8:28:35] +[titan] 2025-07-10 03:08:01,794 - root - INFO - step: 61520 loss: 16.4597 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9262 +[titan] 2025-07-10 03:08:01,794 - root - INFO - lr: 1.1875e-04 gnorm: 0.81 [13:33:00< 8:28:31] +[titan] 2025-07-10 03:08:05,688 - root - INFO - step: 61525 loss: 16.5063 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 13.9765 +[titan] 2025-07-10 03:08:05,689 - root - INFO - lr: 1.1873e-04 gnorm: 0.82 [13:33:04< 8:28:27] +[titan] 2025-07-10 03:08:09,589 - root - INFO - step: 61530 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 14.0040 +[titan] 2025-07-10 03:08:09,590 - root - INFO - lr: 1.1870e-04 gnorm: 0.84 [13:33:08< 8:28:23] +[titan] 2025-07-10 03:08:13,507 - root - INFO - step: 61535 loss: 16.3864 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8790 +[titan] 2025-07-10 03:08:13,508 - root - INFO - lr: 1.1868e-04 gnorm: 0.90 [13:33:12< 8:28:19] +[titan] 2025-07-10 03:08:17,402 - root - INFO - step: 61540 loss: 16.5169 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5285 global_avg_mtp_loss: 13.9884 +[titan] 2025-07-10 03:08:17,402 - root - INFO - lr: 1.1866e-04 gnorm: 0.89 [13:33:16< 8:28:15] +[titan] 2025-07-10 03:08:21,315 - root - INFO - step: 61545 loss: 16.4833 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9442 +[titan] 2025-07-10 03:08:21,315 - root - INFO - lr: 1.1864e-04 gnorm: 0.88 [13:33:20< 8:28:11] +[titan] 2025-07-10 03:08:24,440 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:08:25,231 - root - INFO - step: 61550 loss: 16.1619 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.6841 +[titan] 2025-07-10 03:08:25,232 - root - INFO - lr: 1.1862e-04 gnorm: 0.95 [13:33:24< 8:28:07] +[titan] 2025-07-10 03:08:29,177 - root - INFO - step: 61555 loss: 16.4081 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8978 +[titan] 2025-07-10 03:08:29,177 - root - INFO - lr: 1.1860e-04 gnorm: 0.88 [13:33:28< 8:28:03] +[titan] 2025-07-10 03:08:33,091 - root - INFO - step: 61560 loss: 16.2194 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7381 +[titan] 2025-07-10 03:08:33,092 - root - INFO - lr: 1.1858e-04 gnorm: 0.95 [13:33:32< 8:27:59] +[titan] 2025-07-10 03:08:36,990 - root - INFO - step: 61565 loss: 16.6004 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-10 03:08:36,990 - root - INFO - lr: 1.1856e-04 gnorm: 0.85 [13:33:35< 8:27:55] +[titan] 2025-07-10 03:08:40,893 - root - INFO - step: 61570 loss: 16.4219 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.9022 +[titan] 2025-07-10 03:08:40,893 - root - INFO - lr: 1.1854e-04 gnorm: 0.86 [13:33:39< 8:27:51] +[titan] 2025-07-10 03:08:44,826 - root - INFO - step: 61575 loss: 16.6132 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0563 +[titan] 2025-07-10 03:08:44,827 - root - INFO - lr: 1.1852e-04 gnorm: 0.87 [13:33:43< 8:27:47] +[titan] 2025-07-10 03:08:48,769 - root - INFO - step: 61580 loss: 16.4464 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9249 +[titan] 2025-07-10 03:08:48,769 - root - INFO - lr: 1.1850e-04 gnorm: 0.89 [13:33:47< 8:27:43] +[titan] 2025-07-10 03:08:52,706 - root - INFO - step: 61585 loss: 16.2332 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7362 +[titan] 2025-07-10 03:08:52,707 - root - INFO - lr: 1.1848e-04 gnorm: 0.96 [13:33:51< 8:27:39] +[titan] 2025-07-10 03:08:56,646 - root - INFO - step: 61590 loss: 16.5929 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0350 +[titan] 2025-07-10 03:08:56,646 - root - INFO - lr: 1.1846e-04 gnorm: 0.91 [13:33:55< 8:27:35] +[titan] 2025-07-10 03:09:00,551 - root - INFO - step: 61595 loss: 16.5063 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9710 +[titan] 2025-07-10 03:09:00,552 - root - INFO - lr: 1.1844e-04 gnorm: 0.90 [13:33:59< 8:27:31] +[titan] 2025-07-10 03:09:03,698 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:09:04,489 - root - INFO - step: 61600 loss: 16.3081 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.8156 +[titan] 2025-07-10 03:09:04,489 - root - INFO - lr: 1.1842e-04 gnorm: 0.89 [13:34:03< 8:27:27] +[titan] 2025-07-10 03:09:08,422 - root - INFO - step: 61605 loss: 16.6794 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 14.1232 +[titan] 2025-07-10 03:09:08,423 - root - INFO - lr: 1.1840e-04 gnorm: 0.87 [13:34:07< 8:27:23] +[titan] 2025-07-10 03:09:12,382 - root - INFO - step: 61610 loss: 16.4529 memory: 44.58GiB(31.99%) tps: 82,768 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9303 +[titan] 2025-07-10 03:09:12,382 - root - INFO - lr: 1.1838e-04 gnorm: 0.79 [13:34:11< 8:27:19] +[titan] 2025-07-10 03:09:16,283 - root - INFO - step: 61615 loss: 16.5673 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 14.0275 +[titan] 2025-07-10 03:09:16,283 - root - INFO - lr: 1.1836e-04 gnorm: 0.83 [13:34:15< 8:27:15] +[titan] 2025-07-10 03:09:20,229 - root - INFO - step: 61620 loss: 16.3719 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.8480 +[titan] 2025-07-10 03:09:20,229 - root - INFO - lr: 1.1834e-04 gnorm: 0.88 [13:34:19< 8:27:11] +[titan] 2025-07-10 03:09:24,122 - root - INFO - step: 61625 loss: 16.4224 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8991 +[titan] 2025-07-10 03:09:24,122 - root - INFO - lr: 1.1832e-04 gnorm: 0.89 [13:34:23< 8:27:07] +[titan] 2025-07-10 03:09:28,069 - root - INFO - step: 61630 loss: 16.3455 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8399 +[titan] 2025-07-10 03:09:28,069 - root - INFO - lr: 1.1830e-04 gnorm: 0.90 [13:34:27< 8:27:03] +[titan] 2025-07-10 03:09:31,991 - root - INFO - step: 61635 loss: 16.1823 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7072 +[titan] 2025-07-10 03:09:31,992 - root - INFO - lr: 1.1828e-04 gnorm: 0.87 [13:34:30< 8:26:59] +[titan] 2025-07-10 03:09:35,895 - root - INFO - step: 61640 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 13.9650 +[titan] 2025-07-10 03:09:35,895 - root - INFO - lr: 1.1826e-04 gnorm: 0.90 [13:34:34< 8:26:55] +[titan] 2025-07-10 03:09:39,791 - root - INFO - step: 61645 loss: 16.4546 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.9416 +[titan] 2025-07-10 03:09:39,792 - root - INFO - lr: 1.1824e-04 gnorm: 0.82 [13:34:38< 8:26:51] +[titan] 2025-07-10 03:09:42,922 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:09:43,715 - root - INFO - step: 61650 loss: 16.3129 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 03:09:43,715 - root - INFO - lr: 1.1822e-04 gnorm: 0.89 [13:34:42< 8:26:47] +[titan] 2025-07-10 03:09:47,624 - root - INFO - step: 61655 loss: 16.2956 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.7988 +[titan] 2025-07-10 03:09:47,624 - root - INFO - lr: 1.1820e-04 gnorm: 0.89 [13:34:46< 8:26:43] +[titan] 2025-07-10 03:09:51,533 - root - INFO - step: 61660 loss: 16.4806 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5282 global_avg_mtp_loss: 13.9524 +[titan] 2025-07-10 03:09:51,534 - root - INFO - lr: 1.1818e-04 gnorm: 0.87 [13:34:50< 8:26:39] +[titan] 2025-07-10 03:09:55,436 - root - INFO - step: 61665 loss: 16.0013 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.4345 global_avg_mtp_loss: 13.5668 +[titan] 2025-07-10 03:09:55,436 - root - INFO - lr: 1.1816e-04 gnorm: 0.86 [13:34:54< 8:26:35] +[titan] 2025-07-10 03:09:59,363 - root - INFO - step: 61670 loss: 16.1530 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.6870 +[titan] 2025-07-10 03:09:59,363 - root - INFO - lr: 1.1814e-04 gnorm: 0.91 [13:34:58< 8:26:31] +[titan] 2025-07-10 03:10:03,319 - root - INFO - step: 61675 loss: 16.3809 memory: 44.58GiB(31.99%) tps: 82,849 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.8591 +[titan] 2025-07-10 03:10:03,319 - root - INFO - lr: 1.1812e-04 gnorm: 0.92 [13:35:02< 8:26:28] +[titan] 2025-07-10 03:10:07,229 - root - INFO - step: 61680 loss: 16.4470 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9247 +[titan] 2025-07-10 03:10:07,230 - root - INFO - lr: 1.1810e-04 gnorm: 0.81 [13:35:06< 8:26:24] +[titan] 2025-07-10 03:10:11,139 - root - INFO - step: 61685 loss: 16.3349 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 03:10:11,139 - root - INFO - lr: 1.1808e-04 gnorm: 0.86 [13:35:10< 8:26:20] +[titan] 2025-07-10 03:10:15,050 - root - INFO - step: 61690 loss: 16.3288 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8102 +[titan] 2025-07-10 03:10:15,050 - root - INFO - lr: 1.1806e-04 gnorm: 0.87 [13:35:14< 8:26:16] +[titan] 2025-07-10 03:10:18,954 - root - INFO - step: 61695 loss: 16.3981 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8864 +[titan] 2025-07-10 03:10:18,954 - root - INFO - lr: 1.1804e-04 gnorm: 0.86 [13:35:17< 8:26:12] +[titan] 2025-07-10 03:10:22,071 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:10:22,861 - root - INFO - step: 61700 loss: 16.3361 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8299 +[titan] 2025-07-10 03:10:22,861 - root - INFO - lr: 1.1802e-04 gnorm: 0.82 [13:35:21< 8:26:08] +[titan] 2025-07-10 03:10:26,790 - root - INFO - step: 61705 loss: 16.3769 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8625 +[titan] 2025-07-10 03:10:26,790 - root - INFO - lr: 1.1800e-04 gnorm: 0.83 [13:35:25< 8:26:04] +[titan] 2025-07-10 03:10:30,749 - root - INFO - step: 61710 loss: 16.3133 memory: 44.58GiB(31.99%) tps: 82,774 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8164 +[titan] 2025-07-10 03:10:30,749 - root - INFO - lr: 1.1798e-04 gnorm: 0.83 [13:35:29< 8:26:00] +[titan] 2025-07-10 03:10:34,655 - root - INFO - step: 61715 loss: 16.5842 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0220 +[titan] 2025-07-10 03:10:34,656 - root - INFO - lr: 1.1796e-04 gnorm: 0.88 [13:35:33< 8:25:56] +[titan] 2025-07-10 03:10:38,565 - root - INFO - step: 61720 loss: 16.3469 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5036 global_avg_mtp_loss: 13.8433 +[titan] 2025-07-10 03:10:38,565 - root - INFO - lr: 1.1794e-04 gnorm: 0.80 [13:35:37< 8:25:52] +[titan] 2025-07-10 03:10:42,486 - root - INFO - step: 61725 loss: 16.0650 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.6101 +[titan] 2025-07-10 03:10:42,486 - root - INFO - lr: 1.1792e-04 gnorm: 0.87 [13:35:41< 8:25:48] +[titan] 2025-07-10 03:10:46,431 - root - INFO - step: 61730 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9060 +[titan] 2025-07-10 03:10:46,431 - root - INFO - lr: 1.1790e-04 gnorm: 0.85 [13:35:45< 8:25:44] +[titan] 2025-07-10 03:10:50,370 - root - INFO - step: 61735 loss: 16.3869 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.8623 +[titan] 2025-07-10 03:10:50,370 - root - INFO - lr: 1.1788e-04 gnorm: 0.89 [13:35:49< 8:25:40] +[titan] 2025-07-10 03:10:54,341 - root - INFO - step: 61740 loss: 16.3519 memory: 44.58GiB(31.99%) tps: 82,524 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8405 +[titan] 2025-07-10 03:10:54,342 - root - INFO - lr: 1.1786e-04 gnorm: 0.89 [13:35:53< 8:25:36] +[titan] 2025-07-10 03:10:58,277 - root - INFO - step: 61745 loss: 16.2558 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7741 +[titan] 2025-07-10 03:10:58,277 - root - INFO - lr: 1.1784e-04 gnorm: 0.87 [13:35:57< 8:25:32] +[titan] 2025-07-10 03:11:01,399 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:11:02,193 - root - INFO - step: 61750 loss: 16.4429 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9186 +[titan] 2025-07-10 03:11:02,193 - root - INFO - lr: 1.1782e-04 gnorm: 0.86 [13:36:01< 8:25:28] +[titan] 2025-07-10 03:11:06,127 - root - INFO - step: 61755 loss: 16.2967 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.7966 +[titan] 2025-07-10 03:11:06,127 - root - INFO - lr: 1.1780e-04 gnorm: 0.85 [13:36:05< 8:25:24] +[titan] 2025-07-10 03:11:10,041 - root - INFO - step: 61760 loss: 16.3336 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8324 +[titan] 2025-07-10 03:11:10,041 - root - INFO - lr: 1.1778e-04 gnorm: 0.85 [13:36:09< 8:25:20] +[titan] 2025-07-10 03:11:13,955 - root - INFO - step: 61765 loss: 16.3861 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8707 +[titan] 2025-07-10 03:11:13,955 - root - INFO - lr: 1.1776e-04 gnorm: 0.88 [13:36:12< 8:25:16] +[titan] 2025-07-10 03:11:17,885 - root - INFO - step: 61770 loss: 16.2265 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7459 +[titan] 2025-07-10 03:11:17,886 - root - INFO - lr: 1.1774e-04 gnorm: 0.87 [13:36:16< 8:25:12] +[titan] 2025-07-10 03:11:21,809 - root - INFO - step: 61775 loss: 16.5630 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 14.0178 +[titan] 2025-07-10 03:11:21,809 - root - INFO - lr: 1.1772e-04 gnorm: 0.97 [13:36:20< 8:25:08] +[titan] 2025-07-10 03:11:25,741 - root - INFO - step: 61780 loss: 16.4962 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9554 +[titan] 2025-07-10 03:11:25,741 - root - INFO - lr: 1.1770e-04 gnorm: 0.93 [13:36:24< 8:25:04] +[titan] 2025-07-10 03:11:29,649 - root - INFO - step: 61785 loss: 16.3367 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8358 +[titan] 2025-07-10 03:11:29,649 - root - INFO - lr: 1.1768e-04 gnorm: 0.85 [13:36:28< 8:25:00] +[titan] 2025-07-10 03:11:33,580 - root - INFO - step: 61790 loss: 16.6167 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 14.0722 +[titan] 2025-07-10 03:11:33,580 - root - INFO - lr: 1.1766e-04 gnorm: 0.92 [13:36:32< 8:24:56] +[titan] 2025-07-10 03:11:37,516 - root - INFO - step: 61795 loss: 16.4037 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.8814 +[titan] 2025-07-10 03:11:37,516 - root - INFO - lr: 1.1764e-04 gnorm: 0.85 [13:36:36< 8:24:52] +[titan] 2025-07-10 03:11:40,678 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:11:41,473 - root - INFO - step: 61800 loss: 16.3454 memory: 44.58GiB(31.99%) tps: 82,806 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.8495 +[titan] 2025-07-10 03:11:41,474 - root - INFO - lr: 1.1762e-04 gnorm: 0.85 [13:36:40< 8:24:48] +[titan] 2025-07-10 03:11:45,437 - root - INFO - step: 61805 loss: 16.3855 memory: 44.58GiB(31.99%) tps: 82,692 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 03:11:45,437 - root - INFO - lr: 1.1760e-04 gnorm: 0.87 [13:36:44< 8:24:44] +[titan] 2025-07-10 03:11:49,353 - root - INFO - step: 61810 loss: 16.5200 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 13.9755 +[titan] 2025-07-10 03:11:49,353 - root - INFO - lr: 1.1758e-04 gnorm: 0.86 [13:36:48< 8:24:40] +[titan] 2025-07-10 03:11:53,272 - root - INFO - step: 61815 loss: 16.5668 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 14.0117 +[titan] 2025-07-10 03:11:53,272 - root - INFO - lr: 1.1756e-04 gnorm: 0.85 [13:36:52< 8:24:36] +[titan] 2025-07-10 03:11:57,200 - root - INFO - step: 61820 loss: 16.4284 memory: 44.58GiB(31.99%) tps: 83,436 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.9058 +[titan] 2025-07-10 03:11:57,200 - root - INFO - lr: 1.1754e-04 gnorm: 0.95 [13:36:56< 8:24:32] +[titan] 2025-07-10 03:12:01,142 - root - INFO - step: 61825 loss: 16.0956 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6315 +[titan] 2025-07-10 03:12:01,143 - root - INFO - lr: 1.1752e-04 gnorm: 0.87 [13:37:00< 8:24:28] +[titan] 2025-07-10 03:12:05,053 - root - INFO - step: 61830 loss: 16.5169 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5372 global_avg_mtp_loss: 13.9797 +[titan] 2025-07-10 03:12:05,053 - root - INFO - lr: 1.1750e-04 gnorm: 0.86 [13:37:04< 8:24:24] +[titan] 2025-07-10 03:12:08,966 - root - INFO - step: 61835 loss: 16.3329 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 03:12:08,966 - root - INFO - lr: 1.1748e-04 gnorm: 0.83 [13:37:07< 8:24:20] +[titan] 2025-07-10 03:12:12,906 - root - INFO - step: 61840 loss: 16.5456 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 14.0060 +[titan] 2025-07-10 03:12:12,906 - root - INFO - lr: 1.1746e-04 gnorm: 0.82 [13:37:11< 8:24:16] +[titan] 2025-07-10 03:12:16,825 - root - INFO - step: 61845 loss: 16.4822 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9580 +[titan] 2025-07-10 03:12:16,826 - root - INFO - lr: 1.1744e-04 gnorm: 0.86 [13:37:15< 8:24:12] +[titan] 2025-07-10 03:12:19,960 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:12:20,747 - root - INFO - step: 61850 loss: 16.2971 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.7989 +[titan] 2025-07-10 03:12:20,748 - root - INFO - lr: 1.1742e-04 gnorm: 0.84 [13:37:19< 8:24:08] +[titan] 2025-07-10 03:12:24,665 - root - INFO - step: 61855 loss: 16.5100 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9773 +[titan] 2025-07-10 03:12:24,665 - root - INFO - lr: 1.1740e-04 gnorm: 0.84 [13:37:23< 8:24:04] +[titan] 2025-07-10 03:12:28,572 - root - INFO - step: 61860 loss: 16.3791 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.8544 +[titan] 2025-07-10 03:12:28,572 - root - INFO - lr: 1.1738e-04 gnorm: 0.85 [13:37:27< 8:24:00] +[titan] 2025-07-10 03:12:32,474 - root - INFO - step: 61865 loss: 16.8509 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.6217 global_avg_mtp_loss: 14.2293 +[titan] 2025-07-10 03:12:32,474 - root - INFO - lr: 1.1736e-04 gnorm: 0.89 [13:37:31< 8:23:56] +[titan] 2025-07-10 03:12:36,415 - root - INFO - step: 61870 loss: 16.3031 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 03:12:36,415 - root - INFO - lr: 1.1734e-04 gnorm: 0.88 [13:37:35< 8:23:52] +[titan] 2025-07-10 03:12:40,338 - root - INFO - step: 61875 loss: 16.3647 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8478 +[titan] 2025-07-10 03:12:40,338 - root - INFO - lr: 1.1732e-04 gnorm: 0.89 [13:37:39< 8:23:48] +[titan] 2025-07-10 03:12:44,248 - root - INFO - step: 61880 loss: 16.5615 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 14.0092 +[titan] 2025-07-10 03:12:44,248 - root - INFO - lr: 1.1730e-04 gnorm: 0.87 [13:37:43< 8:23:44] +[titan] 2025-07-10 03:12:48,154 - root - INFO - step: 61885 loss: 16.6007 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5640 global_avg_mtp_loss: 14.0366 +[titan] 2025-07-10 03:12:48,154 - root - INFO - lr: 1.1728e-04 gnorm: 0.81 [13:37:47< 8:23:40] +[titan] 2025-07-10 03:12:52,101 - root - INFO - step: 61890 loss: 16.1746 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.6944 +[titan] 2025-07-10 03:12:52,101 - root - INFO - lr: 1.1726e-04 gnorm: 0.89 [13:37:51< 8:23:36] +[titan] 2025-07-10 03:12:56,017 - root - INFO - step: 61895 loss: 16.4864 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9514 +[titan] 2025-07-10 03:12:56,017 - root - INFO - lr: 1.1724e-04 gnorm: 0.90 [13:37:54< 8:23:32] +[titan] 2025-07-10 03:12:59,132 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:12:59,922 - root - INFO - step: 61900 loss: 16.4793 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9474 +[titan] 2025-07-10 03:12:59,922 - root - INFO - lr: 1.1722e-04 gnorm: 0.90 [13:37:58< 8:23:28] +[titan] 2025-07-10 03:13:03,828 - root - INFO - step: 61905 loss: 16.2694 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7840 +[titan] 2025-07-10 03:13:03,828 - root - INFO - lr: 1.1720e-04 gnorm: 0.98 [13:38:02< 8:23:24] +[titan] 2025-07-10 03:13:07,770 - root - INFO - step: 61910 loss: 16.5389 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5449 global_avg_mtp_loss: 13.9939 +[titan] 2025-07-10 03:13:07,770 - root - INFO - lr: 1.1718e-04 gnorm: 0.89 [13:38:06< 8:23:20] +[titan] 2025-07-10 03:13:11,688 - root - INFO - step: 61915 loss: 16.1983 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7285 +[titan] 2025-07-10 03:13:11,688 - root - INFO - lr: 1.1716e-04 gnorm: 0.88 [13:38:10< 8:23:16] +[titan] 2025-07-10 03:13:15,609 - root - INFO - step: 61920 loss: 16.7435 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5890 global_avg_mtp_loss: 14.1546 +[titan] 2025-07-10 03:13:15,609 - root - INFO - lr: 1.1714e-04 gnorm: 0.86 [13:38:14< 8:23:12] +[titan] 2025-07-10 03:13:19,765 - root - INFO - step: 61925 loss: 16.3995 memory: 44.58GiB(31.99%) tps: 78,852 tflops: 272.13 mfu: 27.52% global_avg_ntp_loss: 2.5183 global_avg_mtp_loss: 13.8812 +[titan] 2025-07-10 03:13:19,765 - root - INFO - lr: 1.1712e-04 gnorm: 0.90 [13:38:18< 8:23:08] +[titan] 2025-07-10 03:13:23,711 - root - INFO - step: 61930 loss: 16.6488 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5762 global_avg_mtp_loss: 14.0726 +[titan] 2025-07-10 03:13:23,712 - root - INFO - lr: 1.1710e-04 gnorm: 0.88 [13:38:22< 8:23:04] +[titan] 2025-07-10 03:13:27,650 - root - INFO - step: 61935 loss: 16.2435 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7577 +[titan] 2025-07-10 03:13:27,651 - root - INFO - lr: 1.1708e-04 gnorm: 0.89 [13:38:26< 8:23:00] +[titan] 2025-07-10 03:13:31,578 - root - INFO - step: 61940 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.8765 +[titan] 2025-07-10 03:13:31,578 - root - INFO - lr: 1.1706e-04 gnorm: 0.86 [13:38:30< 8:22:56] +[titan] 2025-07-10 03:13:35,507 - root - INFO - step: 61945 loss: 16.6509 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0875 +[titan] 2025-07-10 03:13:35,508 - root - INFO - lr: 1.1704e-04 gnorm: 0.83 [13:38:34< 8:22:52] +[titan] 2025-07-10 03:13:38,714 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:13:39,508 - root - INFO - step: 61950 loss: 16.2491 memory: 44.58GiB(31.99%) tps: 81,912 tflops: 282.69 mfu: 28.58% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7651 +[titan] 2025-07-10 03:13:39,508 - root - INFO - lr: 1.1702e-04 gnorm: 0.84 [13:38:38< 8:22:48] +[titan] 2025-07-10 03:13:41,241 - root - INFO - Dumping profiler traces at step 61952 +[titan] 2025-07-10 03:13:41,274 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:13:43,665 - root - INFO - step: 61955 loss: 16.4494 memory: 44.58GiB(31.99%) tps: 78,840 tflops: 272.09 mfu: 27.51% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9254 +[titan] 2025-07-10 03:13:43,665 - root - INFO - lr: 1.1700e-04 gnorm: 0.85 [13:38:42< 8:22:44] +[titan] 2025-07-10 03:13:47,619 - root - INFO - step: 61960 loss: 16.4229 memory: 44.58GiB(31.99%) tps: 82,868 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.9088 +[titan] 2025-07-10 03:13:47,620 - root - INFO - lr: 1.1698e-04 gnorm: 0.97 [13:38:46< 8:22:40] +[titan] 2025-07-10 03:13:51,518 - root - INFO - step: 61965 loss: 16.2561 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.7484 +[titan] 2025-07-10 03:13:51,518 - root - INFO - lr: 1.1696e-04 gnorm: 0.90 [13:38:50< 8:22:36] +[titan] 2025-07-10 03:13:55,436 - root - INFO - step: 61970 loss: 16.3686 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-10 03:13:55,437 - root - INFO - lr: 1.1694e-04 gnorm: 0.90 [13:38:54< 8:22:32] +[titan] 2025-07-10 03:13:59,349 - root - INFO - step: 61975 loss: 16.6010 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5679 global_avg_mtp_loss: 14.0331 +[titan] 2025-07-10 03:13:59,349 - root - INFO - lr: 1.1692e-04 gnorm: 0.87 [13:38:58< 8:22:28] +[titan] 2025-07-10 03:14:03,255 - root - INFO - step: 61980 loss: 16.3280 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8266 +[titan] 2025-07-10 03:14:03,255 - root - INFO - lr: 1.1690e-04 gnorm: 0.86 [13:39:02< 8:22:24] +[titan] 2025-07-10 03:14:07,176 - root - INFO - step: 61985 loss: 16.6236 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5619 global_avg_mtp_loss: 14.0617 +[titan] 2025-07-10 03:14:07,176 - root - INFO - lr: 1.1688e-04 gnorm: 0.82 [13:39:06< 8:22:20] +[titan] 2025-07-10 03:14:11,080 - root - INFO - step: 61990 loss: 16.6529 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5600 global_avg_mtp_loss: 14.0930 +[titan] 2025-07-10 03:14:11,080 - root - INFO - lr: 1.1686e-04 gnorm: 0.80 [13:39:10< 8:22:16] +[titan] 2025-07-10 03:14:15,057 - root - INFO - step: 61995 loss: 16.4916 memory: 44.58GiB(31.99%) tps: 82,401 tflops: 284.38 mfu: 28.75% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 13.9518 +[titan] 2025-07-10 03:14:15,057 - root - INFO - lr: 1.1684e-04 gnorm: 0.87 [13:39:13< 8:22:13] +[titan] 2025-07-10 03:14:18,175 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:14:18,969 - root - INFO - step: 62000 loss: 16.2125 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7367 +[titan] 2025-07-10 03:14:18,969 - root - INFO - lr: 1.1682e-04 gnorm: 0.84 [13:39:17< 8:22:09] +[titan] 2025-07-10 03:14:22,922 - root - INFO - step: 62005 loss: 16.1331 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4626 global_avg_mtp_loss: 13.6706 +[titan] 2025-07-10 03:14:22,922 - root - INFO - lr: 1.1680e-04 gnorm: 0.88 [13:39:21< 8:22:05] +[titan] 2025-07-10 03:14:26,866 - root - INFO - step: 62010 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8543 +[titan] 2025-07-10 03:14:26,866 - root - INFO - lr: 1.1678e-04 gnorm: 0.89 [13:39:25< 8:22:01] +[titan] 2025-07-10 03:14:30,770 - root - INFO - step: 62015 loss: 16.3680 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 03:14:30,771 - root - INFO - lr: 1.1676e-04 gnorm: 0.86 [13:39:29< 8:21:57] +[titan] 2025-07-10 03:14:34,715 - root - INFO - step: 62020 loss: 16.1599 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.6832 +[titan] 2025-07-10 03:14:34,715 - root - INFO - lr: 1.1674e-04 gnorm: 0.89 [13:39:33< 8:21:53] +[titan] 2025-07-10 03:14:38,632 - root - INFO - step: 62025 loss: 16.6113 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5617 global_avg_mtp_loss: 14.0496 +[titan] 2025-07-10 03:14:38,632 - root - INFO - lr: 1.1672e-04 gnorm: 0.86 [13:39:37< 8:21:49] +[titan] 2025-07-10 03:14:42,580 - root - INFO - step: 62030 loss: 16.4201 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 03:14:42,580 - root - INFO - lr: 1.1670e-04 gnorm: 0.85 [13:39:41< 8:21:45] +[titan] 2025-07-10 03:14:46,505 - root - INFO - step: 62035 loss: 16.5419 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 14.0004 +[titan] 2025-07-10 03:14:46,506 - root - INFO - lr: 1.1668e-04 gnorm: 0.89 [13:39:45< 8:21:41] +[titan] 2025-07-10 03:14:50,426 - root - INFO - step: 62040 loss: 16.2645 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.7689 +[titan] 2025-07-10 03:14:50,426 - root - INFO - lr: 1.1666e-04 gnorm: 0.87 [13:39:49< 8:21:37] +[titan] 2025-07-10 03:14:54,357 - root - INFO - step: 62045 loss: 16.5776 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5485 global_avg_mtp_loss: 14.0291 +[titan] 2025-07-10 03:14:54,357 - root - INFO - lr: 1.1664e-04 gnorm: 0.86 [13:39:53< 8:21:33] +[titan] 2025-07-10 03:14:57,504 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:14:58,295 - root - INFO - step: 62050 loss: 16.6629 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.1146 +[titan] 2025-07-10 03:14:58,295 - root - INFO - lr: 1.1662e-04 gnorm: 0.85 [13:39:57< 8:21:29] +[titan] 2025-07-10 03:15:02,223 - root - INFO - step: 62055 loss: 16.0999 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.6275 +[titan] 2025-07-10 03:15:02,223 - root - INFO - lr: 1.1660e-04 gnorm: 0.83 [13:40:01< 8:21:25] +[titan] 2025-07-10 03:15:06,167 - root - INFO - step: 62060 loss: 16.4820 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.9646 +[titan] 2025-07-10 03:15:06,167 - root - INFO - lr: 1.1658e-04 gnorm: 0.85 [13:40:05< 8:21:21] +[titan] 2025-07-10 03:15:10,081 - root - INFO - step: 62065 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-10 03:15:10,081 - root - INFO - lr: 1.1656e-04 gnorm: 0.87 [13:40:09< 8:21:17] +[titan] 2025-07-10 03:15:14,045 - root - INFO - step: 62070 loss: 16.5346 memory: 44.58GiB(31.99%) tps: 82,683 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.5490 global_avg_mtp_loss: 13.9856 +[titan] 2025-07-10 03:15:14,045 - root - INFO - lr: 1.1654e-04 gnorm: 0.88 [13:40:12< 8:21:13] +[titan] 2025-07-10 03:15:17,966 - root - INFO - step: 62075 loss: 16.3507 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-10 03:15:17,966 - root - INFO - lr: 1.1652e-04 gnorm: 0.88 [13:40:16< 8:21:09] +[titan] 2025-07-10 03:15:21,911 - root - INFO - step: 62080 loss: 16.3446 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.8415 +[titan] 2025-07-10 03:15:21,911 - root - INFO - lr: 1.1650e-04 gnorm: 0.87 [13:40:20< 8:21:05] +[titan] 2025-07-10 03:15:25,808 - root - INFO - step: 62085 loss: 16.2580 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.7629 +[titan] 2025-07-10 03:15:25,808 - root - INFO - lr: 1.1648e-04 gnorm: 0.84 [13:40:24< 8:21:01] +[titan] 2025-07-10 03:15:29,738 - root - INFO - step: 62090 loss: 16.2686 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.7655 +[titan] 2025-07-10 03:15:29,738 - root - INFO - lr: 1.1646e-04 gnorm: 0.87 [13:40:28< 8:20:57] +[titan] 2025-07-10 03:15:33,706 - root - INFO - step: 62095 loss: 16.4063 memory: 44.58GiB(31.99%) tps: 82,591 tflops: 285.04 mfu: 28.82% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8890 +[titan] 2025-07-10 03:15:33,706 - root - INFO - lr: 1.1644e-04 gnorm: 0.91 [13:40:32< 8:20:53] +[titan] 2025-07-10 03:15:36,868 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:15:37,651 - root - INFO - step: 62100 loss: 16.4009 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.8759 +[titan] 2025-07-10 03:15:37,651 - root - INFO - lr: 1.1642e-04 gnorm: 0.91 [13:40:36< 8:20:49] +[titan] 2025-07-10 03:15:41,567 - root - INFO - step: 62105 loss: 16.3441 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8509 +[titan] 2025-07-10 03:15:41,567 - root - INFO - lr: 1.1640e-04 gnorm: 0.88 [13:40:40< 8:20:45] +[titan] 2025-07-10 03:15:45,478 - root - INFO - step: 62110 loss: 16.1806 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7047 +[titan] 2025-07-10 03:15:45,478 - root - INFO - lr: 1.1638e-04 gnorm: 0.88 [13:40:44< 8:20:41] +[titan] 2025-07-10 03:15:49,394 - root - INFO - step: 62115 loss: 16.5220 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9894 +[titan] 2025-07-10 03:15:49,394 - root - INFO - lr: 1.1636e-04 gnorm: 0.88 [13:40:48< 8:20:37] +[titan] 2025-07-10 03:15:53,287 - root - INFO - step: 62120 loss: 16.3562 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8472 +[titan] 2025-07-10 03:15:53,287 - root - INFO - lr: 1.1634e-04 gnorm: 0.87 [13:40:52< 8:20:33] +[titan] 2025-07-10 03:15:57,216 - root - INFO - step: 62125 loss: 16.5300 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 14.0091 +[titan] 2025-07-10 03:15:57,216 - root - INFO - lr: 1.1632e-04 gnorm: 0.82 [13:40:56< 8:20:29] +[titan] 2025-07-10 03:16:01,176 - root - INFO - step: 62130 loss: 16.4799 memory: 44.58GiB(31.99%) tps: 82,757 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9560 +[titan] 2025-07-10 03:16:01,176 - root - INFO - lr: 1.1630e-04 gnorm: 0.94 [13:41:00< 8:20:25] +[titan] 2025-07-10 03:16:05,074 - root - INFO - step: 62135 loss: 16.3637 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.8672 +[titan] 2025-07-10 03:16:05,074 - root - INFO - lr: 1.1628e-04 gnorm: 0.81 [13:41:04< 8:20:21] +[titan] 2025-07-10 03:16:08,972 - root - INFO - step: 62140 loss: 16.2559 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7786 +[titan] 2025-07-10 03:16:08,973 - root - INFO - lr: 1.1626e-04 gnorm: 0.87 [13:41:07< 8:20:17] +[titan] 2025-07-10 03:16:12,883 - root - INFO - step: 62145 loss: 16.5029 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9690 +[titan] 2025-07-10 03:16:12,884 - root - INFO - lr: 1.1624e-04 gnorm: 0.83 [13:41:11< 8:20:13] +[titan] 2025-07-10 03:16:15,998 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:16:16,797 - root - INFO - step: 62150 loss: 16.2435 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.7524 +[titan] 2025-07-10 03:16:16,797 - root - INFO - lr: 1.1622e-04 gnorm: 0.88 [13:41:15< 8:20:09] +[titan] 2025-07-10 03:16:20,718 - root - INFO - step: 62155 loss: 16.4263 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.8961 +[titan] 2025-07-10 03:16:20,718 - root - INFO - lr: 1.1620e-04 gnorm: 0.87 [13:41:19< 8:20:05] +[titan] 2025-07-10 03:16:24,623 - root - INFO - step: 62160 loss: 16.4423 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.9244 +[titan] 2025-07-10 03:16:24,624 - root - INFO - lr: 1.1618e-04 gnorm: 0.94 [13:41:23< 8:20:01] +[titan] 2025-07-10 03:16:28,516 - root - INFO - step: 62165 loss: 16.5378 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.5407 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-10 03:16:28,516 - root - INFO - lr: 1.1616e-04 gnorm: 0.86 [13:41:27< 8:19:57] +[titan] 2025-07-10 03:16:32,436 - root - INFO - step: 62170 loss: 16.5273 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9903 +[titan] 2025-07-10 03:16:32,437 - root - INFO - lr: 1.1614e-04 gnorm: 0.84 [13:41:31< 8:19:53] +[titan] 2025-07-10 03:16:36,340 - root - INFO - step: 62175 loss: 16.6074 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0440 +[titan] 2025-07-10 03:16:36,340 - root - INFO - lr: 1.1612e-04 gnorm: 0.89 [13:41:35< 8:19:49] +[titan] 2025-07-10 03:16:40,231 - root - INFO - step: 62180 loss: 16.2113 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 03:16:40,232 - root - INFO - lr: 1.1610e-04 gnorm: 0.89 [13:41:39< 8:19:45] +[titan] 2025-07-10 03:16:44,155 - root - INFO - step: 62185 loss: 16.1269 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4571 global_avg_mtp_loss: 13.6699 +[titan] 2025-07-10 03:16:44,155 - root - INFO - lr: 1.1608e-04 gnorm: 0.88 [13:41:43< 8:19:41] +[titan] 2025-07-10 03:16:48,051 - root - INFO - step: 62190 loss: 16.4269 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 03:16:48,051 - root - INFO - lr: 1.1606e-04 gnorm: 0.83 [13:41:46< 8:19:37] +[titan] 2025-07-10 03:16:51,949 - root - INFO - step: 62195 loss: 16.5874 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 03:16:51,949 - root - INFO - lr: 1.1604e-04 gnorm: 0.88 [13:41:50< 8:19:33] +[titan] 2025-07-10 03:16:55,066 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:16:55,855 - root - INFO - step: 62200 loss: 16.0813 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6244 +[titan] 2025-07-10 03:16:55,855 - root - INFO - lr: 1.1602e-04 gnorm: 0.88 [13:41:54< 8:19:29] +[titan] 2025-07-10 03:16:59,777 - root - INFO - step: 62205 loss: 16.6240 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0648 +[titan] 2025-07-10 03:16:59,777 - root - INFO - lr: 1.1600e-04 gnorm: 0.85 [13:41:58< 8:19:25] +[titan] 2025-07-10 03:17:03,677 - root - INFO - step: 62210 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8214 +[titan] 2025-07-10 03:17:03,678 - root - INFO - lr: 1.1598e-04 gnorm: 0.86 [13:42:02< 8:19:21] +[titan] 2025-07-10 03:17:07,598 - root - INFO - step: 62215 loss: 16.2184 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7353 +[titan] 2025-07-10 03:17:07,598 - root - INFO - lr: 1.1596e-04 gnorm: 0.90 [13:42:06< 8:19:17] +[titan] 2025-07-10 03:17:11,512 - root - INFO - step: 62220 loss: 16.6259 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5519 global_avg_mtp_loss: 14.0740 +[titan] 2025-07-10 03:17:11,512 - root - INFO - lr: 1.1594e-04 gnorm: 0.80 [13:42:10< 8:19:13] +[titan] 2025-07-10 03:17:15,443 - root - INFO - step: 62225 loss: 16.0591 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.5951 +[titan] 2025-07-10 03:17:15,443 - root - INFO - lr: 1.1592e-04 gnorm: 0.85 [13:42:14< 8:19:09] +[titan] 2025-07-10 03:17:19,369 - root - INFO - step: 62230 loss: 16.4571 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9248 +[titan] 2025-07-10 03:17:19,369 - root - INFO - lr: 1.1590e-04 gnorm: 0.88 [13:42:18< 8:19:05] +[titan] 2025-07-10 03:17:23,282 - root - INFO - step: 62235 loss: 16.6616 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5663 global_avg_mtp_loss: 14.0953 +[titan] 2025-07-10 03:17:23,282 - root - INFO - lr: 1.1588e-04 gnorm: 0.86 [13:42:22< 8:19:01] +[titan] 2025-07-10 03:17:27,194 - root - INFO - step: 62240 loss: 16.3022 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8045 +[titan] 2025-07-10 03:17:27,194 - root - INFO - lr: 1.1586e-04 gnorm: 0.88 [13:42:26< 8:18:57] +[titan] 2025-07-10 03:17:31,129 - root - INFO - step: 62245 loss: 16.4410 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9050 +[titan] 2025-07-10 03:17:31,129 - root - INFO - lr: 1.1584e-04 gnorm: 0.92 [13:42:30< 8:18:53] +[titan] 2025-07-10 03:17:34,253 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:17:35,040 - root - INFO - step: 62250 loss: 16.4070 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8837 +[titan] 2025-07-10 03:17:35,040 - root - INFO - lr: 1.1582e-04 gnorm: 0.87 [13:42:33< 8:18:49] +[titan] 2025-07-10 03:17:38,953 - root - INFO - step: 62255 loss: 16.4773 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9387 +[titan] 2025-07-10 03:17:38,953 - root - INFO - lr: 1.1580e-04 gnorm: 0.84 [13:42:37< 8:18:45] +[titan] 2025-07-10 03:17:42,858 - root - INFO - step: 62260 loss: 16.2814 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 03:17:42,858 - root - INFO - lr: 1.1578e-04 gnorm: 0.96 [13:42:41< 8:18:41] +[titan] 2025-07-10 03:17:46,794 - root - INFO - step: 62265 loss: 15.9869 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.5301 +[titan] 2025-07-10 03:17:46,794 - root - INFO - lr: 1.1576e-04 gnorm: 0.86 [13:42:45< 8:18:37] +[titan] 2025-07-10 03:17:50,796 - root - INFO - step: 62270 loss: 16.2811 memory: 44.58GiB(31.99%) tps: 81,889 tflops: 282.61 mfu: 28.58% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.7873 +[titan] 2025-07-10 03:17:50,796 - root - INFO - lr: 1.1574e-04 gnorm: 0.91 [13:42:49< 8:18:33] +[titan] 2025-07-10 03:17:54,731 - root - INFO - step: 62275 loss: 16.3373 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-10 03:17:54,731 - root - INFO - lr: 1.1572e-04 gnorm: 0.85 [13:42:53< 8:18:29] +[titan] 2025-07-10 03:17:58,671 - root - INFO - step: 62280 loss: 16.5193 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5472 global_avg_mtp_loss: 13.9721 +[titan] 2025-07-10 03:17:58,672 - root - INFO - lr: 1.1570e-04 gnorm: 0.86 [13:42:57< 8:18:25] +[titan] 2025-07-10 03:18:02,607 - root - INFO - step: 62285 loss: 16.7104 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5936 global_avg_mtp_loss: 14.1167 +[titan] 2025-07-10 03:18:02,608 - root - INFO - lr: 1.1568e-04 gnorm: 0.90 [13:43:01< 8:18:21] +[titan] 2025-07-10 03:18:06,554 - root - INFO - step: 62290 loss: 16.6194 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5566 global_avg_mtp_loss: 14.0627 +[titan] 2025-07-10 03:18:06,554 - root - INFO - lr: 1.1566e-04 gnorm: 0.88 [13:43:05< 8:18:17] +[titan] 2025-07-10 03:18:10,494 - root - INFO - step: 62295 loss: 16.5998 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5582 global_avg_mtp_loss: 14.0416 +[titan] 2025-07-10 03:18:10,495 - root - INFO - lr: 1.1564e-04 gnorm: 0.94 [13:43:09< 8:18:13] +[titan] 2025-07-10 03:18:13,615 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:18:14,402 - root - INFO - step: 62300 loss: 16.3323 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8343 +[titan] 2025-07-10 03:18:14,402 - root - INFO - lr: 1.1562e-04 gnorm: 1.13 [13:43:13< 8:18:09] +[titan] 2025-07-10 03:18:18,315 - root - INFO - step: 62305 loss: 16.1231 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.6557 +[titan] 2025-07-10 03:18:18,315 - root - INFO - lr: 1.1560e-04 gnorm: 0.84 [13:43:17< 8:18:05] +[titan] 2025-07-10 03:18:22,268 - root - INFO - step: 62310 loss: 16.3603 memory: 44.58GiB(31.99%) tps: 82,887 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8545 +[titan] 2025-07-10 03:18:22,269 - root - INFO - lr: 1.1558e-04 gnorm: 0.88 [13:43:21< 8:18:01] +[titan] 2025-07-10 03:18:26,193 - root - INFO - step: 62315 loss: 16.6465 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0831 +[titan] 2025-07-10 03:18:26,194 - root - INFO - lr: 1.1556e-04 gnorm: 0.87 [13:43:25< 8:17:57] +[titan] 2025-07-10 03:18:30,116 - root - INFO - step: 62320 loss: 16.2630 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7846 +[titan] 2025-07-10 03:18:30,116 - root - INFO - lr: 1.1554e-04 gnorm: 0.86 [13:43:29< 8:17:53] +[titan] 2025-07-10 03:18:34,038 - root - INFO - step: 62325 loss: 16.2975 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.8129 +[titan] 2025-07-10 03:18:34,038 - root - INFO - lr: 1.1552e-04 gnorm: 0.86 [13:43:32< 8:17:49] +[titan] 2025-07-10 03:18:37,971 - root - INFO - step: 62330 loss: 16.2439 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7453 +[titan] 2025-07-10 03:18:37,971 - root - INFO - lr: 1.1550e-04 gnorm: 0.85 [13:43:36< 8:17:45] +[titan] 2025-07-10 03:18:41,886 - root - INFO - step: 62335 loss: 16.4235 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5918 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 03:18:41,886 - root - INFO - lr: 1.1548e-04 gnorm: 5.95 [13:43:40< 8:17:41] +[titan] 2025-07-10 03:18:45,788 - root - INFO - step: 62340 loss: 16.1545 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.6777 +[titan] 2025-07-10 03:18:45,789 - root - INFO - lr: 1.1546e-04 gnorm: 0.98 [13:43:44< 8:17:37] +[titan] 2025-07-10 03:18:49,732 - root - INFO - step: 62345 loss: 16.7007 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5694 global_avg_mtp_loss: 14.1313 +[titan] 2025-07-10 03:18:49,732 - root - INFO - lr: 1.1544e-04 gnorm: 0.93 [13:43:48< 8:17:33] +[titan] 2025-07-10 03:18:52,834 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:18:53,626 - root - INFO - step: 62350 loss: 16.1811 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.6893 +[titan] 2025-07-10 03:18:53,626 - root - INFO - lr: 1.1542e-04 gnorm: 0.97 [13:43:52< 8:17:29] +[titan] 2025-07-10 03:18:57,552 - root - INFO - step: 62355 loss: 16.4967 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9709 +[titan] 2025-07-10 03:18:57,552 - root - INFO - lr: 1.1540e-04 gnorm: 0.87 [13:43:56< 8:17:25] +[titan] 2025-07-10 03:19:01,460 - root - INFO - step: 62360 loss: 16.5155 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5458 global_avg_mtp_loss: 13.9697 +[titan] 2025-07-10 03:19:01,460 - root - INFO - lr: 1.1538e-04 gnorm: 0.89 [13:44:00< 8:17:21] +[titan] 2025-07-10 03:19:05,379 - root - INFO - step: 62365 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7932 +[titan] 2025-07-10 03:19:05,379 - root - INFO - lr: 1.1536e-04 gnorm: 0.84 [13:44:04< 8:17:17] +[titan] 2025-07-10 03:19:09,291 - root - INFO - step: 62370 loss: 16.3898 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 03:19:09,292 - root - INFO - lr: 1.1534e-04 gnorm: 0.84 [13:44:08< 8:17:13] +[titan] 2025-07-10 03:19:13,202 - root - INFO - step: 62375 loss: 16.2623 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7772 +[titan] 2025-07-10 03:19:13,202 - root - INFO - lr: 1.1532e-04 gnorm: 0.87 [13:44:12< 8:17:09] +[titan] 2025-07-10 03:19:17,186 - root - INFO - step: 62380 loss: 16.2836 memory: 44.58GiB(31.99%) tps: 82,265 tflops: 283.91 mfu: 28.71% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.8004 +[titan] 2025-07-10 03:19:17,186 - root - INFO - lr: 1.1530e-04 gnorm: 0.82 [13:44:16< 8:17:05] +[titan] 2025-07-10 03:19:21,115 - root - INFO - step: 62385 loss: 16.3379 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8312 +[titan] 2025-07-10 03:19:21,115 - root - INFO - lr: 1.1528e-04 gnorm: 0.82 [13:44:20< 8:17:01] +[titan] 2025-07-10 03:19:25,037 - root - INFO - step: 62390 loss: 16.3761 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-10 03:19:25,037 - root - INFO - lr: 1.1526e-04 gnorm: 0.89 [13:44:23< 8:16:57] +[titan] 2025-07-10 03:19:28,943 - root - INFO - step: 62395 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7119 +[titan] 2025-07-10 03:19:28,943 - root - INFO - lr: 1.1524e-04 gnorm: 0.85 [13:44:27< 8:16:53] +[titan] 2025-07-10 03:19:32,079 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:19:32,865 - root - INFO - step: 62400 loss: 16.5726 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.5531 global_avg_mtp_loss: 14.0196 +[titan] 2025-07-10 03:19:32,865 - root - INFO - lr: 1.1522e-04 gnorm: 0.85 [13:44:31< 8:16:49] +[titan] 2025-07-10 03:19:36,782 - root - INFO - step: 62405 loss: 16.4758 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.9523 +[titan] 2025-07-10 03:19:36,783 - root - INFO - lr: 1.1520e-04 gnorm: 0.91 [13:44:35< 8:16:45] +[titan] 2025-07-10 03:19:40,739 - root - INFO - step: 62410 loss: 16.4631 memory: 44.58GiB(31.99%) tps: 82,817 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.9301 +[titan] 2025-07-10 03:19:40,740 - root - INFO - lr: 1.1518e-04 gnorm: 0.86 [13:44:39< 8:16:41] +[titan] 2025-07-10 03:19:44,649 - root - INFO - step: 62415 loss: 16.3095 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 03:19:44,649 - root - INFO - lr: 1.1516e-04 gnorm: 0.86 [13:44:43< 8:16:37] +[titan] 2025-07-10 03:19:48,547 - root - INFO - step: 62420 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9019 +[titan] 2025-07-10 03:19:48,548 - root - INFO - lr: 1.1514e-04 gnorm: 0.86 [13:44:47< 8:16:33] +[titan] 2025-07-10 03:19:52,448 - root - INFO - step: 62425 loss: 16.5690 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 14.0306 +[titan] 2025-07-10 03:19:52,449 - root - INFO - lr: 1.1512e-04 gnorm: 0.88 [13:44:51< 8:16:29] +[titan] 2025-07-10 03:19:56,371 - root - INFO - step: 62430 loss: 16.2152 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7351 +[titan] 2025-07-10 03:19:56,371 - root - INFO - lr: 1.1510e-04 gnorm: 0.83 [13:44:55< 8:16:25] +[titan] 2025-07-10 03:20:00,287 - root - INFO - step: 62435 loss: 16.3347 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8377 +[titan] 2025-07-10 03:20:00,288 - root - INFO - lr: 1.1508e-04 gnorm: 0.87 [13:44:59< 8:16:21] +[titan] 2025-07-10 03:20:04,216 - root - INFO - step: 62440 loss: 16.3584 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.8388 +[titan] 2025-07-10 03:20:04,216 - root - INFO - lr: 1.1506e-04 gnorm: 0.88 [13:45:03< 8:16:17] +[titan] 2025-07-10 03:20:08,119 - root - INFO - step: 62445 loss: 16.4704 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9396 +[titan] 2025-07-10 03:20:08,119 - root - INFO - lr: 1.1504e-04 gnorm: 0.90 [13:45:07< 8:16:13] +[titan] 2025-07-10 03:20:11,232 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:20:12,029 - root - INFO - step: 62450 loss: 16.4841 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9330 +[titan] 2025-07-10 03:20:12,029 - root - INFO - lr: 1.1502e-04 gnorm: 0.88 [13:45:10< 8:16:09] +[titan] 2025-07-10 03:20:15,992 - root - INFO - step: 62455 loss: 16.3682 memory: 44.58GiB(31.99%) tps: 82,688 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8609 +[titan] 2025-07-10 03:20:15,993 - root - INFO - lr: 1.1500e-04 gnorm: 0.83 [13:45:14< 8:16:06] +[titan] 2025-07-10 03:20:19,919 - root - INFO - step: 62460 loss: 16.4363 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.9210 +[titan] 2025-07-10 03:20:19,919 - root - INFO - lr: 1.1498e-04 gnorm: 0.89 [13:45:18< 8:16:02] +[titan] 2025-07-10 03:20:23,331 - root - INFO - Dumping profiler traces at step 62464 +[titan] 2025-07-10 03:20:23,363 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:20:24,158 - root - INFO - step: 62465 loss: 16.4962 memory: 44.58GiB(31.99%) tps: 77,307 tflops: 266.80 mfu: 26.98% global_avg_ntp_loss: 2.5430 global_avg_mtp_loss: 13.9532 +[titan] 2025-07-10 03:20:24,158 - root - INFO - lr: 1.1496e-04 gnorm: 0.89 [13:45:23< 8:15:58] +[titan] 2025-07-10 03:20:28,081 - root - INFO - step: 62470 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8632 +[titan] 2025-07-10 03:20:28,081 - root - INFO - lr: 1.1494e-04 gnorm: 0.84 [13:45:26< 8:15:54] +[titan] 2025-07-10 03:20:32,019 - root - INFO - step: 62475 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 03:20:32,019 - root - INFO - lr: 1.1492e-04 gnorm: 0.87 [13:45:30< 8:15:50] +[titan] 2025-07-10 03:20:35,928 - root - INFO - step: 62480 loss: 16.6138 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0592 +[titan] 2025-07-10 03:20:35,928 - root - INFO - lr: 1.1490e-04 gnorm: 0.81 [13:45:34< 8:15:46] +[titan] 2025-07-10 03:20:39,847 - root - INFO - step: 62485 loss: 16.0413 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.5765 +[titan] 2025-07-10 03:20:39,847 - root - INFO - lr: 1.1488e-04 gnorm: 0.83 [13:45:38< 8:15:42] +[titan] 2025-07-10 03:20:43,785 - root - INFO - step: 62490 loss: 16.7196 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5797 global_avg_mtp_loss: 14.1399 +[titan] 2025-07-10 03:20:43,785 - root - INFO - lr: 1.1486e-04 gnorm: 0.85 [13:45:42< 8:15:38] +[titan] 2025-07-10 03:20:47,738 - root - INFO - step: 62495 loss: 16.3025 memory: 44.58GiB(31.99%) tps: 82,914 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8020 +[titan] 2025-07-10 03:20:47,738 - root - INFO - lr: 1.1484e-04 gnorm: 0.86 [13:45:46< 8:15:34] +[titan] 2025-07-10 03:20:50,860 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:20:51,661 - root - INFO - step: 62500 loss: 16.2712 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.7667 +[titan] 2025-07-10 03:20:51,661 - root - INFO - lr: 1.1482e-04 gnorm: 0.87 [13:45:50< 8:15:30] +[titan] 2025-07-10 03:20:55,566 - root - INFO - step: 62505 loss: 16.3476 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8423 +[titan] 2025-07-10 03:20:55,566 - root - INFO - lr: 1.1480e-04 gnorm: 0.89 [13:45:54< 8:15:26] +[titan] 2025-07-10 03:20:59,509 - root - INFO - step: 62510 loss: 16.3418 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8278 +[titan] 2025-07-10 03:20:59,509 - root - INFO - lr: 1.1478e-04 gnorm: 0.94 [13:45:58< 8:15:22] +[titan] 2025-07-10 03:21:03,481 - root - INFO - step: 62515 loss: 16.5812 memory: 44.58GiB(31.99%) tps: 82,495 tflops: 284.70 mfu: 28.79% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 14.0367 +[titan] 2025-07-10 03:21:03,481 - root - INFO - lr: 1.1476e-04 gnorm: 0.90 [13:46:02< 8:15:18] +[titan] 2025-07-10 03:21:07,416 - root - INFO - step: 62520 loss: 16.1749 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.6929 +[titan] 2025-07-10 03:21:07,416 - root - INFO - lr: 1.1474e-04 gnorm: 0.82 [13:46:06< 8:15:14] +[titan] 2025-07-10 03:21:11,341 - root - INFO - step: 62525 loss: 16.3148 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.8088 +[titan] 2025-07-10 03:21:11,341 - root - INFO - lr: 1.1472e-04 gnorm: 0.88 [13:46:10< 8:15:10] +[titan] 2025-07-10 03:21:15,284 - root - INFO - step: 62530 loss: 15.9616 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4368 global_avg_mtp_loss: 13.5248 +[titan] 2025-07-10 03:21:15,285 - root - INFO - lr: 1.1470e-04 gnorm: 0.92 [13:46:14< 8:15:06] +[titan] 2025-07-10 03:21:19,209 - root - INFO - step: 62535 loss: 16.3566 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8656 +[titan] 2025-07-10 03:21:19,210 - root - INFO - lr: 1.1468e-04 gnorm: 0.94 [13:46:18< 8:15:02] +[titan] 2025-07-10 03:21:23,152 - root - INFO - step: 62540 loss: 16.5014 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9625 +[titan] 2025-07-10 03:21:23,152 - root - INFO - lr: 1.1467e-04 gnorm: 0.95 [13:46:22< 8:14:58] +[titan] 2025-07-10 03:21:27,069 - root - INFO - step: 62545 loss: 15.9285 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.4823 +[titan] 2025-07-10 03:21:27,069 - root - INFO - lr: 1.1465e-04 gnorm: 0.91 [13:46:25< 8:14:54] +[titan] 2025-07-10 03:21:30,198 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:21:30,991 - root - INFO - step: 62550 loss: 16.1334 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6739 +[titan] 2025-07-10 03:21:30,991 - root - INFO - lr: 1.1463e-04 gnorm: 0.83 [13:46:29< 8:14:50] +[titan] 2025-07-10 03:21:34,936 - root - INFO - step: 62555 loss: 16.5707 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5502 global_avg_mtp_loss: 14.0205 +[titan] 2025-07-10 03:21:34,937 - root - INFO - lr: 1.1461e-04 gnorm: 0.89 [13:46:33< 8:14:46] +[titan] 2025-07-10 03:21:38,895 - root - INFO - step: 62560 loss: 16.2553 memory: 44.58GiB(31.99%) tps: 82,785 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7677 +[titan] 2025-07-10 03:21:38,895 - root - INFO - lr: 1.1459e-04 gnorm: 0.90 [13:46:37< 8:14:42] +[titan] 2025-07-10 03:21:42,811 - root - INFO - step: 62565 loss: 16.4924 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.9596 +[titan] 2025-07-10 03:21:42,812 - root - INFO - lr: 1.1457e-04 gnorm: 0.85 [13:46:41< 8:14:38] +[titan] 2025-07-10 03:21:46,724 - root - INFO - step: 62570 loss: 16.4118 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8954 +[titan] 2025-07-10 03:21:46,724 - root - INFO - lr: 1.1455e-04 gnorm: 0.85 [13:46:45< 8:14:34] +[titan] 2025-07-10 03:21:50,640 - root - INFO - step: 62575 loss: 16.1015 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6366 +[titan] 2025-07-10 03:21:50,640 - root - INFO - lr: 1.1453e-04 gnorm: 0.88 [13:46:49< 8:14:30] +[titan] 2025-07-10 03:21:54,593 - root - INFO - step: 62580 loss: 16.1405 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6692 +[titan] 2025-07-10 03:21:54,593 - root - INFO - lr: 1.1451e-04 gnorm: 0.88 [13:46:53< 8:14:26] +[titan] 2025-07-10 03:21:58,484 - root - INFO - step: 62585 loss: 16.6538 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0908 +[titan] 2025-07-10 03:21:58,484 - root - INFO - lr: 1.1449e-04 gnorm: 0.87 [13:46:57< 8:14:22] +[titan] 2025-07-10 03:22:02,412 - root - INFO - step: 62590 loss: 16.2268 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7371 +[titan] 2025-07-10 03:22:02,413 - root - INFO - lr: 1.1447e-04 gnorm: 0.81 [13:47:01< 8:14:18] +[titan] 2025-07-10 03:22:06,333 - root - INFO - step: 62595 loss: 16.4025 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 03:22:06,333 - root - INFO - lr: 1.1445e-04 gnorm: 0.87 [13:47:05< 8:14:14] +[titan] 2025-07-10 03:22:09,454 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:22:10,241 - root - INFO - step: 62600 loss: 16.3241 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8168 +[titan] 2025-07-10 03:22:10,241 - root - INFO - lr: 1.1443e-04 gnorm: 0.86 [13:47:09< 8:14:10] +[titan] 2025-07-10 03:22:14,167 - root - INFO - step: 62605 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9575 +[titan] 2025-07-10 03:22:14,167 - root - INFO - lr: 1.1441e-04 gnorm: 0.82 [13:47:13< 8:14:06] +[titan] 2025-07-10 03:22:18,097 - root - INFO - step: 62610 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9505 +[titan] 2025-07-10 03:22:18,098 - root - INFO - lr: 1.1439e-04 gnorm: 0.86 [13:47:16< 8:14:02] +[titan] 2025-07-10 03:22:21,992 - root - INFO - step: 62615 loss: 16.5559 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 13.9965 +[titan] 2025-07-10 03:22:21,992 - root - INFO - lr: 1.1437e-04 gnorm: 0.88 [13:47:20< 8:13:58] +[titan] 2025-07-10 03:22:25,907 - root - INFO - step: 62620 loss: 16.1789 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7054 +[titan] 2025-07-10 03:22:25,907 - root - INFO - lr: 1.1435e-04 gnorm: 0.89 [13:47:24< 8:13:54] +[titan] 2025-07-10 03:22:29,816 - root - INFO - step: 62625 loss: 16.3558 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.8428 +[titan] 2025-07-10 03:22:29,816 - root - INFO - lr: 1.1433e-04 gnorm: 0.87 [13:47:28< 8:13:50] +[titan] 2025-07-10 03:22:33,710 - root - INFO - step: 62630 loss: 16.5868 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0208 +[titan] 2025-07-10 03:22:33,710 - root - INFO - lr: 1.1431e-04 gnorm: 0.93 [13:47:32< 8:13:46] +[titan] 2025-07-10 03:22:37,626 - root - INFO - step: 62635 loss: 16.6427 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.0595 +[titan] 2025-07-10 03:22:37,626 - root - INFO - lr: 1.1429e-04 gnorm: 0.93 [13:47:36< 8:13:42] +[titan] 2025-07-10 03:22:41,530 - root - INFO - step: 62640 loss: 16.4974 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9590 +[titan] 2025-07-10 03:22:41,531 - root - INFO - lr: 1.1427e-04 gnorm: 0.93 [13:47:40< 8:13:38] +[titan] 2025-07-10 03:22:45,534 - root - INFO - step: 62645 loss: 16.5708 memory: 44.58GiB(31.99%) tps: 81,861 tflops: 282.52 mfu: 28.57% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 14.0382 +[titan] 2025-07-10 03:22:45,534 - root - INFO - lr: 1.1425e-04 gnorm: 0.89 [13:47:44< 8:13:34] +[titan] 2025-07-10 03:22:48,663 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:22:49,459 - root - INFO - step: 62650 loss: 16.4845 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-10 03:22:49,459 - root - INFO - lr: 1.1423e-04 gnorm: 0.86 [13:47:48< 8:13:30] +[titan] 2025-07-10 03:22:53,381 - root - INFO - step: 62655 loss: 16.4586 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9386 +[titan] 2025-07-10 03:22:53,381 - root - INFO - lr: 1.1421e-04 gnorm: 0.86 [13:47:52< 8:13:26] +[titan] 2025-07-10 03:22:57,368 - root - INFO - step: 62660 loss: 16.1606 memory: 44.58GiB(31.99%) tps: 82,188 tflops: 283.64 mfu: 28.68% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.6827 +[titan] 2025-07-10 03:22:57,368 - root - INFO - lr: 1.1419e-04 gnorm: 0.88 [13:47:56< 8:13:22] +[titan] 2025-07-10 03:23:01,286 - root - INFO - step: 62665 loss: 16.1445 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.6609 +[titan] 2025-07-10 03:23:01,287 - root - INFO - lr: 1.1417e-04 gnorm: 0.84 [13:48:00< 8:13:18] +[titan] 2025-07-10 03:23:05,207 - root - INFO - step: 62670 loss: 16.3749 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5111 global_avg_mtp_loss: 13.8638 +[titan] 2025-07-10 03:23:05,207 - root - INFO - lr: 1.1415e-04 gnorm: 0.85 [13:48:04< 8:13:14] +[titan] 2025-07-10 03:23:09,107 - root - INFO - step: 62675 loss: 16.3455 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8404 +[titan] 2025-07-10 03:23:09,107 - root - INFO - lr: 1.1413e-04 gnorm: 0.91 [13:48:07< 8:13:10] +[titan] 2025-07-10 03:23:13,015 - root - INFO - step: 62680 loss: 16.2785 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7857 +[titan] 2025-07-10 03:23:13,016 - root - INFO - lr: 1.1411e-04 gnorm: 0.87 [13:48:11< 8:13:06] +[titan] 2025-07-10 03:23:16,974 - root - INFO - step: 62685 loss: 16.4677 memory: 44.58GiB(31.99%) tps: 82,791 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9358 +[titan] 2025-07-10 03:23:16,974 - root - INFO - lr: 1.1409e-04 gnorm: 0.85 [13:48:15< 8:13:02] +[titan] 2025-07-10 03:23:20,898 - root - INFO - step: 62690 loss: 16.5327 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.9971 +[titan] 2025-07-10 03:23:20,898 - root - INFO - lr: 1.1407e-04 gnorm: 0.83 [13:48:19< 8:12:58] +[titan] 2025-07-10 03:23:24,820 - root - INFO - step: 62695 loss: 16.3131 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8219 +[titan] 2025-07-10 03:23:24,820 - root - INFO - lr: 1.1405e-04 gnorm: 0.89 [13:48:23< 8:12:54] +[titan] 2025-07-10 03:23:27,955 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:23:28,745 - root - INFO - step: 62700 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 03:23:28,745 - root - INFO - lr: 1.1403e-04 gnorm: 0.87 [13:48:27< 8:12:50] +[titan] 2025-07-10 03:23:32,718 - root - INFO - step: 62705 loss: 16.2323 memory: 44.58GiB(31.99%) tps: 82,480 tflops: 284.65 mfu: 28.78% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7543 +[titan] 2025-07-10 03:23:32,719 - root - INFO - lr: 1.1401e-04 gnorm: 0.86 [13:48:31< 8:12:46] +[titan] 2025-07-10 03:23:36,652 - root - INFO - step: 62710 loss: 16.5063 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9676 +[titan] 2025-07-10 03:23:36,652 - root - INFO - lr: 1.1399e-04 gnorm: 0.87 [13:48:35< 8:12:42] +[titan] 2025-07-10 03:23:40,588 - root - INFO - step: 62715 loss: 16.4896 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 13.9599 +[titan] 2025-07-10 03:23:40,588 - root - INFO - lr: 1.1397e-04 gnorm: 0.91 [13:48:39< 8:12:38] +[titan] 2025-07-10 03:23:44,498 - root - INFO - step: 62720 loss: 16.3722 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.8599 +[titan] 2025-07-10 03:23:44,498 - root - INFO - lr: 1.1395e-04 gnorm: 0.85 [13:48:43< 8:12:34] +[titan] 2025-07-10 03:23:48,410 - root - INFO - step: 62725 loss: 16.4919 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9679 +[titan] 2025-07-10 03:23:48,410 - root - INFO - lr: 1.1393e-04 gnorm: 0.84 [13:48:47< 8:12:30] +[titan] 2025-07-10 03:23:52,360 - root - INFO - step: 62730 loss: 16.5332 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9952 +[titan] 2025-07-10 03:23:52,361 - root - INFO - lr: 1.1391e-04 gnorm: 0.86 [13:48:51< 8:12:26] +[titan] 2025-07-10 03:23:56,283 - root - INFO - step: 62735 loss: 16.4509 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9300 +[titan] 2025-07-10 03:23:56,283 - root - INFO - lr: 1.1389e-04 gnorm: 0.84 [13:48:55< 8:12:23] +[titan] 2025-07-10 03:24:00,204 - root - INFO - step: 62740 loss: 16.3582 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5149 global_avg_mtp_loss: 13.8433 +[titan] 2025-07-10 03:24:00,204 - root - INFO - lr: 1.1387e-04 gnorm: 0.89 [13:48:59< 8:12:19] +[titan] 2025-07-10 03:24:04,097 - root - INFO - step: 62745 loss: 16.6686 memory: 44.58GiB(31.99%) tps: 84,172 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5675 global_avg_mtp_loss: 14.1011 +[titan] 2025-07-10 03:24:04,097 - root - INFO - lr: 1.1385e-04 gnorm: 0.88 [13:49:02< 8:12:15] +[titan] 2025-07-10 03:24:07,219 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:24:08,006 - root - INFO - step: 62750 loss: 16.6366 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5724 global_avg_mtp_loss: 14.0642 +[titan] 2025-07-10 03:24:08,006 - root - INFO - lr: 1.1383e-04 gnorm: 0.91 [13:49:06< 8:12:11] +[titan] 2025-07-10 03:24:11,918 - root - INFO - step: 62755 loss: 16.3556 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8506 +[titan] 2025-07-10 03:24:11,918 - root - INFO - lr: 1.1381e-04 gnorm: 0.96 [13:49:10< 8:12:07] +[titan] 2025-07-10 03:24:15,827 - root - INFO - step: 62760 loss: 16.0336 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4502 global_avg_mtp_loss: 13.5834 +[titan] 2025-07-10 03:24:15,828 - root - INFO - lr: 1.1379e-04 gnorm: 0.83 [13:49:14< 8:12:03] +[titan] 2025-07-10 03:24:19,745 - root - INFO - step: 62765 loss: 16.3567 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8495 +[titan] 2025-07-10 03:24:19,746 - root - INFO - lr: 1.1377e-04 gnorm: 0.87 [13:49:18< 8:11:59] +[titan] 2025-07-10 03:24:23,725 - root - INFO - step: 62770 loss: 16.3370 memory: 44.58GiB(31.99%) tps: 82,340 tflops: 284.17 mfu: 28.73% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8345 +[titan] 2025-07-10 03:24:23,726 - root - INFO - lr: 1.1375e-04 gnorm: 0.93 [13:49:22< 8:11:55] +[titan] 2025-07-10 03:24:27,648 - root - INFO - step: 62775 loss: 16.0374 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.5714 +[titan] 2025-07-10 03:24:27,648 - root - INFO - lr: 1.1373e-04 gnorm: 0.87 [13:49:26< 8:11:51] +[titan] 2025-07-10 03:24:31,566 - root - INFO - step: 62780 loss: 16.4828 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-10 03:24:31,567 - root - INFO - lr: 1.1371e-04 gnorm: 0.86 [13:49:30< 8:11:47] +[titan] 2025-07-10 03:24:35,505 - root - INFO - step: 62785 loss: 16.4501 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.9315 +[titan] 2025-07-10 03:24:35,505 - root - INFO - lr: 1.1369e-04 gnorm: 0.88 [13:49:34< 8:11:43] +[titan] 2025-07-10 03:24:39,413 - root - INFO - step: 62790 loss: 16.1797 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.6924 +[titan] 2025-07-10 03:24:39,413 - root - INFO - lr: 1.1367e-04 gnorm: 0.83 [13:49:38< 8:11:39] +[titan] 2025-07-10 03:24:43,320 - root - INFO - step: 62795 loss: 16.3103 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8071 +[titan] 2025-07-10 03:24:43,320 - root - INFO - lr: 1.1365e-04 gnorm: 0.89 [13:49:42< 8:11:35] +[titan] 2025-07-10 03:24:46,445 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:24:47,248 - root - INFO - step: 62800 loss: 16.2332 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7363 +[titan] 2025-07-10 03:24:47,248 - root - INFO - lr: 1.1363e-04 gnorm: 0.83 [13:49:46< 8:11:31] +[titan] 2025-07-10 03:24:51,192 - root - INFO - step: 62805 loss: 16.6015 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5577 global_avg_mtp_loss: 14.0438 +[titan] 2025-07-10 03:24:51,192 - root - INFO - lr: 1.1361e-04 gnorm: 0.86 [13:49:50< 8:11:27] +[titan] 2025-07-10 03:24:55,167 - root - INFO - step: 62810 loss: 16.1683 memory: 44.58GiB(31.99%) tps: 82,444 tflops: 284.53 mfu: 28.77% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.6845 +[titan] 2025-07-10 03:24:55,167 - root - INFO - lr: 1.1359e-04 gnorm: 0.89 [13:49:54< 8:11:23] +[titan] 2025-07-10 03:24:59,123 - root - INFO - step: 62815 loss: 16.5668 memory: 44.58GiB(31.99%) tps: 82,842 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 14.0191 +[titan] 2025-07-10 03:24:59,123 - root - INFO - lr: 1.1357e-04 gnorm: 0.91 [13:49:57< 8:11:19] +[titan] 2025-07-10 03:25:03,040 - root - INFO - step: 62820 loss: 16.5163 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.9870 +[titan] 2025-07-10 03:25:03,040 - root - INFO - lr: 1.1355e-04 gnorm: 0.84 [13:50:01< 8:11:15] +[titan] 2025-07-10 03:25:06,956 - root - INFO - step: 62825 loss: 16.8331 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5936 global_avg_mtp_loss: 14.2395 +[titan] 2025-07-10 03:25:06,956 - root - INFO - lr: 1.1353e-04 gnorm: 0.87 [13:50:05< 8:11:11] +[titan] 2025-07-10 03:25:10,911 - root - INFO - step: 62830 loss: 16.4454 memory: 44.58GiB(31.99%) tps: 82,868 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9207 +[titan] 2025-07-10 03:25:10,911 - root - INFO - lr: 1.1351e-04 gnorm: 0.91 [13:50:09< 8:11:07] +[titan] 2025-07-10 03:25:14,865 - root - INFO - step: 62835 loss: 16.4512 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9212 +[titan] 2025-07-10 03:25:14,865 - root - INFO - lr: 1.1349e-04 gnorm: 0.87 [13:50:13< 8:11:03] +[titan] 2025-07-10 03:25:18,763 - root - INFO - step: 62840 loss: 16.3751 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8748 +[titan] 2025-07-10 03:25:18,763 - root - INFO - lr: 1.1347e-04 gnorm: 0.88 [13:50:17< 8:10:59] +[titan] 2025-07-10 03:25:22,696 - root - INFO - step: 62845 loss: 16.5246 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9835 +[titan] 2025-07-10 03:25:22,697 - root - INFO - lr: 1.1346e-04 gnorm: 0.86 [13:50:21< 8:10:55] +[titan] 2025-07-10 03:25:25,813 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:25:26,602 - root - INFO - step: 62850 loss: 16.5976 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5552 global_avg_mtp_loss: 14.0424 +[titan] 2025-07-10 03:25:26,602 - root - INFO - lr: 1.1344e-04 gnorm: 0.87 [13:50:25< 8:10:51] +[titan] 2025-07-10 03:25:30,549 - root - INFO - step: 62855 loss: 16.0574 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.5965 +[titan] 2025-07-10 03:25:30,550 - root - INFO - lr: 1.1342e-04 gnorm: 0.87 [13:50:29< 8:10:47] +[titan] 2025-07-10 03:25:34,515 - root - INFO - step: 62860 loss: 16.4034 memory: 44.58GiB(31.99%) tps: 82,642 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8942 +[titan] 2025-07-10 03:25:34,515 - root - INFO - lr: 1.1340e-04 gnorm: 0.93 [13:50:33< 8:10:43] +[titan] 2025-07-10 03:25:38,430 - root - INFO - step: 62865 loss: 16.3402 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8370 +[titan] 2025-07-10 03:25:38,430 - root - INFO - lr: 1.1338e-04 gnorm: 0.85 [13:50:37< 8:10:39] +[titan] 2025-07-10 03:25:42,374 - root - INFO - step: 62870 loss: 16.4436 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5239 global_avg_mtp_loss: 13.9197 +[titan] 2025-07-10 03:25:42,374 - root - INFO - lr: 1.1336e-04 gnorm: 0.97 [13:50:41< 8:10:35] +[titan] 2025-07-10 03:25:46,312 - root - INFO - step: 62875 loss: 16.4895 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.9606 +[titan] 2025-07-10 03:25:46,312 - root - INFO - lr: 1.1334e-04 gnorm: 0.90 [13:50:45< 8:10:31] +[titan] 2025-07-10 03:25:50,245 - root - INFO - step: 62880 loss: 16.5824 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 14.0398 +[titan] 2025-07-10 03:25:50,246 - root - INFO - lr: 1.1332e-04 gnorm: 0.90 [13:50:49< 8:10:27] +[titan] 2025-07-10 03:25:54,168 - root - INFO - step: 62885 loss: 16.5393 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 14.0057 +[titan] 2025-07-10 03:25:54,169 - root - INFO - lr: 1.1330e-04 gnorm: 0.94 [13:50:53< 8:10:23] +[titan] 2025-07-10 03:25:58,115 - root - INFO - step: 62890 loss: 16.4966 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 13.9609 +[titan] 2025-07-10 03:25:58,115 - root - INFO - lr: 1.1328e-04 gnorm: 0.91 [13:50:56< 8:10:19] +[titan] 2025-07-10 03:26:02,026 - root - INFO - step: 62895 loss: 16.2044 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7248 +[titan] 2025-07-10 03:26:02,026 - root - INFO - lr: 1.1326e-04 gnorm: 0.89 [13:51:00< 8:10:15] +[titan] 2025-07-10 03:26:05,156 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:26:05,947 - root - INFO - step: 62900 loss: 16.3812 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8749 +[titan] 2025-07-10 03:26:05,947 - root - INFO - lr: 1.1324e-04 gnorm: 0.85 [13:51:04< 8:10:11] +[titan] 2025-07-10 03:26:09,875 - root - INFO - step: 62905 loss: 16.4933 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9481 +[titan] 2025-07-10 03:26:09,875 - root - INFO - lr: 1.1322e-04 gnorm: 0.89 [13:51:08< 8:10:07] +[titan] 2025-07-10 03:26:13,792 - root - INFO - step: 62910 loss: 16.4083 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.8931 +[titan] 2025-07-10 03:26:13,792 - root - INFO - lr: 1.1320e-04 gnorm: 0.88 [13:51:12< 8:10:03] +[titan] 2025-07-10 03:26:17,734 - root - INFO - step: 62915 loss: 16.2909 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.7732 +[titan] 2025-07-10 03:26:17,734 - root - INFO - lr: 1.1318e-04 gnorm: 0.98 [13:51:16< 8:09:59] +[titan] 2025-07-10 03:26:21,667 - root - INFO - step: 62920 loss: 16.2801 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.7753 +[titan] 2025-07-10 03:26:21,667 - root - INFO - lr: 1.1316e-04 gnorm: 0.89 [13:51:20< 8:09:55] +[titan] 2025-07-10 03:26:25,622 - root - INFO - step: 62925 loss: 16.3885 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 03:26:25,622 - root - INFO - lr: 1.1314e-04 gnorm: 0.86 [13:51:24< 8:09:51] +[titan] 2025-07-10 03:26:29,552 - root - INFO - step: 62930 loss: 16.1266 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4654 global_avg_mtp_loss: 13.6612 +[titan] 2025-07-10 03:26:29,552 - root - INFO - lr: 1.1312e-04 gnorm: 0.90 [13:51:28< 8:09:47] +[titan] 2025-07-10 03:26:33,466 - root - INFO - step: 62935 loss: 16.5245 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.9904 +[titan] 2025-07-10 03:26:33,466 - root - INFO - lr: 1.1310e-04 gnorm: 0.86 [13:51:32< 8:09:43] +[titan] 2025-07-10 03:26:37,375 - root - INFO - step: 62940 loss: 16.2604 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7769 +[titan] 2025-07-10 03:26:37,375 - root - INFO - lr: 1.1308e-04 gnorm: 0.87 [13:51:36< 8:09:39] +[titan] 2025-07-10 03:26:41,305 - root - INFO - step: 62945 loss: 16.3437 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8315 +[titan] 2025-07-10 03:26:41,306 - root - INFO - lr: 1.1306e-04 gnorm: 0.91 [13:51:40< 8:09:35] +[titan] 2025-07-10 03:26:44,433 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:26:45,220 - root - INFO - step: 62950 loss: 16.3787 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8598 +[titan] 2025-07-10 03:26:45,220 - root - INFO - lr: 1.1304e-04 gnorm: 0.89 [13:51:44< 8:09:31] +[titan] 2025-07-10 03:26:49,123 - root - INFO - step: 62955 loss: 16.1872 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.6949 +[titan] 2025-07-10 03:26:49,124 - root - INFO - lr: 1.1302e-04 gnorm: 0.86 [13:51:47< 8:09:27] +[titan] 2025-07-10 03:26:53,027 - root - INFO - step: 62960 loss: 16.6236 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5689 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-10 03:26:53,027 - root - INFO - lr: 1.1300e-04 gnorm: 0.85 [13:51:51< 8:09:23] +[titan] 2025-07-10 03:26:56,940 - root - INFO - step: 62965 loss: 16.5444 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5574 global_avg_mtp_loss: 13.9870 +[titan] 2025-07-10 03:26:56,940 - root - INFO - lr: 1.1298e-04 gnorm: 0.87 [13:51:55< 8:09:19] +[titan] 2025-07-10 03:27:00,840 - root - INFO - step: 62970 loss: 16.3495 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8494 +[titan] 2025-07-10 03:27:00,840 - root - INFO - lr: 1.1296e-04 gnorm: 0.89 [13:51:59< 8:09:15] +[titan] 2025-07-10 03:27:04,822 - root - INFO - step: 62975 loss: 16.4579 memory: 44.58GiB(31.99%) tps: 82,291 tflops: 284.00 mfu: 28.72% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.9416 +[titan] 2025-07-10 03:27:04,823 - root - INFO - lr: 1.1294e-04 gnorm: 0.87 [13:52:03< 8:09:11] +[titan] 2025-07-10 03:27:05,763 - root - INFO - Dumping profiler traces at step 62976 +[titan] 2025-07-10 03:27:05,796 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:27:08,917 - root - INFO - step: 62980 loss: 16.4310 memory: 44.58GiB(31.99%) tps: 80,030 tflops: 276.20 mfu: 27.93% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 03:27:08,918 - root - INFO - lr: 1.1292e-04 gnorm: 0.83 [13:52:07< 8:09:07] +[titan] 2025-07-10 03:27:12,818 - root - INFO - step: 62985 loss: 16.1090 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.6355 +[titan] 2025-07-10 03:27:12,818 - root - INFO - lr: 1.1290e-04 gnorm: 0.90 [13:52:11< 8:09:03] +[titan] 2025-07-10 03:27:16,771 - root - INFO - step: 62990 loss: 16.2749 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.7759 +[titan] 2025-07-10 03:27:16,771 - root - INFO - lr: 1.1288e-04 gnorm: 0.89 [13:52:15< 8:08:59] +[titan] 2025-07-10 03:27:20,685 - root - INFO - step: 62995 loss: 16.2793 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.7804 +[titan] 2025-07-10 03:27:20,685 - root - INFO - lr: 1.1286e-04 gnorm: 0.88 [13:52:19< 8:08:55] +[titan] 2025-07-10 03:27:23,801 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:27:24,591 - root - INFO - step: 63000 loss: 16.3995 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8806 +[titan] 2025-07-10 03:27:24,591 - root - INFO - lr: 1.1284e-04 gnorm: 0.91 [13:52:23< 8:08:51] +[titan] 2025-07-10 03:27:28,504 - root - INFO - step: 63005 loss: 16.2156 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7397 +[titan] 2025-07-10 03:27:28,504 - root - INFO - lr: 1.1282e-04 gnorm: 0.87 [13:52:27< 8:08:47] +[titan] 2025-07-10 03:27:32,410 - root - INFO - step: 63010 loss: 16.3549 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5111 global_avg_mtp_loss: 13.8438 +[titan] 2025-07-10 03:27:32,410 - root - INFO - lr: 1.1280e-04 gnorm: 0.97 [13:52:31< 8:08:43] +[titan] 2025-07-10 03:27:36,316 - root - INFO - step: 63015 loss: 16.2178 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4930 global_avg_mtp_loss: 13.7249 +[titan] 2025-07-10 03:27:36,316 - root - INFO - lr: 1.1278e-04 gnorm: 0.87 [13:52:35< 8:08:39] +[titan] 2025-07-10 03:27:40,261 - root - INFO - step: 63020 loss: 16.4676 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.9421 +[titan] 2025-07-10 03:27:40,262 - root - INFO - lr: 1.1276e-04 gnorm: 0.89 [13:52:39< 8:08:35] +[titan] 2025-07-10 03:27:44,189 - root - INFO - step: 63025 loss: 16.4609 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-10 03:27:44,189 - root - INFO - lr: 1.1274e-04 gnorm: 0.93 [13:52:43< 8:08:31] +[titan] 2025-07-10 03:27:48,092 - root - INFO - step: 63030 loss: 16.3294 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8347 +[titan] 2025-07-10 03:27:48,093 - root - INFO - lr: 1.1272e-04 gnorm: 0.92 [13:52:46< 8:08:27] +[titan] 2025-07-10 03:27:51,992 - root - INFO - step: 63035 loss: 16.4021 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 03:27:51,992 - root - INFO - lr: 1.1270e-04 gnorm: 1.42 [13:52:50< 8:08:23] +[titan] 2025-07-10 03:27:55,921 - root - INFO - step: 63040 loss: 16.4080 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.8832 +[titan] 2025-07-10 03:27:55,921 - root - INFO - lr: 1.1268e-04 gnorm: 0.83 [13:52:54< 8:08:19] +[titan] 2025-07-10 03:27:59,823 - root - INFO - step: 63045 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.7865 +[titan] 2025-07-10 03:27:59,823 - root - INFO - lr: 1.1266e-04 gnorm: 0.84 [13:52:58< 8:08:15] +[titan] 2025-07-10 03:28:02,947 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:28:03,734 - root - INFO - step: 63050 loss: 16.1208 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.6492 +[titan] 2025-07-10 03:28:03,734 - root - INFO - lr: 1.1264e-04 gnorm: 0.88 [13:53:02< 8:08:11] +[titan] 2025-07-10 03:28:07,664 - root - INFO - step: 63055 loss: 16.3492 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 03:28:07,664 - root - INFO - lr: 1.1262e-04 gnorm: 0.87 [13:53:06< 8:08:07] +[titan] 2025-07-10 03:28:11,580 - root - INFO - step: 63060 loss: 15.9615 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4305 global_avg_mtp_loss: 13.5310 +[titan] 2025-07-10 03:28:11,580 - root - INFO - lr: 1.1260e-04 gnorm: 0.86 [13:53:10< 8:08:03] +[titan] 2025-07-10 03:28:15,493 - root - INFO - step: 63065 loss: 16.2947 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.8089 +[titan] 2025-07-10 03:28:15,493 - root - INFO - lr: 1.1259e-04 gnorm: 0.92 [13:53:14< 8:07:59] +[titan] 2025-07-10 03:28:19,400 - root - INFO - step: 63070 loss: 16.4191 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.8975 +[titan] 2025-07-10 03:28:19,401 - root - INFO - lr: 1.1257e-04 gnorm: 0.94 [13:53:18< 8:07:55] +[titan] 2025-07-10 03:28:23,306 - root - INFO - step: 63075 loss: 16.2298 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7427 +[titan] 2025-07-10 03:28:23,306 - root - INFO - lr: 1.1255e-04 gnorm: 0.87 [13:53:22< 8:07:51] +[titan] 2025-07-10 03:28:27,214 - root - INFO - step: 63080 loss: 16.5037 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9736 +[titan] 2025-07-10 03:28:27,215 - root - INFO - lr: 1.1253e-04 gnorm: 0.90 [13:53:26< 8:07:47] +[titan] 2025-07-10 03:28:31,116 - root - INFO - step: 63085 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9610 +[titan] 2025-07-10 03:28:31,116 - root - INFO - lr: 1.1251e-04 gnorm: 0.84 [13:53:29< 8:07:43] +[titan] 2025-07-10 03:28:35,069 - root - INFO - step: 63090 loss: 16.2901 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.7921 +[titan] 2025-07-10 03:28:35,069 - root - INFO - lr: 1.1249e-04 gnorm: 0.90 [13:53:33< 8:07:39] +[titan] 2025-07-10 03:28:38,988 - root - INFO - step: 63095 loss: 16.4967 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9488 +[titan] 2025-07-10 03:28:38,988 - root - INFO - lr: 1.1247e-04 gnorm: 0.83 [13:53:37< 8:07:36] +[titan] 2025-07-10 03:28:42,104 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:28:42,893 - root - INFO - step: 63100 loss: 16.1238 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6648 +[titan] 2025-07-10 03:28:42,894 - root - INFO - lr: 1.1245e-04 gnorm: 0.87 [13:53:41< 8:07:32] +[titan] 2025-07-10 03:28:46,797 - root - INFO - step: 63105 loss: 16.5569 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 14.0073 +[titan] 2025-07-10 03:28:46,797 - root - INFO - lr: 1.1243e-04 gnorm: 0.91 [13:53:45< 8:07:28] +[titan] 2025-07-10 03:28:50,721 - root - INFO - step: 63110 loss: 16.1286 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4494 global_avg_mtp_loss: 13.6792 +[titan] 2025-07-10 03:28:50,721 - root - INFO - lr: 1.1241e-04 gnorm: 0.88 [13:53:49< 8:07:24] +[titan] 2025-07-10 03:28:54,625 - root - INFO - step: 63115 loss: 16.4098 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-10 03:28:54,625 - root - INFO - lr: 1.1239e-04 gnorm: 0.94 [13:53:53< 8:07:20] +[titan] 2025-07-10 03:28:58,531 - root - INFO - step: 63120 loss: 16.4672 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9412 +[titan] 2025-07-10 03:28:58,532 - root - INFO - lr: 1.1237e-04 gnorm: 0.90 [13:53:57< 8:07:16] +[titan] 2025-07-10 03:29:02,457 - root - INFO - step: 63125 loss: 16.1298 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6637 +[titan] 2025-07-10 03:29:02,457 - root - INFO - lr: 1.1235e-04 gnorm: 0.93 [13:54:01< 8:07:12] +[titan] 2025-07-10 03:29:06,370 - root - INFO - step: 63130 loss: 15.8241 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4206 global_avg_mtp_loss: 13.4034 +[titan] 2025-07-10 03:29:06,370 - root - INFO - lr: 1.1233e-04 gnorm: 0.84 [13:54:05< 8:07:08] +[titan] 2025-07-10 03:29:10,286 - root - INFO - step: 63135 loss: 16.1887 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7130 +[titan] 2025-07-10 03:29:10,286 - root - INFO - lr: 1.1231e-04 gnorm: 0.88 [13:54:09< 8:07:04] +[titan] 2025-07-10 03:29:14,232 - root - INFO - step: 63140 loss: 16.5101 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5378 global_avg_mtp_loss: 13.9724 +[titan] 2025-07-10 03:29:14,233 - root - INFO - lr: 1.1229e-04 gnorm: 0.87 [13:54:13< 8:07:00] +[titan] 2025-07-10 03:29:18,136 - root - INFO - step: 63145 loss: 16.3641 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-10 03:29:18,137 - root - INFO - lr: 1.1227e-04 gnorm: 0.87 [13:54:16< 8:06:56] +[titan] 2025-07-10 03:29:21,259 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:29:22,055 - root - INFO - step: 63150 loss: 16.0398 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.5810 +[titan] 2025-07-10 03:29:22,056 - root - INFO - lr: 1.1225e-04 gnorm: 0.86 [13:54:20< 8:06:52] +[titan] 2025-07-10 03:29:25,970 - root - INFO - step: 63155 loss: 16.4427 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9155 +[titan] 2025-07-10 03:29:25,970 - root - INFO - lr: 1.1223e-04 gnorm: 0.83 [13:54:24< 8:06:48] +[titan] 2025-07-10 03:29:29,893 - root - INFO - step: 63160 loss: 16.5218 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9812 +[titan] 2025-07-10 03:29:29,893 - root - INFO - lr: 1.1221e-04 gnorm: 0.84 [13:54:28< 8:06:44] +[titan] 2025-07-10 03:29:33,795 - root - INFO - step: 63165 loss: 16.3566 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8506 +[titan] 2025-07-10 03:29:33,795 - root - INFO - lr: 1.1219e-04 gnorm: 0.86 [13:54:32< 8:06:40] +[titan] 2025-07-10 03:29:37,700 - root - INFO - step: 63170 loss: 16.2787 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.7789 +[titan] 2025-07-10 03:29:37,700 - root - INFO - lr: 1.1217e-04 gnorm: 0.87 [13:54:36< 8:06:36] +[titan] 2025-07-10 03:29:41,612 - root - INFO - step: 63175 loss: 16.5562 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0067 +[titan] 2025-07-10 03:29:41,612 - root - INFO - lr: 1.1215e-04 gnorm: 0.85 [13:54:40< 8:06:32] +[titan] 2025-07-10 03:29:45,517 - root - INFO - step: 63180 loss: 16.2169 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7301 +[titan] 2025-07-10 03:29:45,517 - root - INFO - lr: 1.1213e-04 gnorm: 0.88 [13:54:44< 8:06:28] +[titan] 2025-07-10 03:29:49,434 - root - INFO - step: 63185 loss: 16.5083 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9703 +[titan] 2025-07-10 03:29:49,435 - root - INFO - lr: 1.1211e-04 gnorm: 0.82 [13:54:48< 8:06:24] +[titan] 2025-07-10 03:29:53,335 - root - INFO - step: 63190 loss: 16.5080 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9676 +[titan] 2025-07-10 03:29:53,335 - root - INFO - lr: 1.1209e-04 gnorm: 0.88 [13:54:52< 8:06:20] +[titan] 2025-07-10 03:29:57,250 - root - INFO - step: 63195 loss: 16.2778 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.7822 +[titan] 2025-07-10 03:29:57,250 - root - INFO - lr: 1.1207e-04 gnorm: 0.86 [13:54:56< 8:06:16] +[titan] 2025-07-10 03:30:00,380 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:30:01,188 - root - INFO - step: 63200 loss: 16.1135 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.6451 +[titan] 2025-07-10 03:30:01,188 - root - INFO - lr: 1.1205e-04 gnorm: 0.88 [13:55:00< 8:06:12] +[titan] 2025-07-10 03:30:05,102 - root - INFO - step: 63205 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8043 +[titan] 2025-07-10 03:30:05,102 - root - INFO - lr: 1.1203e-04 gnorm: 0.91 [13:55:03< 8:06:08] +[titan] 2025-07-10 03:30:09,012 - root - INFO - step: 63210 loss: 16.2152 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.7139 +[titan] 2025-07-10 03:30:09,012 - root - INFO - lr: 1.1201e-04 gnorm: 0.89 [13:55:07< 8:06:04] +[titan] 2025-07-10 03:30:12,968 - root - INFO - step: 63215 loss: 16.8904 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.6047 global_avg_mtp_loss: 14.2857 +[titan] 2025-07-10 03:30:12,968 - root - INFO - lr: 1.1199e-04 gnorm: 0.92 [13:55:11< 8:06:00] +[titan] 2025-07-10 03:30:16,907 - root - INFO - step: 63220 loss: 16.4242 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.9040 +[titan] 2025-07-10 03:30:16,907 - root - INFO - lr: 1.1197e-04 gnorm: 0.88 [13:55:15< 8:05:56] +[titan] 2025-07-10 03:30:20,806 - root - INFO - step: 63225 loss: 16.6584 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5648 global_avg_mtp_loss: 14.0936 +[titan] 2025-07-10 03:30:20,806 - root - INFO - lr: 1.1195e-04 gnorm: 0.89 [13:55:19< 8:05:52] +[titan] 2025-07-10 03:30:24,734 - root - INFO - step: 63230 loss: 16.4332 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9095 +[titan] 2025-07-10 03:30:24,734 - root - INFO - lr: 1.1193e-04 gnorm: 0.84 [13:55:23< 8:05:48] +[titan] 2025-07-10 03:30:28,655 - root - INFO - step: 63235 loss: 16.2069 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7229 +[titan] 2025-07-10 03:30:28,655 - root - INFO - lr: 1.1191e-04 gnorm: 0.90 [13:55:27< 8:05:44] +[titan] 2025-07-10 03:30:32,568 - root - INFO - step: 63240 loss: 16.5501 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 14.0110 +[titan] 2025-07-10 03:30:32,569 - root - INFO - lr: 1.1190e-04 gnorm: 0.90 [13:55:31< 8:05:40] +[titan] 2025-07-10 03:30:36,499 - root - INFO - step: 63245 loss: 16.6361 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5625 global_avg_mtp_loss: 14.0737 +[titan] 2025-07-10 03:30:36,500 - root - INFO - lr: 1.1188e-04 gnorm: 0.90 [13:55:35< 8:05:36] +[titan] 2025-07-10 03:30:39,640 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:30:40,435 - root - INFO - step: 63250 loss: 16.3823 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.8571 +[titan] 2025-07-10 03:30:40,435 - root - INFO - lr: 1.1186e-04 gnorm: 0.86 [13:55:39< 8:05:32] +[titan] 2025-07-10 03:30:44,370 - root - INFO - step: 63255 loss: 16.4065 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8956 +[titan] 2025-07-10 03:30:44,370 - root - INFO - lr: 1.1184e-04 gnorm: 0.86 [13:55:43< 8:05:28] +[titan] 2025-07-10 03:30:48,289 - root - INFO - step: 63260 loss: 16.3049 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.8136 +[titan] 2025-07-10 03:30:48,289 - root - INFO - lr: 1.1182e-04 gnorm: 0.88 [13:55:47< 8:05:24] +[titan] 2025-07-10 03:30:52,199 - root - INFO - step: 63265 loss: 16.3430 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 03:30:52,200 - root - INFO - lr: 1.1180e-04 gnorm: 0.90 [13:55:51< 8:05:20] +[titan] 2025-07-10 03:30:56,113 - root - INFO - step: 63270 loss: 16.2204 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.7324 +[titan] 2025-07-10 03:30:56,114 - root - INFO - lr: 1.1178e-04 gnorm: 0.88 [13:55:54< 8:05:16] +[titan] 2025-07-10 03:31:00,046 - root - INFO - step: 63275 loss: 16.6140 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.5588 global_avg_mtp_loss: 14.0552 +[titan] 2025-07-10 03:31:00,046 - root - INFO - lr: 1.1176e-04 gnorm: 0.86 [13:55:58< 8:05:12] +[titan] 2025-07-10 03:31:03,949 - root - INFO - step: 63280 loss: 16.5546 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 14.0145 +[titan] 2025-07-10 03:31:03,950 - root - INFO - lr: 1.1174e-04 gnorm: 0.90 [13:56:02< 8:05:08] +[titan] 2025-07-10 03:31:07,915 - root - INFO - step: 63285 loss: 16.3286 memory: 44.58GiB(31.99%) tps: 82,632 tflops: 285.18 mfu: 28.83% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 03:31:07,916 - root - INFO - lr: 1.1172e-04 gnorm: 0.87 [13:56:06< 8:05:04] +[titan] 2025-07-10 03:31:11,854 - root - INFO - step: 63290 loss: 16.3198 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5094 global_avg_mtp_loss: 13.8104 +[titan] 2025-07-10 03:31:11,855 - root - INFO - lr: 1.1170e-04 gnorm: 0.88 [13:56:10< 8:05:00] +[titan] 2025-07-10 03:31:15,832 - root - INFO - step: 63295 loss: 16.0858 memory: 44.58GiB(31.99%) tps: 82,389 tflops: 284.34 mfu: 28.75% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6322 +[titan] 2025-07-10 03:31:15,832 - root - INFO - lr: 1.1168e-04 gnorm: 0.85 [13:56:14< 8:04:56] +[titan] 2025-07-10 03:31:18,953 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:31:19,750 - root - INFO - step: 63300 loss: 16.3532 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8461 +[titan] 2025-07-10 03:31:19,750 - root - INFO - lr: 1.1166e-04 gnorm: 0.87 [13:56:18< 8:04:52] +[titan] 2025-07-10 03:31:23,659 - root - INFO - step: 63305 loss: 16.3669 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-10 03:31:23,659 - root - INFO - lr: 1.1164e-04 gnorm: 0.90 [13:56:22< 8:04:48] +[titan] 2025-07-10 03:31:27,590 - root - INFO - step: 63310 loss: 16.4865 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.9640 +[titan] 2025-07-10 03:31:27,590 - root - INFO - lr: 1.1162e-04 gnorm: 0.87 [13:56:26< 8:04:44] +[titan] 2025-07-10 03:31:31,497 - root - INFO - step: 63315 loss: 16.4205 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.9010 +[titan] 2025-07-10 03:31:31,497 - root - INFO - lr: 1.1160e-04 gnorm: 0.85 [13:56:30< 8:04:40] +[titan] 2025-07-10 03:31:35,417 - root - INFO - step: 63320 loss: 16.5710 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5460 global_avg_mtp_loss: 14.0250 +[titan] 2025-07-10 03:31:35,418 - root - INFO - lr: 1.1158e-04 gnorm: 0.86 [13:56:34< 8:04:36] +[titan] 2025-07-10 03:31:39,333 - root - INFO - step: 63325 loss: 16.5646 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 14.0188 +[titan] 2025-07-10 03:31:39,333 - root - INFO - lr: 1.1156e-04 gnorm: 0.91 [13:56:38< 8:04:32] +[titan] 2025-07-10 03:31:43,243 - root - INFO - step: 63330 loss: 16.4982 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9630 +[titan] 2025-07-10 03:31:43,243 - root - INFO - lr: 1.1154e-04 gnorm: 0.96 [13:56:42< 8:04:28] +[titan] 2025-07-10 03:31:47,194 - root - INFO - step: 63335 loss: 16.4475 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9243 +[titan] 2025-07-10 03:31:47,194 - root - INFO - lr: 1.1152e-04 gnorm: 0.83 [13:56:46< 8:04:24] +[titan] 2025-07-10 03:31:51,099 - root - INFO - step: 63340 loss: 15.9998 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.5464 +[titan] 2025-07-10 03:31:51,099 - root - INFO - lr: 1.1150e-04 gnorm: 0.91 [13:56:49< 8:04:20] +[titan] 2025-07-10 03:31:55,003 - root - INFO - step: 63345 loss: 16.5551 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 14.0163 +[titan] 2025-07-10 03:31:55,003 - root - INFO - lr: 1.1148e-04 gnorm: 0.89 [13:56:53< 8:04:16] +[titan] 2025-07-10 03:31:58,137 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:31:58,926 - root - INFO - step: 63350 loss: 16.4806 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.9472 +[titan] 2025-07-10 03:31:58,927 - root - INFO - lr: 1.1146e-04 gnorm: 0.85 [13:56:57< 8:04:12] +[titan] 2025-07-10 03:32:02,851 - root - INFO - step: 63355 loss: 16.2793 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.7835 +[titan] 2025-07-10 03:32:02,851 - root - INFO - lr: 1.1144e-04 gnorm: 0.87 [13:57:01< 8:04:08] +[titan] 2025-07-10 03:32:06,758 - root - INFO - step: 63360 loss: 16.5440 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 14.0038 +[titan] 2025-07-10 03:32:06,759 - root - INFO - lr: 1.1142e-04 gnorm: 0.89 [13:57:05< 8:04:04] +[titan] 2025-07-10 03:32:10,682 - root - INFO - step: 63365 loss: 16.5556 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 14.0104 +[titan] 2025-07-10 03:32:10,683 - root - INFO - lr: 1.1140e-04 gnorm: 0.86 [13:57:09< 8:04:00] +[titan] 2025-07-10 03:32:14,601 - root - INFO - step: 63370 loss: 16.2389 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4766 global_avg_mtp_loss: 13.7623 +[titan] 2025-07-10 03:32:14,601 - root - INFO - lr: 1.1138e-04 gnorm: 0.85 [13:57:13< 8:03:56] +[titan] 2025-07-10 03:32:18,519 - root - INFO - step: 63375 loss: 16.4689 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 13.9341 +[titan] 2025-07-10 03:32:18,519 - root - INFO - lr: 1.1136e-04 gnorm: 0.89 [13:57:17< 8:03:52] +[titan] 2025-07-10 03:32:22,424 - root - INFO - step: 63380 loss: 16.5368 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 13.9952 +[titan] 2025-07-10 03:32:22,425 - root - INFO - lr: 1.1134e-04 gnorm: 0.97 [13:57:21< 8:03:48] +[titan] 2025-07-10 03:32:26,324 - root - INFO - step: 63385 loss: 16.4563 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.9338 +[titan] 2025-07-10 03:32:26,324 - root - INFO - lr: 1.1132e-04 gnorm: 0.92 [13:57:25< 8:03:44] +[titan] 2025-07-10 03:32:30,231 - root - INFO - step: 63390 loss: 16.5594 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5603 global_avg_mtp_loss: 13.9990 +[titan] 2025-07-10 03:32:30,232 - root - INFO - lr: 1.1130e-04 gnorm: 0.92 [13:57:29< 8:03:40] +[titan] 2025-07-10 03:32:34,152 - root - INFO - step: 63395 loss: 16.6329 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5616 global_avg_mtp_loss: 14.0712 +[titan] 2025-07-10 03:32:34,152 - root - INFO - lr: 1.1129e-04 gnorm: 0.89 [13:57:32< 8:03:36] +[titan] 2025-07-10 03:32:37,265 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:32:38,051 - root - INFO - step: 63400 loss: 16.5830 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 14.0428 +[titan] 2025-07-10 03:32:38,052 - root - INFO - lr: 1.1127e-04 gnorm: 0.84 [13:57:36< 8:03:32] +[titan] 2025-07-10 03:32:41,976 - root - INFO - step: 63405 loss: 16.5262 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 13.9916 +[titan] 2025-07-10 03:32:41,977 - root - INFO - lr: 1.1125e-04 gnorm: 0.88 [13:57:40< 8:03:28] +[titan] 2025-07-10 03:32:45,875 - root - INFO - step: 63410 loss: 16.4177 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.9143 +[titan] 2025-07-10 03:32:45,875 - root - INFO - lr: 1.1123e-04 gnorm: 0.92 [13:57:44< 8:03:24] +[titan] 2025-07-10 03:32:49,788 - root - INFO - step: 63415 loss: 16.2158 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7416 +[titan] 2025-07-10 03:32:49,788 - root - INFO - lr: 1.1121e-04 gnorm: 0.90 [13:57:48< 8:03:20] +[titan] 2025-07-10 03:32:53,694 - root - INFO - step: 63420 loss: 16.4570 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5285 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 03:32:53,695 - root - INFO - lr: 1.1119e-04 gnorm: 0.89 [13:57:52< 8:03:16] +[titan] 2025-07-10 03:32:57,607 - root - INFO - step: 63425 loss: 16.2943 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 03:32:57,607 - root - INFO - lr: 1.1117e-04 gnorm: 0.90 [13:57:56< 8:03:12] +[titan] 2025-07-10 03:33:01,531 - root - INFO - step: 63430 loss: 16.5034 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9833 +[titan] 2025-07-10 03:33:01,532 - root - INFO - lr: 1.1115e-04 gnorm: 0.89 [13:58:00< 8:03:08] +[titan] 2025-07-10 03:33:05,457 - root - INFO - step: 63435 loss: 16.2702 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.7791 +[titan] 2025-07-10 03:33:05,457 - root - INFO - lr: 1.1113e-04 gnorm: 0.86 [13:58:04< 8:03:04] +[titan] 2025-07-10 03:33:09,393 - root - INFO - step: 63440 loss: 16.2125 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7266 +[titan] 2025-07-10 03:33:09,393 - root - INFO - lr: 1.1111e-04 gnorm: 0.90 [13:58:08< 8:03:00] +[titan] 2025-07-10 03:33:13,311 - root - INFO - step: 63445 loss: 16.4858 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9579 +[titan] 2025-07-10 03:33:13,312 - root - INFO - lr: 1.1109e-04 gnorm: 0.89 [13:58:12< 8:02:56] +[titan] 2025-07-10 03:33:16,457 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:33:17,266 - root - INFO - step: 63450 loss: 16.3234 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8166 +[titan] 2025-07-10 03:33:17,267 - root - INFO - lr: 1.1107e-04 gnorm: 0.87 [13:58:16< 8:02:52] +[titan] 2025-07-10 03:33:21,192 - root - INFO - step: 63455 loss: 16.5363 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 14.0064 +[titan] 2025-07-10 03:33:21,192 - root - INFO - lr: 1.1105e-04 gnorm: 0.83 [13:58:19< 8:02:48] +[titan] 2025-07-10 03:33:25,116 - root - INFO - step: 63460 loss: 16.3236 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8273 +[titan] 2025-07-10 03:33:25,116 - root - INFO - lr: 1.1103e-04 gnorm: 0.87 [13:58:23< 8:02:44] +[titan] 2025-07-10 03:33:29,056 - root - INFO - step: 63465 loss: 16.4346 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.9180 +[titan] 2025-07-10 03:33:29,057 - root - INFO - lr: 1.1101e-04 gnorm: 0.88 [13:58:27< 8:02:40] +[titan] 2025-07-10 03:33:32,970 - root - INFO - step: 63470 loss: 16.4716 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.9360 +[titan] 2025-07-10 03:33:32,971 - root - INFO - lr: 1.1099e-04 gnorm: 0.84 [13:58:31< 8:02:36] +[titan] 2025-07-10 03:33:36,883 - root - INFO - step: 63475 loss: 16.4084 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.8886 +[titan] 2025-07-10 03:33:36,884 - root - INFO - lr: 1.1097e-04 gnorm: 0.85 [13:58:35< 8:02:32] +[titan] 2025-07-10 03:33:40,799 - root - INFO - step: 63480 loss: 16.3050 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8061 +[titan] 2025-07-10 03:33:40,799 - root - INFO - lr: 1.1095e-04 gnorm: 0.92 [13:58:39< 8:02:28] +[titan] 2025-07-10 03:33:44,791 - root - INFO - step: 63485 loss: 16.2783 memory: 44.58GiB(31.99%) tps: 82,081 tflops: 283.28 mfu: 28.64% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7821 +[titan] 2025-07-10 03:33:44,791 - root - INFO - lr: 1.1093e-04 gnorm: 0.83 [13:58:43< 8:02:24] +[titan] 2025-07-10 03:33:47,294 - root - INFO - Dumping profiler traces at step 63488 +[titan] 2025-07-10 03:33:47,326 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:33:48,895 - root - INFO - step: 63490 loss: 16.4462 memory: 44.58GiB(31.99%) tps: 79,846 tflops: 275.56 mfu: 27.86% global_avg_ntp_loss: 2.5253 global_avg_mtp_loss: 13.9209 +[titan] 2025-07-10 03:33:48,896 - root - INFO - lr: 1.1091e-04 gnorm: 0.88 [13:58:47< 8:02:21] +[titan] 2025-07-10 03:33:52,810 - root - INFO - step: 63495 loss: 16.4907 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9492 +[titan] 2025-07-10 03:33:52,811 - root - INFO - lr: 1.1089e-04 gnorm: 0.84 [13:58:51< 8:02:17] +[titan] 2025-07-10 03:33:55,976 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:33:56,778 - root - INFO - step: 63500 loss: 16.3737 memory: 44.58GiB(31.99%) tps: 82,599 tflops: 285.06 mfu: 28.82% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.8718 +[titan] 2025-07-10 03:33:56,778 - root - INFO - lr: 1.1087e-04 gnorm: 0.88 [13:58:55< 8:02:13] +[titan] 2025-07-10 03:34:00,693 - root - INFO - step: 63505 loss: 16.5866 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 14.0339 +[titan] 2025-07-10 03:34:00,694 - root - INFO - lr: 1.1085e-04 gnorm: 0.85 [13:58:59< 8:02:09] +[titan] 2025-07-10 03:34:04,630 - root - INFO - step: 63510 loss: 16.2069 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7219 +[titan] 2025-07-10 03:34:04,630 - root - INFO - lr: 1.1083e-04 gnorm: 0.83 [13:59:03< 8:02:05] +[titan] 2025-07-10 03:34:08,593 - root - INFO - step: 63515 loss: 16.7783 memory: 44.58GiB(31.99%) tps: 82,683 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.5844 global_avg_mtp_loss: 14.1939 +[titan] 2025-07-10 03:34:08,594 - root - INFO - lr: 1.1081e-04 gnorm: 0.92 [13:59:07< 8:02:01] +[titan] 2025-07-10 03:34:12,517 - root - INFO - step: 63520 loss: 16.4726 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9466 +[titan] 2025-07-10 03:34:12,518 - root - INFO - lr: 1.1079e-04 gnorm: 0.85 [13:59:11< 8:01:57] +[titan] 2025-07-10 03:34:16,452 - root - INFO - step: 63525 loss: 16.4200 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.9105 +[titan] 2025-07-10 03:34:16,452 - root - INFO - lr: 1.1077e-04 gnorm: 0.84 [13:59:15< 8:01:53] +[titan] 2025-07-10 03:34:20,382 - root - INFO - step: 63530 loss: 16.2767 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.7835 +[titan] 2025-07-10 03:34:20,382 - root - INFO - lr: 1.1076e-04 gnorm: 0.91 [13:59:19< 8:01:49] +[titan] 2025-07-10 03:34:24,282 - root - INFO - step: 63535 loss: 16.4503 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.9328 +[titan] 2025-07-10 03:34:24,282 - root - INFO - lr: 1.1074e-04 gnorm: 0.86 [13:59:23< 8:01:45] +[titan] 2025-07-10 03:34:28,254 - root - INFO - step: 63540 loss: 16.2124 memory: 44.58GiB(31.99%) tps: 82,499 tflops: 284.72 mfu: 28.79% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.7291 +[titan] 2025-07-10 03:34:28,254 - root - INFO - lr: 1.1072e-04 gnorm: 0.91 [13:59:27< 8:01:41] +[titan] 2025-07-10 03:34:32,182 - root - INFO - step: 63545 loss: 16.2952 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.7981 +[titan] 2025-07-10 03:34:32,183 - root - INFO - lr: 1.1070e-04 gnorm: 0.94 [13:59:30< 8:01:37] +[titan] 2025-07-10 03:34:35,336 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:34:36,142 - root - INFO - step: 63550 loss: 16.4523 memory: 44.58GiB(31.99%) tps: 82,765 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 03:34:36,142 - root - INFO - lr: 1.1068e-04 gnorm: 0.86 [13:59:34< 8:01:33] +[titan] 2025-07-10 03:34:40,089 - root - INFO - step: 63555 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7450 +[titan] 2025-07-10 03:34:40,089 - root - INFO - lr: 1.1066e-04 gnorm: 0.92 [13:59:38< 8:01:29] +[titan] 2025-07-10 03:34:44,025 - root - INFO - step: 63560 loss: 16.5039 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9775 +[titan] 2025-07-10 03:34:44,025 - root - INFO - lr: 1.1064e-04 gnorm: 0.97 [13:59:42< 8:01:25] +[titan] 2025-07-10 03:34:47,942 - root - INFO - step: 63565 loss: 16.5373 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9934 +[titan] 2025-07-10 03:34:47,942 - root - INFO - lr: 1.1062e-04 gnorm: 0.92 [13:59:46< 8:01:21] +[titan] 2025-07-10 03:34:51,845 - root - INFO - step: 63570 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4920 global_avg_mtp_loss: 13.6925 +[titan] 2025-07-10 03:34:51,846 - root - INFO - lr: 1.1060e-04 gnorm: 0.93 [13:59:50< 8:01:17] +[titan] 2025-07-10 03:34:55,751 - root - INFO - step: 63575 loss: 16.5773 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0266 +[titan] 2025-07-10 03:34:55,751 - root - INFO - lr: 1.1058e-04 gnorm: 0.86 [13:59:54< 8:01:13] +[titan] 2025-07-10 03:34:59,698 - root - INFO - step: 63580 loss: 16.3155 memory: 44.58GiB(31.99%) tps: 83,030 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8065 +[titan] 2025-07-10 03:34:59,698 - root - INFO - lr: 1.1056e-04 gnorm: 0.91 [13:59:58< 8:01:09] +[titan] 2025-07-10 03:35:03,607 - root - INFO - step: 63585 loss: 16.2262 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7474 +[titan] 2025-07-10 03:35:03,607 - root - INFO - lr: 1.1054e-04 gnorm: 0.92 [14:00:02< 8:01:05] +[titan] 2025-07-10 03:35:07,560 - root - INFO - step: 63590 loss: 16.3953 memory: 44.58GiB(31.99%) tps: 82,908 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8715 +[titan] 2025-07-10 03:35:07,560 - root - INFO - lr: 1.1052e-04 gnorm: 1.00 [14:00:06< 8:01:01] +[titan] 2025-07-10 03:35:11,453 - root - INFO - step: 63595 loss: 16.6445 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.5630 global_avg_mtp_loss: 14.0815 +[titan] 2025-07-10 03:35:11,453 - root - INFO - lr: 1.1050e-04 gnorm: 0.88 [14:00:10< 8:00:57] +[titan] 2025-07-10 03:35:14,590 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:35:15,386 - root - INFO - step: 63600 loss: 16.2270 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.7262 +[titan] 2025-07-10 03:35:15,387 - root - INFO - lr: 1.1048e-04 gnorm: 0.92 [14:00:14< 8:00:53] +[titan] 2025-07-10 03:35:19,325 - root - INFO - step: 63605 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8227 +[titan] 2025-07-10 03:35:19,325 - root - INFO - lr: 1.1046e-04 gnorm: 0.85 [14:00:18< 8:00:49] +[titan] 2025-07-10 03:35:23,256 - root - INFO - step: 63610 loss: 16.0642 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.5966 +[titan] 2025-07-10 03:35:23,256 - root - INFO - lr: 1.1044e-04 gnorm: 0.87 [14:00:22< 8:00:45] +[titan] 2025-07-10 03:35:27,176 - root - INFO - step: 63615 loss: 16.3429 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8363 +[titan] 2025-07-10 03:35:27,177 - root - INFO - lr: 1.1042e-04 gnorm: 0.89 [14:00:25< 8:00:41] +[titan] 2025-07-10 03:35:31,102 - root - INFO - step: 63620 loss: 16.3896 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8675 +[titan] 2025-07-10 03:35:31,102 - root - INFO - lr: 1.1040e-04 gnorm: 0.89 [14:00:29< 8:00:37] +[titan] 2025-07-10 03:35:35,012 - root - INFO - step: 63625 loss: 16.3574 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8505 +[titan] 2025-07-10 03:35:35,012 - root - INFO - lr: 1.1038e-04 gnorm: 0.87 [14:00:33< 8:00:33] +[titan] 2025-07-10 03:35:38,940 - root - INFO - step: 63630 loss: 16.0948 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6306 +[titan] 2025-07-10 03:35:38,941 - root - INFO - lr: 1.1036e-04 gnorm: 0.88 [14:00:37< 8:00:29] +[titan] 2025-07-10 03:35:42,847 - root - INFO - step: 63635 loss: 16.3767 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8755 +[titan] 2025-07-10 03:35:42,847 - root - INFO - lr: 1.1034e-04 gnorm: 0.92 [14:00:41< 8:00:25] +[titan] 2025-07-10 03:35:46,789 - root - INFO - step: 63640 loss: 16.3120 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.8161 +[titan] 2025-07-10 03:35:46,790 - root - INFO - lr: 1.1032e-04 gnorm: 0.89 [14:00:45< 8:00:21] +[titan] 2025-07-10 03:35:50,686 - root - INFO - step: 63645 loss: 16.3746 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8734 +[titan] 2025-07-10 03:35:50,686 - root - INFO - lr: 1.1030e-04 gnorm: 0.85 [14:00:49< 8:00:17] +[titan] 2025-07-10 03:35:53,800 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:35:54,588 - root - INFO - step: 63650 loss: 16.6256 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 14.0861 +[titan] 2025-07-10 03:35:54,588 - root - INFO - lr: 1.1028e-04 gnorm: 0.85 [14:00:53< 8:00:13] +[titan] 2025-07-10 03:35:58,521 - root - INFO - step: 63655 loss: 16.4406 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9147 +[titan] 2025-07-10 03:35:58,521 - root - INFO - lr: 1.1027e-04 gnorm: 0.90 [14:00:57< 8:00:09] +[titan] 2025-07-10 03:36:02,442 - root - INFO - step: 63660 loss: 16.0974 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6343 +[titan] 2025-07-10 03:36:02,442 - root - INFO - lr: 1.1025e-04 gnorm: 0.86 [14:01:01< 8:00:05] +[titan] 2025-07-10 03:36:06,390 - root - INFO - step: 63665 loss: 16.5335 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9927 +[titan] 2025-07-10 03:36:06,390 - root - INFO - lr: 1.1023e-04 gnorm: 0.89 [14:01:05< 8:00:01] +[titan] 2025-07-10 03:36:10,289 - root - INFO - step: 63670 loss: 16.5873 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 14.0480 +[titan] 2025-07-10 03:36:10,289 - root - INFO - lr: 1.1021e-04 gnorm: 0.85 [14:01:09< 7:59:57] +[titan] 2025-07-10 03:36:14,203 - root - INFO - step: 63675 loss: 16.3325 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 03:36:14,204 - root - INFO - lr: 1.1019e-04 gnorm: 0.85 [14:01:12< 7:59:53] +[titan] 2025-07-10 03:36:18,112 - root - INFO - step: 63680 loss: 16.1913 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7188 +[titan] 2025-07-10 03:36:18,113 - root - INFO - lr: 1.1017e-04 gnorm: 0.85 [14:01:16< 7:59:49] +[titan] 2025-07-10 03:36:22,037 - root - INFO - step: 63685 loss: 16.7368 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5606 global_avg_mtp_loss: 14.1762 +[titan] 2025-07-10 03:36:22,037 - root - INFO - lr: 1.1015e-04 gnorm: 0.86 [14:01:20< 7:59:45] +[titan] 2025-07-10 03:36:25,972 - root - INFO - step: 63690 loss: 16.3089 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8100 +[titan] 2025-07-10 03:36:25,972 - root - INFO - lr: 1.1013e-04 gnorm: 0.91 [14:01:24< 7:59:41] +[titan] 2025-07-10 03:36:29,895 - root - INFO - step: 63695 loss: 16.7637 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5843 global_avg_mtp_loss: 14.1794 +[titan] 2025-07-10 03:36:29,895 - root - INFO - lr: 1.1011e-04 gnorm: 0.85 [14:01:28< 7:59:37] +[titan] 2025-07-10 03:36:33,009 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:36:33,813 - root - INFO - step: 63700 loss: 16.2503 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7597 +[titan] 2025-07-10 03:36:33,814 - root - INFO - lr: 1.1009e-04 gnorm: 0.86 [14:01:32< 7:59:33] +[titan] 2025-07-10 03:36:37,732 - root - INFO - step: 63705 loss: 16.4411 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9110 +[titan] 2025-07-10 03:36:37,732 - root - INFO - lr: 1.1007e-04 gnorm: 0.83 [14:01:36< 7:59:29] +[titan] 2025-07-10 03:36:41,641 - root - INFO - step: 63710 loss: 16.2437 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.7693 +[titan] 2025-07-10 03:36:41,641 - root - INFO - lr: 1.1005e-04 gnorm: 0.92 [14:01:40< 7:59:25] +[titan] 2025-07-10 03:36:45,565 - root - INFO - step: 63715 loss: 16.4541 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5491 global_avg_mtp_loss: 13.9051 +[titan] 2025-07-10 03:36:45,565 - root - INFO - lr: 1.1003e-04 gnorm: 0.83 [14:01:44< 7:59:21] +[titan] 2025-07-10 03:36:49,527 - root - INFO - step: 63720 loss: 16.1796 memory: 44.58GiB(31.99%) tps: 82,712 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7044 +[titan] 2025-07-10 03:36:49,527 - root - INFO - lr: 1.1001e-04 gnorm: 0.90 [14:01:48< 7:59:17] +[titan] 2025-07-10 03:36:53,455 - root - INFO - step: 63725 loss: 16.2651 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.7746 +[titan] 2025-07-10 03:36:53,456 - root - INFO - lr: 1.0999e-04 gnorm: 0.81 [14:01:52< 7:59:13] +[titan] 2025-07-10 03:36:57,394 - root - INFO - step: 63730 loss: 16.3389 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8337 +[titan] 2025-07-10 03:36:57,394 - root - INFO - lr: 1.0997e-04 gnorm: 0.88 [14:01:56< 7:59:09] +[titan] 2025-07-10 03:37:01,331 - root - INFO - step: 63735 loss: 16.7350 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5804 global_avg_mtp_loss: 14.1546 +[titan] 2025-07-10 03:37:01,331 - root - INFO - lr: 1.0995e-04 gnorm: 1.02 [14:02:00< 7:59:05] +[titan] 2025-07-10 03:37:05,273 - root - INFO - step: 63740 loss: 16.2781 memory: 44.58GiB(31.99%) tps: 83,129 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 03:37:05,273 - root - INFO - lr: 1.0993e-04 gnorm: 0.85 [14:02:04< 7:59:01] +[titan] 2025-07-10 03:37:09,180 - root - INFO - step: 63745 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8181 +[titan] 2025-07-10 03:37:09,180 - root - INFO - lr: 1.0991e-04 gnorm: 0.90 [14:02:07< 7:58:57] +[titan] 2025-07-10 03:37:12,355 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:37:13,146 - root - INFO - step: 63750 loss: 16.5859 memory: 44.58GiB(31.99%) tps: 82,632 tflops: 285.18 mfu: 28.84% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 14.0404 +[titan] 2025-07-10 03:37:13,146 - root - INFO - lr: 1.0989e-04 gnorm: 0.85 [14:02:11< 7:58:53] +[titan] 2025-07-10 03:37:17,123 - root - INFO - step: 63755 loss: 16.4611 memory: 44.58GiB(31.99%) tps: 82,401 tflops: 284.38 mfu: 28.75% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9311 +[titan] 2025-07-10 03:37:17,123 - root - INFO - lr: 1.0987e-04 gnorm: 0.90 [14:02:15< 7:58:49] +[titan] 2025-07-10 03:37:21,070 - root - INFO - step: 63760 loss: 16.4124 memory: 44.58GiB(31.99%) tps: 83,039 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.8937 +[titan] 2025-07-10 03:37:21,070 - root - INFO - lr: 1.0985e-04 gnorm: 0.87 [14:02:19< 7:58:45] +[titan] 2025-07-10 03:37:24,987 - root - INFO - step: 63765 loss: 16.3440 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 03:37:24,987 - root - INFO - lr: 1.0983e-04 gnorm: 0.92 [14:02:23< 7:58:41] +[titan] 2025-07-10 03:37:28,881 - root - INFO - step: 63770 loss: 16.2196 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7409 +[titan] 2025-07-10 03:37:28,881 - root - INFO - lr: 1.0982e-04 gnorm: 0.89 [14:02:27< 7:58:37] +[titan] 2025-07-10 03:37:32,775 - root - INFO - step: 63775 loss: 16.4111 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.8766 +[titan] 2025-07-10 03:37:32,776 - root - INFO - lr: 1.0980e-04 gnorm: 0.87 [14:02:31< 7:58:33] +[titan] 2025-07-10 03:37:36,694 - root - INFO - step: 63780 loss: 16.3419 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8422 +[titan] 2025-07-10 03:37:36,694 - root - INFO - lr: 1.0978e-04 gnorm: 0.92 [14:02:35< 7:58:29] +[titan] 2025-07-10 03:37:40,615 - root - INFO - step: 63785 loss: 16.5064 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9654 +[titan] 2025-07-10 03:37:40,615 - root - INFO - lr: 1.0976e-04 gnorm: 0.85 [14:02:39< 7:58:25] +[titan] 2025-07-10 03:37:44,537 - root - INFO - step: 63790 loss: 16.0366 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.5817 +[titan] 2025-07-10 03:37:44,537 - root - INFO - lr: 1.0974e-04 gnorm: 0.88 [14:02:43< 7:58:21] +[titan] 2025-07-10 03:37:48,455 - root - INFO - step: 63795 loss: 16.4479 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.9149 +[titan] 2025-07-10 03:37:48,455 - root - INFO - lr: 1.0972e-04 gnorm: 0.88 [14:02:47< 7:58:17] +[titan] 2025-07-10 03:37:51,573 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:37:52,360 - root - INFO - step: 63800 loss: 16.3352 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8360 +[titan] 2025-07-10 03:37:52,360 - root - INFO - lr: 1.0970e-04 gnorm: 0.87 [14:02:51< 7:58:13] +[titan] 2025-07-10 03:37:56,260 - root - INFO - step: 63805 loss: 16.5552 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 14.0155 +[titan] 2025-07-10 03:37:56,261 - root - INFO - lr: 1.0968e-04 gnorm: 0.87 [14:02:55< 7:58:09] +[titan] 2025-07-10 03:38:00,195 - root - INFO - step: 63810 loss: 16.2606 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.7647 +[titan] 2025-07-10 03:38:00,195 - root - INFO - lr: 1.0966e-04 gnorm: 0.84 [14:02:58< 7:58:05] +[titan] 2025-07-10 03:38:04,144 - root - INFO - step: 63815 loss: 16.1980 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7177 +[titan] 2025-07-10 03:38:04,144 - root - INFO - lr: 1.0964e-04 gnorm: 0.85 [14:03:02< 7:58:02] +[titan] 2025-07-10 03:38:08,059 - root - INFO - step: 63820 loss: 16.5251 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9859 +[titan] 2025-07-10 03:38:08,059 - root - INFO - lr: 1.0962e-04 gnorm: 0.85 [14:03:06< 7:57:58] +[titan] 2025-07-10 03:38:12,012 - root - INFO - step: 63825 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.8155 +[titan] 2025-07-10 03:38:12,012 - root - INFO - lr: 1.0960e-04 gnorm: 0.82 [14:03:10< 7:57:54] +[titan] 2025-07-10 03:38:15,975 - root - INFO - step: 63830 loss: 16.4548 memory: 44.58GiB(31.99%) tps: 82,688 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.9358 +[titan] 2025-07-10 03:38:15,975 - root - INFO - lr: 1.0958e-04 gnorm: 0.86 [14:03:14< 7:57:50] +[titan] 2025-07-10 03:38:19,899 - root - INFO - step: 63835 loss: 16.5822 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5434 global_avg_mtp_loss: 14.0388 +[titan] 2025-07-10 03:38:19,899 - root - INFO - lr: 1.0956e-04 gnorm: 0.85 [14:03:18< 7:57:46] +[titan] 2025-07-10 03:38:23,819 - root - INFO - step: 63840 loss: 16.8604 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.6186 global_avg_mtp_loss: 14.2418 +[titan] 2025-07-10 03:38:23,820 - root - INFO - lr: 1.0954e-04 gnorm: 0.88 [14:03:22< 7:57:42] +[titan] 2025-07-10 03:38:27,720 - root - INFO - step: 63845 loss: 16.3295 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8303 +[titan] 2025-07-10 03:38:27,720 - root - INFO - lr: 1.0952e-04 gnorm: 0.89 [14:03:26< 7:57:38] +[titan] 2025-07-10 03:38:30,832 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:38:31,619 - root - INFO - step: 63850 loss: 16.4046 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.33% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8932 +[titan] 2025-07-10 03:38:31,620 - root - INFO - lr: 1.0950e-04 gnorm: 0.90 [14:03:30< 7:57:34] +[titan] 2025-07-10 03:38:35,706 - root - INFO - step: 63855 loss: 16.3342 memory: 44.58GiB(31.99%) tps: 80,183 tflops: 276.72 mfu: 27.98% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 03:38:35,707 - root - INFO - lr: 1.0948e-04 gnorm: 0.87 [14:03:34< 7:57:30] +[titan] 2025-07-10 03:38:39,629 - root - INFO - step: 63860 loss: 16.4296 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9012 +[titan] 2025-07-10 03:38:39,629 - root - INFO - lr: 1.0946e-04 gnorm: 0.88 [14:03:38< 7:57:26] +[titan] 2025-07-10 03:38:43,544 - root - INFO - step: 63865 loss: 16.1666 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.6878 +[titan] 2025-07-10 03:38:43,544 - root - INFO - lr: 1.0944e-04 gnorm: 0.88 [14:03:42< 7:57:22] +[titan] 2025-07-10 03:38:47,483 - root - INFO - step: 63870 loss: 16.3813 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8693 +[titan] 2025-07-10 03:38:47,483 - root - INFO - lr: 1.0942e-04 gnorm: 0.84 [14:03:46< 7:57:18] +[titan] 2025-07-10 03:38:51,433 - root - INFO - step: 63875 loss: 16.4964 memory: 44.58GiB(31.99%) tps: 82,963 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9667 +[titan] 2025-07-10 03:38:51,433 - root - INFO - lr: 1.0941e-04 gnorm: 0.84 [14:03:50< 7:57:14] +[titan] 2025-07-10 03:38:55,330 - root - INFO - step: 63880 loss: 16.2123 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7195 +[titan] 2025-07-10 03:38:55,331 - root - INFO - lr: 1.0939e-04 gnorm: 0.87 [14:03:54< 7:57:10] +[titan] 2025-07-10 03:38:59,255 - root - INFO - step: 63885 loss: 16.2776 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.7656 +[titan] 2025-07-10 03:38:59,255 - root - INFO - lr: 1.0937e-04 gnorm: 0.84 [14:03:58< 7:57:06] +[titan] 2025-07-10 03:39:03,197 - root - INFO - step: 63890 loss: 16.4142 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 03:39:03,197 - root - INFO - lr: 1.0935e-04 gnorm: 0.96 [14:04:01< 7:57:02] +[titan] 2025-07-10 03:39:07,103 - root - INFO - step: 63895 loss: 16.4682 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9372 +[titan] 2025-07-10 03:39:07,103 - root - INFO - lr: 1.0933e-04 gnorm: 0.88 [14:04:05< 7:56:58] +[titan] 2025-07-10 03:39:10,243 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:39:11,046 - root - INFO - step: 63900 loss: 16.1312 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.6484 +[titan] 2025-07-10 03:39:11,047 - root - INFO - lr: 1.0931e-04 gnorm: 0.91 [14:04:09< 7:56:54] +[titan] 2025-07-10 03:39:14,990 - root - INFO - step: 63905 loss: 16.4367 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.9086 +[titan] 2025-07-10 03:39:14,991 - root - INFO - lr: 1.0929e-04 gnorm: 0.88 [14:04:13< 7:56:50] +[titan] 2025-07-10 03:39:18,943 - root - INFO - step: 63910 loss: 16.2638 memory: 44.58GiB(31.99%) tps: 82,918 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7851 +[titan] 2025-07-10 03:39:18,943 - root - INFO - lr: 1.0927e-04 gnorm: 0.94 [14:04:17< 7:56:46] +[titan] 2025-07-10 03:39:22,847 - root - INFO - step: 63915 loss: 16.5111 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5585 global_avg_mtp_loss: 13.9525 +[titan] 2025-07-10 03:39:22,848 - root - INFO - lr: 1.0925e-04 gnorm: 0.91 [14:04:21< 7:56:42] +[titan] 2025-07-10 03:39:26,762 - root - INFO - step: 63920 loss: 16.6286 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5583 global_avg_mtp_loss: 14.0702 +[titan] 2025-07-10 03:39:26,762 - root - INFO - lr: 1.0923e-04 gnorm: 0.87 [14:04:25< 7:56:38] +[titan] 2025-07-10 03:39:30,679 - root - INFO - step: 63925 loss: 16.5461 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 14.0111 +[titan] 2025-07-10 03:39:30,679 - root - INFO - lr: 1.0921e-04 gnorm: 0.86 [14:04:29< 7:56:34] +[titan] 2025-07-10 03:39:34,598 - root - INFO - step: 63930 loss: 16.6208 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5708 global_avg_mtp_loss: 14.0500 +[titan] 2025-07-10 03:39:34,598 - root - INFO - lr: 1.0919e-04 gnorm: 0.88 [14:04:33< 7:56:30] +[titan] 2025-07-10 03:39:38,523 - root - INFO - step: 63935 loss: 16.4414 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5206 global_avg_mtp_loss: 13.9208 +[titan] 2025-07-10 03:39:38,523 - root - INFO - lr: 1.0917e-04 gnorm: 0.90 [14:04:37< 7:56:26] +[titan] 2025-07-10 03:39:42,519 - root - INFO - step: 63940 loss: 16.0948 memory: 44.58GiB(31.99%) tps: 82,000 tflops: 283.00 mfu: 28.61% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.6342 +[titan] 2025-07-10 03:39:42,520 - root - INFO - lr: 1.0915e-04 gnorm: 0.99 [14:04:41< 7:56:22] +[titan] 2025-07-10 03:39:46,452 - root - INFO - step: 63945 loss: 16.5440 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 14.0127 +[titan] 2025-07-10 03:39:46,453 - root - INFO - lr: 1.0913e-04 gnorm: 0.91 [14:04:45< 7:56:18] +[titan] 2025-07-10 03:39:49,596 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:39:50,383 - root - INFO - step: 63950 loss: 16.6272 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0649 +[titan] 2025-07-10 03:39:50,384 - root - INFO - lr: 1.0911e-04 gnorm: 0.87 [14:04:49< 7:56:14] +[titan] 2025-07-10 03:39:54,286 - root - INFO - step: 63955 loss: 16.4559 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.9372 +[titan] 2025-07-10 03:39:54,286 - root - INFO - lr: 1.0909e-04 gnorm: 0.87 [14:04:53< 7:56:10] +[titan] 2025-07-10 03:39:58,260 - root - INFO - step: 63960 loss: 16.5867 memory: 44.58GiB(31.99%) tps: 82,451 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.5609 global_avg_mtp_loss: 14.0258 +[titan] 2025-07-10 03:39:58,260 - root - INFO - lr: 1.0907e-04 gnorm: 0.95 [14:04:57< 7:56:06] +[titan] 2025-07-10 03:40:02,182 - root - INFO - step: 63965 loss: 16.3495 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8450 +[titan] 2025-07-10 03:40:02,182 - root - INFO - lr: 1.0905e-04 gnorm: 0.87 [14:05:00< 7:56:02] +[titan] 2025-07-10 03:40:06,081 - root - INFO - step: 63970 loss: 16.5304 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-10 03:40:06,081 - root - INFO - lr: 1.0903e-04 gnorm: 0.90 [14:05:04< 7:55:58] +[titan] 2025-07-10 03:40:10,029 - root - INFO - step: 63975 loss: 16.5538 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 14.0114 +[titan] 2025-07-10 03:40:10,029 - root - INFO - lr: 1.0901e-04 gnorm: 0.87 [14:05:08< 7:55:54] +[titan] 2025-07-10 03:40:13,951 - root - INFO - step: 63980 loss: 16.2614 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.7722 +[titan] 2025-07-10 03:40:13,951 - root - INFO - lr: 1.0900e-04 gnorm: 0.89 [14:05:12< 7:55:50] +[titan] 2025-07-10 03:40:17,863 - root - INFO - step: 63985 loss: 16.1352 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.6570 +[titan] 2025-07-10 03:40:17,864 - root - INFO - lr: 1.0898e-04 gnorm: 0.91 [14:05:16< 7:55:46] +[titan] 2025-07-10 03:40:21,775 - root - INFO - step: 63990 loss: 16.4490 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9303 +[titan] 2025-07-10 03:40:21,775 - root - INFO - lr: 1.0896e-04 gnorm: 0.90 [14:05:20< 7:55:42] +[titan] 2025-07-10 03:40:25,709 - root - INFO - step: 63995 loss: 16.4480 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-10 03:40:25,709 - root - INFO - lr: 1.0894e-04 gnorm: 0.96 [14:05:24< 7:55:38] +[titan] 2025-07-10 03:40:28,925 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:40:29,730 - root - INFO - step: 64000 loss: 16.3459 memory: 44.58GiB(31.99%) tps: 81,509 tflops: 281.30 mfu: 28.44% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8484 +[titan] 2025-07-10 03:40:29,730 - root - INFO - lr: 1.0892e-04 gnorm: 0.89 [14:05:28< 7:55:34] +[titan] 2025-07-10 03:40:29,887 - root - INFO - Dumping profiler traces at step 64000 +[titan] 2025-07-10 03:40:29,922 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 03:40:33,825 - root - INFO - step: 64005 loss: 16.4245 memory: 44.58GiB(31.99%) tps: 80,010 tflops: 276.13 mfu: 27.92% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9077 +[titan] 2025-07-10 03:40:33,826 - root - INFO - lr: 1.0890e-04 gnorm: 0.86 [14:05:32< 7:55:30] +[titan] 2025-07-10 03:40:37,741 - root - INFO - step: 64010 loss: 16.4955 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9778 +[titan] 2025-07-10 03:40:37,741 - root - INFO - lr: 1.0888e-04 gnorm: 0.93 [14:05:36< 7:55:26] +[titan] 2025-07-10 03:40:41,733 - root - INFO - step: 64015 loss: 16.7318 memory: 44.58GiB(31.99%) tps: 82,088 tflops: 283.30 mfu: 28.65% global_avg_ntp_loss: 2.5810 global_avg_mtp_loss: 14.1508 +[titan] 2025-07-10 03:40:41,733 - root - INFO - lr: 1.0886e-04 gnorm: 0.92 [14:05:40< 7:55:22] +[titan] 2025-07-10 03:40:45,687 - root - INFO - step: 64020 loss: 16.3295 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8305 +[titan] 2025-07-10 03:40:45,687 - root - INFO - lr: 1.0884e-04 gnorm: 0.89 [14:05:44< 7:55:18] +[titan] 2025-07-10 03:40:49,652 - root - INFO - step: 64025 loss: 16.2705 memory: 44.58GiB(31.99%) tps: 82,650 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.7659 +[titan] 2025-07-10 03:40:49,652 - root - INFO - lr: 1.0882e-04 gnorm: 0.84 [14:05:48< 7:55:15] +[titan] 2025-07-10 03:40:53,606 - root - INFO - step: 64030 loss: 16.4990 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.9608 +[titan] 2025-07-10 03:40:53,607 - root - INFO - lr: 1.0880e-04 gnorm: 0.87 [14:05:52< 7:55:11] +[titan] 2025-07-10 03:40:57,549 - root - INFO - step: 64035 loss: 16.4801 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9457 +[titan] 2025-07-10 03:40:57,549 - root - INFO - lr: 1.0878e-04 gnorm: 0.86 [14:05:56< 7:55:07] +[titan] 2025-07-10 03:41:01,496 - root - INFO - step: 64040 loss: 16.3137 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8183 +[titan] 2025-07-10 03:41:01,496 - root - INFO - lr: 1.0876e-04 gnorm: 0.84 [14:06:00< 7:55:03] +[titan] 2025-07-10 03:41:05,393 - root - INFO - step: 64045 loss: 16.2710 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7809 +[titan] 2025-07-10 03:41:05,393 - root - INFO - lr: 1.0874e-04 gnorm: 0.89 [14:06:04< 7:54:59] +[titan] 2025-07-10 03:41:08,539 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:41:09,329 - root - INFO - step: 64050 loss: 16.5015 memory: 44.58GiB(31.99%) tps: 83,244 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 13.9552 +[titan] 2025-07-10 03:41:09,330 - root - INFO - lr: 1.0872e-04 gnorm: 0.89 [14:06:08< 7:54:55] +[titan] 2025-07-10 03:41:13,261 - root - INFO - step: 64055 loss: 16.1049 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.6281 +[titan] 2025-07-10 03:41:13,261 - root - INFO - lr: 1.0870e-04 gnorm: 0.90 [14:06:12< 7:54:51] +[titan] 2025-07-10 03:41:17,171 - root - INFO - step: 64060 loss: 16.5988 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 14.0432 +[titan] 2025-07-10 03:41:17,171 - root - INFO - lr: 1.0868e-04 gnorm: 0.88 [14:06:15< 7:54:47] +[titan] 2025-07-10 03:41:21,088 - root - INFO - step: 64065 loss: 16.2850 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7896 +[titan] 2025-07-10 03:41:21,088 - root - INFO - lr: 1.0866e-04 gnorm: 0.86 [14:06:19< 7:54:43] +[titan] 2025-07-10 03:41:24,992 - root - INFO - step: 64070 loss: 16.3927 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8801 +[titan] 2025-07-10 03:41:24,992 - root - INFO - lr: 1.0864e-04 gnorm: 0.89 [14:06:23< 7:54:39] +[titan] 2025-07-10 03:41:28,900 - root - INFO - step: 64075 loss: 16.7136 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5842 global_avg_mtp_loss: 14.1294 +[titan] 2025-07-10 03:41:28,901 - root - INFO - lr: 1.0863e-04 gnorm: 0.90 [14:06:27< 7:54:35] +[titan] 2025-07-10 03:41:32,815 - root - INFO - step: 64080 loss: 16.5302 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5439 global_avg_mtp_loss: 13.9863 +[titan] 2025-07-10 03:41:32,815 - root - INFO - lr: 1.0861e-04 gnorm: 0.89 [14:06:31< 7:54:31] +[titan] 2025-07-10 03:41:36,720 - root - INFO - step: 64085 loss: 16.3870 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8762 +[titan] 2025-07-10 03:41:36,720 - root - INFO - lr: 1.0859e-04 gnorm: 0.98 [14:06:35< 7:54:27] +[titan] 2025-07-10 03:41:40,633 - root - INFO - step: 64090 loss: 16.3269 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8285 +[titan] 2025-07-10 03:41:40,634 - root - INFO - lr: 1.0857e-04 gnorm: 0.96 [14:06:39< 7:54:23] +[titan] 2025-07-10 03:41:44,551 - root - INFO - step: 64095 loss: 16.4422 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9164 +[titan] 2025-07-10 03:41:44,552 - root - INFO - lr: 1.0855e-04 gnorm: 0.89 [14:06:43< 7:54:19] +[titan] 2025-07-10 03:41:47,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:41:48,473 - root - INFO - step: 64100 loss: 16.4436 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.9186 +[titan] 2025-07-10 03:41:48,474 - root - INFO - lr: 1.0853e-04 gnorm: 0.89 [14:06:47< 7:54:15] +[titan] 2025-07-10 03:41:52,392 - root - INFO - step: 64105 loss: 16.4324 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.8977 +[titan] 2025-07-10 03:41:52,392 - root - INFO - lr: 1.0851e-04 gnorm: 0.91 [14:06:51< 7:54:11] +[titan] 2025-07-10 03:41:56,366 - root - INFO - step: 64110 loss: 16.4300 memory: 44.58GiB(31.99%) tps: 82,459 tflops: 284.58 mfu: 28.77% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.9069 +[titan] 2025-07-10 03:41:56,367 - root - INFO - lr: 1.0849e-04 gnorm: 0.89 [14:06:55< 7:54:07] +[titan] 2025-07-10 03:42:00,275 - root - INFO - step: 64115 loss: 16.2654 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7736 +[titan] 2025-07-10 03:42:00,275 - root - INFO - lr: 1.0847e-04 gnorm: 0.87 [14:06:59< 7:54:03] +[titan] 2025-07-10 03:42:04,182 - root - INFO - step: 64120 loss: 16.4419 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-10 03:42:04,182 - root - INFO - lr: 1.0845e-04 gnorm: 0.93 [14:07:02< 7:53:59] +[titan] 2025-07-10 03:42:08,093 - root - INFO - step: 64125 loss: 16.4602 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9285 +[titan] 2025-07-10 03:42:08,094 - root - INFO - lr: 1.0843e-04 gnorm: 0.88 [14:07:06< 7:53:55] +[titan] 2025-07-10 03:42:12,032 - root - INFO - step: 64130 loss: 16.3117 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8090 +[titan] 2025-07-10 03:42:12,032 - root - INFO - lr: 1.0841e-04 gnorm: 0.93 [14:07:10< 7:53:51] +[titan] 2025-07-10 03:42:15,956 - root - INFO - step: 64135 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8096 +[titan] 2025-07-10 03:42:15,956 - root - INFO - lr: 1.0839e-04 gnorm: 0.87 [14:07:14< 7:53:47] +[titan] 2025-07-10 03:42:19,900 - root - INFO - step: 64140 loss: 16.2245 memory: 44.58GiB(31.99%) tps: 83,091 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.4766 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 03:42:19,900 - root - INFO - lr: 1.0837e-04 gnorm: 0.90 [14:07:18< 7:53:43] +[titan] 2025-07-10 03:42:23,837 - root - INFO - step: 64145 loss: 16.3857 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8714 +[titan] 2025-07-10 03:42:23,837 - root - INFO - lr: 1.0835e-04 gnorm: 0.93 [14:07:22< 7:53:39] +[titan] 2025-07-10 03:42:26,980 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:42:27,768 - root - INFO - step: 64150 loss: 16.2998 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.7978 +[titan] 2025-07-10 03:42:27,769 - root - INFO - lr: 1.0833e-04 gnorm: 0.86 [14:07:26< 7:53:35] +[titan] 2025-07-10 03:42:31,680 - root - INFO - step: 64155 loss: 16.7336 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.1401 +[titan] 2025-07-10 03:42:31,681 - root - INFO - lr: 1.0831e-04 gnorm: 0.92 [14:07:30< 7:53:31] +[titan] 2025-07-10 03:42:35,631 - root - INFO - step: 64160 loss: 16.3028 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.8073 +[titan] 2025-07-10 03:42:35,632 - root - INFO - lr: 1.0829e-04 gnorm: 0.89 [14:07:34< 7:53:27] +[titan] 2025-07-10 03:42:39,537 - root - INFO - step: 64165 loss: 16.5592 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0119 +[titan] 2025-07-10 03:42:39,538 - root - INFO - lr: 1.0828e-04 gnorm: 0.89 [14:07:38< 7:53:23] +[titan] 2025-07-10 03:42:43,480 - root - INFO - step: 64170 loss: 16.4180 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5235 global_avg_mtp_loss: 13.8945 +[titan] 2025-07-10 03:42:43,480 - root - INFO - lr: 1.0826e-04 gnorm: 0.89 [14:07:42< 7:53:19] +[titan] 2025-07-10 03:42:47,483 - root - INFO - step: 64175 loss: 16.4258 memory: 44.58GiB(31.99%) tps: 81,861 tflops: 282.52 mfu: 28.57% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-10 03:42:47,484 - root - INFO - lr: 1.0824e-04 gnorm: 0.85 [14:07:46< 7:53:15] +[titan] 2025-07-10 03:42:51,470 - root - INFO - step: 64180 loss: 16.4739 memory: 44.58GiB(31.99%) tps: 82,200 tflops: 283.69 mfu: 28.68% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9515 +[titan] 2025-07-10 03:42:51,470 - root - INFO - lr: 1.0822e-04 gnorm: 0.93 [14:07:50< 7:53:11] +[titan] 2025-07-10 03:42:55,388 - root - INFO - step: 64185 loss: 16.1200 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.6392 +[titan] 2025-07-10 03:42:55,389 - root - INFO - lr: 1.0820e-04 gnorm: 0.94 [14:07:54< 7:53:07] +[titan] 2025-07-10 03:42:59,342 - root - INFO - step: 64190 loss: 16.2373 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4496 global_avg_mtp_loss: 13.7877 +[titan] 2025-07-10 03:42:59,343 - root - INFO - lr: 1.0818e-04 gnorm: 0.94 [14:07:58< 7:53:03] +[titan] 2025-07-10 03:43:03,279 - root - INFO - step: 64195 loss: 16.5177 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9766 +[titan] 2025-07-10 03:43:03,279 - root - INFO - lr: 1.0816e-04 gnorm: 0.85 [14:08:02< 7:52:59] +[titan] 2025-07-10 03:43:06,441 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:43:07,227 - root - INFO - step: 64200 loss: 16.2693 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.97% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7857 +[titan] 2025-07-10 03:43:07,227 - root - INFO - lr: 1.0814e-04 gnorm: 0.88 [14:08:05< 7:52:55] +[titan] 2025-07-10 03:43:11,166 - root - INFO - step: 64205 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 13.9950 +[titan] 2025-07-10 03:43:11,166 - root - INFO - lr: 1.0812e-04 gnorm: 0.84 [14:08:09< 7:52:51] +[titan] 2025-07-10 03:43:15,098 - root - INFO - step: 64210 loss: 16.2880 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.8007 +[titan] 2025-07-10 03:43:15,098 - root - INFO - lr: 1.0810e-04 gnorm: 0.85 [14:08:13< 7:52:47] +[titan] 2025-07-10 03:43:19,030 - root - INFO - step: 64215 loss: 16.1648 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.7132 +[titan] 2025-07-10 03:43:19,031 - root - INFO - lr: 1.0808e-04 gnorm: 0.87 [14:08:17< 7:52:43] +[titan] 2025-07-10 03:43:22,948 - root - INFO - step: 64220 loss: 15.9012 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4238 global_avg_mtp_loss: 13.4774 +[titan] 2025-07-10 03:43:22,948 - root - INFO - lr: 1.0806e-04 gnorm: 0.87 [14:08:21< 7:52:39] +[titan] 2025-07-10 03:43:26,853 - root - INFO - step: 64225 loss: 16.4015 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8861 +[titan] 2025-07-10 03:43:26,854 - root - INFO - lr: 1.0804e-04 gnorm: 0.86 [14:08:25< 7:52:35] +[titan] 2025-07-10 03:43:30,787 - root - INFO - step: 64230 loss: 16.7081 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5816 global_avg_mtp_loss: 14.1265 +[titan] 2025-07-10 03:43:30,787 - root - INFO - lr: 1.0802e-04 gnorm: 0.89 [14:08:29< 7:52:31] +[titan] 2025-07-10 03:43:34,692 - root - INFO - step: 64235 loss: 16.5658 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0077 +[titan] 2025-07-10 03:43:34,692 - root - INFO - lr: 1.0800e-04 gnorm: 0.90 [14:08:33< 7:52:27] +[titan] 2025-07-10 03:43:38,631 - root - INFO - step: 64240 loss: 16.1253 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.6417 +[titan] 2025-07-10 03:43:38,631 - root - INFO - lr: 1.0798e-04 gnorm: 0.86 [14:08:37< 7:52:23] +[titan] 2025-07-10 03:43:42,622 - root - INFO - step: 64245 loss: 16.3100 memory: 44.58GiB(31.99%) tps: 82,114 tflops: 283.39 mfu: 28.65% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8007 +[titan] 2025-07-10 03:43:42,622 - root - INFO - lr: 1.0796e-04 gnorm: 0.99 [14:08:41< 7:52:19] +[titan] 2025-07-10 03:43:45,753 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:43:46,644 - root - INFO - step: 64250 loss: 16.5499 memory: 44.58GiB(31.99%) tps: 81,471 tflops: 281.17 mfu: 28.43% global_avg_ntp_loss: 2.5514 global_avg_mtp_loss: 13.9985 +[titan] 2025-07-10 03:43:46,644 - root - INFO - lr: 1.0795e-04 gnorm: 0.94 [14:08:45< 7:52:15] +[titan] 2025-07-10 03:43:50,566 - root - INFO - step: 64255 loss: 16.4225 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9057 +[titan] 2025-07-10 03:43:50,566 - root - INFO - lr: 1.0793e-04 gnorm: 0.88 [14:08:49< 7:52:11] +[titan] 2025-07-10 03:43:54,474 - root - INFO - step: 64260 loss: 16.7246 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5709 global_avg_mtp_loss: 14.1537 +[titan] 2025-07-10 03:43:54,474 - root - INFO - lr: 1.0791e-04 gnorm: 0.84 [14:08:53< 7:52:07] +[titan] 2025-07-10 03:43:58,421 - root - INFO - step: 64265 loss: 16.4932 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5294 global_avg_mtp_loss: 13.9637 +[titan] 2025-07-10 03:43:58,421 - root - INFO - lr: 1.0789e-04 gnorm: 0.81 [14:08:57< 7:52:03] +[titan] 2025-07-10 03:44:02,364 - root - INFO - step: 64270 loss: 16.2406 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7596 +[titan] 2025-07-10 03:44:02,365 - root - INFO - lr: 1.0787e-04 gnorm: 0.95 [14:09:01< 7:51:59] +[titan] 2025-07-10 03:44:06,277 - root - INFO - step: 64275 loss: 16.5398 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5361 global_avg_mtp_loss: 14.0037 +[titan] 2025-07-10 03:44:06,277 - root - INFO - lr: 1.0785e-04 gnorm: 0.86 [14:09:05< 7:51:55] +[titan] 2025-07-10 03:44:10,192 - root - INFO - step: 64280 loss: 16.5628 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 14.0186 +[titan] 2025-07-10 03:44:10,192 - root - INFO - lr: 1.0783e-04 gnorm: 0.89 [14:09:08< 7:51:52] +[titan] 2025-07-10 03:44:14,115 - root - INFO - step: 64285 loss: 16.5658 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0153 +[titan] 2025-07-10 03:44:14,115 - root - INFO - lr: 1.0781e-04 gnorm: 0.92 [14:09:12< 7:51:48] +[titan] 2025-07-10 03:44:18,044 - root - INFO - step: 64290 loss: 16.3022 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 03:44:18,044 - root - INFO - lr: 1.0779e-04 gnorm: 0.84 [14:09:16< 7:51:44] +[titan] 2025-07-10 03:44:21,946 - root - INFO - step: 64295 loss: 16.2557 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7693 +[titan] 2025-07-10 03:44:21,947 - root - INFO - lr: 1.0777e-04 gnorm: 0.86 [14:09:20< 7:51:40] +[titan] 2025-07-10 03:44:25,067 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:44:25,872 - root - INFO - step: 64300 loss: 16.3683 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.8565 +[titan] 2025-07-10 03:44:25,872 - root - INFO - lr: 1.0775e-04 gnorm: 0.90 [14:09:24< 7:51:36] +[titan] 2025-07-10 03:44:29,776 - root - INFO - step: 64305 loss: 16.4000 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-10 03:44:29,776 - root - INFO - lr: 1.0773e-04 gnorm: 0.95 [14:09:28< 7:51:32] +[titan] 2025-07-10 03:44:33,676 - root - INFO - step: 64310 loss: 16.3807 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8708 +[titan] 2025-07-10 03:44:33,676 - root - INFO - lr: 1.0771e-04 gnorm: 0.84 [14:09:32< 7:51:28] +[titan] 2025-07-10 03:44:37,620 - root - INFO - step: 64315 loss: 16.1351 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.6708 +[titan] 2025-07-10 03:44:37,621 - root - INFO - lr: 1.0769e-04 gnorm: 0.88 [14:09:36< 7:51:24] +[titan] 2025-07-10 03:44:41,515 - root - INFO - step: 64320 loss: 16.4397 memory: 44.58GiB(31.99%) tps: 84,144 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.9211 +[titan] 2025-07-10 03:44:41,515 - root - INFO - lr: 1.0767e-04 gnorm: 0.87 [14:09:40< 7:51:20] +[titan] 2025-07-10 03:44:45,418 - root - INFO - step: 64325 loss: 16.3805 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8799 +[titan] 2025-07-10 03:44:45,418 - root - INFO - lr: 1.0765e-04 gnorm: 0.88 [14:09:44< 7:51:16] +[titan] 2025-07-10 03:44:49,357 - root - INFO - step: 64330 loss: 16.2569 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.7657 +[titan] 2025-07-10 03:44:49,357 - root - INFO - lr: 1.0763e-04 gnorm: 0.89 [14:09:48< 7:51:12] +[titan] 2025-07-10 03:44:53,281 - root - INFO - step: 64335 loss: 16.1451 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 03:44:53,281 - root - INFO - lr: 1.0762e-04 gnorm: 0.87 [14:09:52< 7:51:08] +[titan] 2025-07-10 03:44:57,181 - root - INFO - step: 64340 loss: 16.6299 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5596 global_avg_mtp_loss: 14.0703 +[titan] 2025-07-10 03:44:57,181 - root - INFO - lr: 1.0760e-04 gnorm: 0.89 [14:09:55< 7:51:04] +[titan] 2025-07-10 03:45:01,092 - root - INFO - step: 64345 loss: 16.1730 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.6923 +[titan] 2025-07-10 03:45:01,092 - root - INFO - lr: 1.0758e-04 gnorm: 0.84 [14:09:59< 7:51:00] +[titan] 2025-07-10 03:45:04,254 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:45:05,043 - root - INFO - step: 64350 loss: 15.9801 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.4298 global_avg_mtp_loss: 13.5503 +[titan] 2025-07-10 03:45:05,043 - root - INFO - lr: 1.0756e-04 gnorm: 0.87 [14:10:03< 7:50:56] +[titan] 2025-07-10 03:45:08,971 - root - INFO - step: 64355 loss: 16.8851 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.6111 global_avg_mtp_loss: 14.2740 +[titan] 2025-07-10 03:45:08,972 - root - INFO - lr: 1.0754e-04 gnorm: 0.85 [14:10:07< 7:50:52] +[titan] 2025-07-10 03:45:12,874 - root - INFO - step: 64360 loss: 16.2590 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7713 +[titan] 2025-07-10 03:45:12,875 - root - INFO - lr: 1.0752e-04 gnorm: 0.85 [14:10:11< 7:50:48] +[titan] 2025-07-10 03:45:16,829 - root - INFO - step: 64365 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8365 +[titan] 2025-07-10 03:45:16,829 - root - INFO - lr: 1.0750e-04 gnorm: 0.96 [14:10:15< 7:50:44] +[titan] 2025-07-10 03:45:20,778 - root - INFO - step: 64370 loss: 16.0759 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4512 global_avg_mtp_loss: 13.6247 +[titan] 2025-07-10 03:45:20,778 - root - INFO - lr: 1.0748e-04 gnorm: 0.96 [14:10:19< 7:50:40] +[titan] 2025-07-10 03:45:24,727 - root - INFO - step: 64375 loss: 16.3657 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-10 03:45:24,727 - root - INFO - lr: 1.0746e-04 gnorm: 0.91 [14:10:23< 7:50:36] +[titan] 2025-07-10 03:45:28,678 - root - INFO - step: 64380 loss: 16.4957 memory: 44.58GiB(31.99%) tps: 82,936 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9547 +[titan] 2025-07-10 03:45:28,679 - root - INFO - lr: 1.0744e-04 gnorm: 0.91 [14:10:27< 7:50:32] +[titan] 2025-07-10 03:45:32,594 - root - INFO - step: 64385 loss: 16.4557 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 03:45:32,594 - root - INFO - lr: 1.0742e-04 gnorm: 0.88 [14:10:31< 7:50:28] +[titan] 2025-07-10 03:45:36,517 - root - INFO - step: 64390 loss: 16.3910 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8784 +[titan] 2025-07-10 03:45:36,517 - root - INFO - lr: 1.0740e-04 gnorm: 0.91 [14:10:35< 7:50:24] +[titan] 2025-07-10 03:45:40,429 - root - INFO - step: 64395 loss: 16.3384 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.8376 +[titan] 2025-07-10 03:45:40,429 - root - INFO - lr: 1.0738e-04 gnorm: 1.07 [14:10:39< 7:50:20] +[titan] 2025-07-10 03:45:43,577 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:45:44,366 - root - INFO - step: 64400 loss: 16.2051 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7327 +[titan] 2025-07-10 03:45:44,366 - root - INFO - lr: 1.0736e-04 gnorm: 0.88 [14:10:43< 7:50:16] +[titan] 2025-07-10 03:45:48,304 - root - INFO - step: 64405 loss: 16.5286 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5294 global_avg_mtp_loss: 13.9992 +[titan] 2025-07-10 03:45:48,304 - root - INFO - lr: 1.0734e-04 gnorm: 0.85 [14:10:47< 7:50:12] +[titan] 2025-07-10 03:45:52,209 - root - INFO - step: 64410 loss: 16.3791 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8779 +[titan] 2025-07-10 03:45:52,210 - root - INFO - lr: 1.0732e-04 gnorm: 0.86 [14:10:50< 7:50:08] +[titan] 2025-07-10 03:45:56,140 - root - INFO - step: 64415 loss: 16.3183 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.8148 +[titan] 2025-07-10 03:45:56,140 - root - INFO - lr: 1.0731e-04 gnorm: 0.97 [14:10:54< 7:50:04] +[titan] 2025-07-10 03:46:00,068 - root - INFO - step: 64420 loss: 16.2666 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.7728 +[titan] 2025-07-10 03:46:00,068 - root - INFO - lr: 1.0729e-04 gnorm: 0.91 [14:10:58< 7:50:00] +[titan] 2025-07-10 03:46:03,990 - root - INFO - step: 64425 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.9659 +[titan] 2025-07-10 03:46:03,990 - root - INFO - lr: 1.0727e-04 gnorm: 0.85 [14:11:02< 7:49:56] +[titan] 2025-07-10 03:46:07,986 - root - INFO - step: 64430 loss: 16.2103 memory: 44.58GiB(31.99%) tps: 82,008 tflops: 283.02 mfu: 28.62% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7319 +[titan] 2025-07-10 03:46:07,986 - root - INFO - lr: 1.0725e-04 gnorm: 0.88 [14:11:06< 7:49:52] +[titan] 2025-07-10 03:46:11,924 - root - INFO - step: 64435 loss: 16.2528 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7664 +[titan] 2025-07-10 03:46:11,924 - root - INFO - lr: 1.0723e-04 gnorm: 0.82 [14:11:10< 7:49:48] +[titan] 2025-07-10 03:46:15,867 - root - INFO - step: 64440 loss: 16.4251 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 13.8954 +[titan] 2025-07-10 03:46:15,867 - root - INFO - lr: 1.0721e-04 gnorm: 0.86 [14:11:14< 7:49:44] +[titan] 2025-07-10 03:46:19,790 - root - INFO - step: 64445 loss: 16.3478 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8487 +[titan] 2025-07-10 03:46:19,790 - root - INFO - lr: 1.0719e-04 gnorm: 0.82 [14:11:18< 7:49:40] +[titan] 2025-07-10 03:46:22,920 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:46:23,708 - root - INFO - step: 64450 loss: 16.3320 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8351 +[titan] 2025-07-10 03:46:23,709 - root - INFO - lr: 1.0717e-04 gnorm: 0.84 [14:11:22< 7:49:36] +[titan] 2025-07-10 03:46:27,625 - root - INFO - step: 64455 loss: 16.2782 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7898 +[titan] 2025-07-10 03:46:27,625 - root - INFO - lr: 1.0715e-04 gnorm: 0.87 [14:11:26< 7:49:32] +[titan] 2025-07-10 03:46:31,564 - root - INFO - step: 64460 loss: 16.5451 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5490 global_avg_mtp_loss: 13.9961 +[titan] 2025-07-10 03:46:31,565 - root - INFO - lr: 1.0713e-04 gnorm: 0.90 [14:11:30< 7:49:28] +[titan] 2025-07-10 03:46:35,470 - root - INFO - step: 64465 loss: 16.5637 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 14.0086 +[titan] 2025-07-10 03:46:35,470 - root - INFO - lr: 1.0711e-04 gnorm: 0.89 [14:11:34< 7:49:24] +[titan] 2025-07-10 03:46:39,387 - root - INFO - step: 64470 loss: 16.4725 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9386 +[titan] 2025-07-10 03:46:39,388 - root - INFO - lr: 1.0709e-04 gnorm: 0.90 [14:11:38< 7:49:20] +[titan] 2025-07-10 03:46:43,349 - root - INFO - step: 64475 loss: 16.7715 memory: 44.58GiB(31.99%) tps: 82,712 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.5922 global_avg_mtp_loss: 14.1792 +[titan] 2025-07-10 03:46:43,350 - root - INFO - lr: 1.0707e-04 gnorm: 0.91 [14:11:42< 7:49:16] +[titan] 2025-07-10 03:46:47,277 - root - INFO - step: 64480 loss: 16.3640 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8594 +[titan] 2025-07-10 03:46:47,277 - root - INFO - lr: 1.0705e-04 gnorm: 0.89 [14:11:45< 7:49:12] +[titan] 2025-07-10 03:46:51,240 - root - INFO - step: 64485 loss: 16.1285 memory: 44.58GiB(31.99%) tps: 82,684 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.6541 +[titan] 2025-07-10 03:46:51,240 - root - INFO - lr: 1.0703e-04 gnorm: 0.88 [14:11:49< 7:49:08] +[titan] 2025-07-10 03:46:55,205 - root - INFO - step: 64490 loss: 16.2335 memory: 44.58GiB(31.99%) tps: 82,663 tflops: 285.28 mfu: 28.85% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7551 +[titan] 2025-07-10 03:46:55,205 - root - INFO - lr: 1.0701e-04 gnorm: 0.86 [14:11:53< 7:49:04] +[titan] 2025-07-10 03:46:59,147 - root - INFO - step: 64495 loss: 16.2807 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7821 +[titan] 2025-07-10 03:46:59,148 - root - INFO - lr: 1.0700e-04 gnorm: 0.94 [14:11:57< 7:49:00] +[titan] 2025-07-10 03:47:02,277 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:47:03,066 - root - INFO - step: 64500 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.7868 +[titan] 2025-07-10 03:47:03,067 - root - INFO - lr: 1.0698e-04 gnorm: 0.96 [14:12:01< 7:48:56] +[titan] 2025-07-10 03:47:06,963 - root - INFO - step: 64505 loss: 16.2310 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7490 +[titan] 2025-07-10 03:47:06,963 - root - INFO - lr: 1.0696e-04 gnorm: 0.92 [14:12:05< 7:48:52] +[titan] 2025-07-10 03:47:10,967 - root - INFO - step: 64510 loss: 16.2714 memory: 44.58GiB(31.99%) tps: 81,839 tflops: 282.44 mfu: 28.56% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.7782 +[titan] 2025-07-10 03:47:10,967 - root - INFO - lr: 1.0694e-04 gnorm: 0.88 [14:12:09< 7:48:48] +[titan] 2025-07-10 03:47:12,719 - root - INFO - Dumping profiler traces at step 64512 +[titan] 2025-07-10 03:47:12,752 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:47:15,145 - root - INFO - step: 64515 loss: 16.3879 memory: 44.58GiB(31.99%) tps: 78,448 tflops: 270.74 mfu: 27.37% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8712 +[titan] 2025-07-10 03:47:15,145 - root - INFO - lr: 1.0692e-04 gnorm: 0.89 [14:12:13< 7:48:45] +[titan] 2025-07-10 03:47:19,098 - root - INFO - step: 64520 loss: 16.4850 memory: 44.58GiB(31.99%) tps: 82,895 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9441 +[titan] 2025-07-10 03:47:19,098 - root - INFO - lr: 1.0690e-04 gnorm: 0.91 [14:12:17< 7:48:41] +[titan] 2025-07-10 03:47:23,029 - root - INFO - step: 64525 loss: 16.3631 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8608 +[titan] 2025-07-10 03:47:23,029 - root - INFO - lr: 1.0688e-04 gnorm: 0.90 [14:12:21< 7:48:37] +[titan] 2025-07-10 03:47:26,923 - root - INFO - step: 64530 loss: 16.1838 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 03:47:26,923 - root - INFO - lr: 1.0686e-04 gnorm: 0.96 [14:12:25< 7:48:33] +[titan] 2025-07-10 03:47:30,819 - root - INFO - step: 64535 loss: 16.3649 memory: 44.58GiB(31.99%) tps: 84,115 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8485 +[titan] 2025-07-10 03:47:30,819 - root - INFO - lr: 1.0684e-04 gnorm: 0.89 [14:12:29< 7:48:29] +[titan] 2025-07-10 03:47:34,754 - root - INFO - step: 64540 loss: 16.6324 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5734 global_avg_mtp_loss: 14.0590 +[titan] 2025-07-10 03:47:34,754 - root - INFO - lr: 1.0682e-04 gnorm: 0.95 [14:12:33< 7:48:25] +[titan] 2025-07-10 03:47:38,648 - root - INFO - step: 64545 loss: 16.5180 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 13.9810 +[titan] 2025-07-10 03:47:38,649 - root - INFO - lr: 1.0680e-04 gnorm: 0.86 [14:12:37< 7:48:21] +[titan] 2025-07-10 03:47:41,770 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:47:42,569 - root - INFO - step: 64550 loss: 16.4695 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9437 +[titan] 2025-07-10 03:47:42,570 - root - INFO - lr: 1.0678e-04 gnorm: 0.88 [14:12:41< 7:48:17] +[titan] 2025-07-10 03:47:46,502 - root - INFO - step: 64555 loss: 16.5264 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9814 +[titan] 2025-07-10 03:47:46,502 - root - INFO - lr: 1.0676e-04 gnorm: 0.90 [14:12:45< 7:48:13] +[titan] 2025-07-10 03:47:50,412 - root - INFO - step: 64560 loss: 16.2000 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7247 +[titan] 2025-07-10 03:47:50,413 - root - INFO - lr: 1.0674e-04 gnorm: 0.90 [14:12:49< 7:48:09] +[titan] 2025-07-10 03:47:54,322 - root - INFO - step: 64565 loss: 16.5034 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 13.9524 +[titan] 2025-07-10 03:47:54,323 - root - INFO - lr: 1.0672e-04 gnorm: 0.90 [14:12:53< 7:48:05] +[titan] 2025-07-10 03:47:58,250 - root - INFO - step: 64570 loss: 16.5034 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.9810 +[titan] 2025-07-10 03:47:58,250 - root - INFO - lr: 1.0671e-04 gnorm: 0.85 [14:12:56< 7:48:01] +[titan] 2025-07-10 03:48:02,191 - root - INFO - step: 64575 loss: 16.0807 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.6126 +[titan] 2025-07-10 03:48:02,191 - root - INFO - lr: 1.0669e-04 gnorm: 0.91 [14:13:00< 7:47:57] +[titan] 2025-07-10 03:48:06,120 - root - INFO - step: 64580 loss: 16.4021 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.8740 +[titan] 2025-07-10 03:48:06,120 - root - INFO - lr: 1.0667e-04 gnorm: 0.86 [14:13:04< 7:47:53] +[titan] 2025-07-10 03:48:10,016 - root - INFO - step: 64585 loss: 16.3098 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.7965 +[titan] 2025-07-10 03:48:10,016 - root - INFO - lr: 1.0665e-04 gnorm: 0.90 [14:13:08< 7:47:49] +[titan] 2025-07-10 03:48:13,985 - root - INFO - step: 64590 loss: 16.4586 memory: 44.58GiB(31.99%) tps: 82,559 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.9435 +[titan] 2025-07-10 03:48:13,985 - root - INFO - lr: 1.0663e-04 gnorm: 0.92 [14:13:12< 7:47:45] +[titan] 2025-07-10 03:48:17,917 - root - INFO - step: 64595 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,339 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 03:48:17,918 - root - INFO - lr: 1.0661e-04 gnorm: 0.98 [14:13:16< 7:47:41] +[titan] 2025-07-10 03:48:21,039 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:48:21,843 - root - INFO - step: 64600 loss: 16.4668 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9366 +[titan] 2025-07-10 03:48:21,843 - root - INFO - lr: 1.0659e-04 gnorm: 0.88 [14:13:20< 7:47:37] +[titan] 2025-07-10 03:48:25,773 - root - INFO - step: 64605 loss: 15.9443 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4486 global_avg_mtp_loss: 13.4957 +[titan] 2025-07-10 03:48:25,773 - root - INFO - lr: 1.0657e-04 gnorm: 1.03 [14:13:24< 7:47:33] +[titan] 2025-07-10 03:48:29,693 - root - INFO - step: 64610 loss: 16.6021 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5559 global_avg_mtp_loss: 14.0462 +[titan] 2025-07-10 03:48:29,694 - root - INFO - lr: 1.0655e-04 gnorm: 0.90 [14:13:28< 7:47:29] +[titan] 2025-07-10 03:48:33,618 - root - INFO - step: 64615 loss: 16.5498 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5463 global_avg_mtp_loss: 14.0036 +[titan] 2025-07-10 03:48:33,618 - root - INFO - lr: 1.0653e-04 gnorm: 0.89 [14:13:32< 7:47:25] +[titan] 2025-07-10 03:48:37,520 - root - INFO - step: 64620 loss: 16.5766 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 14.0412 +[titan] 2025-07-10 03:48:37,520 - root - INFO - lr: 1.0651e-04 gnorm: 0.96 [14:13:36< 7:47:21] +[titan] 2025-07-10 03:48:41,419 - root - INFO - step: 64625 loss: 16.3085 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8026 +[titan] 2025-07-10 03:48:41,419 - root - INFO - lr: 1.0649e-04 gnorm: 0.92 [14:13:40< 7:47:17] +[titan] 2025-07-10 03:48:45,401 - root - INFO - step: 64630 loss: 16.3140 memory: 44.58GiB(31.99%) tps: 82,299 tflops: 284.03 mfu: 28.72% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8117 +[titan] 2025-07-10 03:48:45,401 - root - INFO - lr: 1.0647e-04 gnorm: 0.87 [14:13:44< 7:47:13] +[titan] 2025-07-10 03:48:49,326 - root - INFO - step: 64635 loss: 16.2146 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7286 +[titan] 2025-07-10 03:48:49,326 - root - INFO - lr: 1.0645e-04 gnorm: 0.87 [14:13:48< 7:47:09] +[titan] 2025-07-10 03:48:53,247 - root - INFO - step: 64640 loss: 16.2678 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-10 03:48:53,247 - root - INFO - lr: 1.0644e-04 gnorm: 0.85 [14:13:51< 7:47:05] +[titan] 2025-07-10 03:48:57,198 - root - INFO - step: 64645 loss: 16.4849 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9566 +[titan] 2025-07-10 03:48:57,198 - root - INFO - lr: 1.0642e-04 gnorm: 0.89 [14:13:55< 7:47:01] +[titan] 2025-07-10 03:49:00,372 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:49:01,176 - root - INFO - step: 64650 loss: 16.5166 memory: 44.58GiB(31.99%) tps: 82,376 tflops: 284.29 mfu: 28.75% global_avg_ntp_loss: 2.5408 global_avg_mtp_loss: 13.9758 +[titan] 2025-07-10 03:49:01,176 - root - INFO - lr: 1.0640e-04 gnorm: 0.95 [14:13:59< 7:46:57] +[titan] 2025-07-10 03:49:05,118 - root - INFO - step: 64655 loss: 16.3156 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.8272 +[titan] 2025-07-10 03:49:05,118 - root - INFO - lr: 1.0638e-04 gnorm: 0.93 [14:14:03< 7:46:53] +[titan] 2025-07-10 03:49:09,049 - root - INFO - step: 64660 loss: 16.2906 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7945 +[titan] 2025-07-10 03:49:09,049 - root - INFO - lr: 1.0636e-04 gnorm: 0.91 [14:14:07< 7:46:49] +[titan] 2025-07-10 03:49:12,976 - root - INFO - step: 64665 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8824 +[titan] 2025-07-10 03:49:12,976 - root - INFO - lr: 1.0634e-04 gnorm: 0.88 [14:14:11< 7:46:45] +[titan] 2025-07-10 03:49:16,901 - root - INFO - step: 64670 loss: 16.0543 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4484 global_avg_mtp_loss: 13.6060 +[titan] 2025-07-10 03:49:16,901 - root - INFO - lr: 1.0632e-04 gnorm: 0.88 [14:14:15< 7:46:41] +[titan] 2025-07-10 03:49:20,814 - root - INFO - step: 64675 loss: 16.4541 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9296 +[titan] 2025-07-10 03:49:20,814 - root - INFO - lr: 1.0630e-04 gnorm: 0.84 [14:14:19< 7:46:37] +[titan] 2025-07-10 03:49:24,729 - root - INFO - step: 64680 loss: 16.4546 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5307 global_avg_mtp_loss: 13.9239 +[titan] 2025-07-10 03:49:24,730 - root - INFO - lr: 1.0628e-04 gnorm: 0.84 [14:14:23< 7:46:33] +[titan] 2025-07-10 03:49:28,652 - root - INFO - step: 64685 loss: 16.1881 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7082 +[titan] 2025-07-10 03:49:28,653 - root - INFO - lr: 1.0626e-04 gnorm: 0.87 [14:14:27< 7:46:29] +[titan] 2025-07-10 03:49:32,590 - root - INFO - step: 64690 loss: 16.4451 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9264 +[titan] 2025-07-10 03:49:32,590 - root - INFO - lr: 1.0624e-04 gnorm: 0.86 [14:14:31< 7:46:25] +[titan] 2025-07-10 03:49:36,497 - root - INFO - step: 64695 loss: 15.9382 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4295 global_avg_mtp_loss: 13.5088 +[titan] 2025-07-10 03:49:36,497 - root - INFO - lr: 1.0622e-04 gnorm: 0.91 [14:14:35< 7:46:21] +[titan] 2025-07-10 03:49:39,623 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:49:40,410 - root - INFO - step: 64700 loss: 16.2771 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7914 +[titan] 2025-07-10 03:49:40,410 - root - INFO - lr: 1.0620e-04 gnorm: 0.88 [14:14:39< 7:46:17] +[titan] 2025-07-10 03:49:44,320 - root - INFO - step: 64705 loss: 16.4740 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9456 +[titan] 2025-07-10 03:49:44,320 - root - INFO - lr: 1.0618e-04 gnorm: 0.87 [14:14:43< 7:46:13] +[titan] 2025-07-10 03:49:48,218 - root - INFO - step: 64710 loss: 16.1899 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7042 +[titan] 2025-07-10 03:49:48,219 - root - INFO - lr: 1.0617e-04 gnorm: 0.92 [14:14:46< 7:46:09] +[titan] 2025-07-10 03:49:52,123 - root - INFO - step: 64715 loss: 16.3717 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.8782 +[titan] 2025-07-10 03:49:52,123 - root - INFO - lr: 1.0615e-04 gnorm: 0.89 [14:14:50< 7:46:05] +[titan] 2025-07-10 03:49:56,027 - root - INFO - step: 64720 loss: 16.4091 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.8829 +[titan] 2025-07-10 03:49:56,027 - root - INFO - lr: 1.0613e-04 gnorm: 0.85 [14:14:54< 7:46:01] +[titan] 2025-07-10 03:49:59,961 - root - INFO - step: 64725 loss: 16.1934 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.7223 +[titan] 2025-07-10 03:49:59,961 - root - INFO - lr: 1.0611e-04 gnorm: 0.90 [14:14:58< 7:45:57] +[titan] 2025-07-10 03:50:03,938 - root - INFO - step: 64730 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 82,387 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.9057 +[titan] 2025-07-10 03:50:03,938 - root - INFO - lr: 1.0609e-04 gnorm: 0.91 [14:15:02< 7:45:53] +[titan] 2025-07-10 03:50:07,895 - root - INFO - step: 64735 loss: 16.3138 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.7916 +[titan] 2025-07-10 03:50:07,895 - root - INFO - lr: 1.0607e-04 gnorm: 0.87 [14:15:06< 7:45:49] +[titan] 2025-07-10 03:50:11,799 - root - INFO - step: 64740 loss: 16.4030 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8868 +[titan] 2025-07-10 03:50:11,799 - root - INFO - lr: 1.0605e-04 gnorm: 0.93 [14:15:10< 7:45:45] +[titan] 2025-07-10 03:50:15,759 - root - INFO - step: 64745 loss: 16.1208 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6566 +[titan] 2025-07-10 03:50:15,759 - root - INFO - lr: 1.0603e-04 gnorm: 0.89 [14:15:14< 7:45:41] +[titan] 2025-07-10 03:50:18,886 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:50:19,682 - root - INFO - step: 64750 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6961 +[titan] 2025-07-10 03:50:19,682 - root - INFO - lr: 1.0601e-04 gnorm: 0.90 [14:15:18< 7:45:37] +[titan] 2025-07-10 03:50:23,585 - root - INFO - step: 64755 loss: 16.5153 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9957 +[titan] 2025-07-10 03:50:23,585 - root - INFO - lr: 1.0599e-04 gnorm: 0.91 [14:15:22< 7:45:33] +[titan] 2025-07-10 03:50:27,498 - root - INFO - step: 64760 loss: 16.8020 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5996 global_avg_mtp_loss: 14.2023 +[titan] 2025-07-10 03:50:27,498 - root - INFO - lr: 1.0597e-04 gnorm: 0.88 [14:15:26< 7:45:29] +[titan] 2025-07-10 03:50:31,443 - root - INFO - step: 64765 loss: 16.3488 memory: 44.58GiB(31.99%) tps: 83,069 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8557 +[titan] 2025-07-10 03:50:31,443 - root - INFO - lr: 1.0595e-04 gnorm: 0.88 [14:15:30< 7:45:25] +[titan] 2025-07-10 03:50:35,398 - root - INFO - step: 64770 loss: 16.2667 memory: 44.58GiB(31.99%) tps: 82,851 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7764 +[titan] 2025-07-10 03:50:35,398 - root - INFO - lr: 1.0593e-04 gnorm: 0.89 [14:15:34< 7:45:21] +[titan] 2025-07-10 03:50:39,333 - root - INFO - step: 64775 loss: 16.4522 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9171 +[titan] 2025-07-10 03:50:39,333 - root - INFO - lr: 1.0591e-04 gnorm: 0.83 [14:15:38< 7:45:17] +[titan] 2025-07-10 03:50:43,254 - root - INFO - step: 64780 loss: 16.3551 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8434 +[titan] 2025-07-10 03:50:43,254 - root - INFO - lr: 1.0590e-04 gnorm: 0.88 [14:15:41< 7:45:13] +[titan] 2025-07-10 03:50:47,168 - root - INFO - step: 64785 loss: 16.5787 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 14.0337 +[titan] 2025-07-10 03:50:47,168 - root - INFO - lr: 1.0588e-04 gnorm: 0.88 [14:15:45< 7:45:09] +[titan] 2025-07-10 03:50:51,080 - root - INFO - step: 64790 loss: 16.4218 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.8923 +[titan] 2025-07-10 03:50:51,081 - root - INFO - lr: 1.0586e-04 gnorm: 0.87 [14:15:49< 7:45:05] +[titan] 2025-07-10 03:50:54,990 - root - INFO - step: 64795 loss: 16.4462 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9197 +[titan] 2025-07-10 03:50:54,990 - root - INFO - lr: 1.0584e-04 gnorm: 0.89 [14:15:53< 7:45:01] +[titan] 2025-07-10 03:50:58,183 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:50:58,969 - root - INFO - step: 64800 loss: 16.2602 memory: 44.58GiB(31.99%) tps: 82,351 tflops: 284.21 mfu: 28.74% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.7596 +[titan] 2025-07-10 03:50:58,969 - root - INFO - lr: 1.0582e-04 gnorm: 0.89 [14:15:57< 7:44:57] +[titan] 2025-07-10 03:51:02,873 - root - INFO - step: 64805 loss: 16.2307 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7355 +[titan] 2025-07-10 03:51:02,874 - root - INFO - lr: 1.0580e-04 gnorm: 0.87 [14:16:01< 7:44:53] +[titan] 2025-07-10 03:51:06,786 - root - INFO - step: 64810 loss: 16.1218 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6556 +[titan] 2025-07-10 03:51:06,786 - root - INFO - lr: 1.0578e-04 gnorm: 0.92 [14:16:05< 7:44:49] +[titan] 2025-07-10 03:51:10,692 - root - INFO - step: 64815 loss: 16.2533 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7660 +[titan] 2025-07-10 03:51:10,693 - root - INFO - lr: 1.0576e-04 gnorm: 0.88 [14:16:09< 7:44:46] +[titan] 2025-07-10 03:51:14,609 - root - INFO - step: 64820 loss: 16.5954 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5623 global_avg_mtp_loss: 14.0331 +[titan] 2025-07-10 03:51:14,609 - root - INFO - lr: 1.0574e-04 gnorm: 0.92 [14:16:13< 7:44:42] +[titan] 2025-07-10 03:51:18,528 - root - INFO - step: 64825 loss: 16.4718 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9360 +[titan] 2025-07-10 03:51:18,528 - root - INFO - lr: 1.0572e-04 gnorm: 0.86 [14:16:17< 7:44:38] +[titan] 2025-07-10 03:51:22,439 - root - INFO - step: 64830 loss: 16.5564 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 14.0173 +[titan] 2025-07-10 03:51:22,439 - root - INFO - lr: 1.0570e-04 gnorm: 0.86 [14:16:21< 7:44:34] +[titan] 2025-07-10 03:51:26,342 - root - INFO - step: 64835 loss: 16.3057 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 03:51:26,343 - root - INFO - lr: 1.0568e-04 gnorm: 0.89 [14:16:25< 7:44:30] +[titan] 2025-07-10 03:51:30,331 - root - INFO - step: 64840 loss: 16.2567 memory: 44.58GiB(31.99%) tps: 82,163 tflops: 283.56 mfu: 28.67% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7606 +[titan] 2025-07-10 03:51:30,331 - root - INFO - lr: 1.0566e-04 gnorm: 0.89 [14:16:29< 7:44:26] +[titan] 2025-07-10 03:51:34,220 - root - INFO - step: 64845 loss: 16.2977 memory: 44.58GiB(31.99%) tps: 84,266 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8063 +[titan] 2025-07-10 03:51:34,220 - root - INFO - lr: 1.0565e-04 gnorm: 0.93 [14:16:32< 7:44:22] +[titan] 2025-07-10 03:51:37,347 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:51:38,137 - root - INFO - step: 64850 loss: 16.3749 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8671 +[titan] 2025-07-10 03:51:38,137 - root - INFO - lr: 1.0563e-04 gnorm: 0.91 [14:16:36< 7:44:18] +[titan] 2025-07-10 03:51:42,098 - root - INFO - step: 64855 loss: 16.5973 memory: 44.58GiB(31.99%) tps: 82,726 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.5499 global_avg_mtp_loss: 14.0474 +[titan] 2025-07-10 03:51:42,098 - root - INFO - lr: 1.0561e-04 gnorm: 0.90 [14:16:40< 7:44:14] +[titan] 2025-07-10 03:51:46,038 - root - INFO - step: 64860 loss: 16.1228 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6589 +[titan] 2025-07-10 03:51:46,038 - root - INFO - lr: 1.0559e-04 gnorm: 0.90 [14:16:44< 7:44:10] +[titan] 2025-07-10 03:51:49,946 - root - INFO - step: 64865 loss: 16.2709 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.7522 +[titan] 2025-07-10 03:51:49,946 - root - INFO - lr: 1.0557e-04 gnorm: 0.89 [14:16:48< 7:44:06] +[titan] 2025-07-10 03:51:53,851 - root - INFO - step: 64870 loss: 16.2146 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7343 +[titan] 2025-07-10 03:51:53,851 - root - INFO - lr: 1.0555e-04 gnorm: 0.84 [14:16:52< 7:44:02] +[titan] 2025-07-10 03:51:57,805 - root - INFO - step: 64875 loss: 16.4434 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-10 03:51:57,805 - root - INFO - lr: 1.0553e-04 gnorm: 0.87 [14:16:56< 7:43:58] +[titan] 2025-07-10 03:52:01,743 - root - INFO - step: 64880 loss: 16.5213 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9878 +[titan] 2025-07-10 03:52:01,743 - root - INFO - lr: 1.0551e-04 gnorm: 0.87 [14:17:00< 7:43:54] +[titan] 2025-07-10 03:52:05,650 - root - INFO - step: 64885 loss: 16.2144 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7345 +[titan] 2025-07-10 03:52:05,650 - root - INFO - lr: 1.0549e-04 gnorm: 0.86 [14:17:04< 7:43:50] +[titan] 2025-07-10 03:52:09,565 - root - INFO - step: 64890 loss: 16.0877 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.6328 +[titan] 2025-07-10 03:52:09,565 - root - INFO - lr: 1.0547e-04 gnorm: 0.88 [14:17:08< 7:43:46] +[titan] 2025-07-10 03:52:13,490 - root - INFO - step: 64895 loss: 16.2588 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7687 +[titan] 2025-07-10 03:52:13,490 - root - INFO - lr: 1.0545e-04 gnorm: 0.92 [14:17:12< 7:43:42] +[titan] 2025-07-10 03:52:16,644 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:52:17,439 - root - INFO - step: 64900 loss: 16.2373 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7537 +[titan] 2025-07-10 03:52:17,439 - root - INFO - lr: 1.0543e-04 gnorm: 0.92 [14:17:16< 7:43:38] +[titan] 2025-07-10 03:52:21,355 - root - INFO - step: 64905 loss: 16.5249 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9831 +[titan] 2025-07-10 03:52:21,356 - root - INFO - lr: 1.0541e-04 gnorm: 0.94 [14:17:20< 7:43:34] +[titan] 2025-07-10 03:52:25,297 - root - INFO - step: 64910 loss: 16.5790 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0296 +[titan] 2025-07-10 03:52:25,297 - root - INFO - lr: 1.0540e-04 gnorm: 0.92 [14:17:23< 7:43:30] +[titan] 2025-07-10 03:52:29,248 - root - INFO - step: 64915 loss: 16.5223 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9798 +[titan] 2025-07-10 03:52:29,248 - root - INFO - lr: 1.0538e-04 gnorm: 0.89 [14:17:27< 7:43:26] +[titan] 2025-07-10 03:52:33,208 - root - INFO - step: 64920 loss: 16.5220 memory: 44.58GiB(31.99%) tps: 82,761 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.5479 global_avg_mtp_loss: 13.9740 +[titan] 2025-07-10 03:52:33,208 - root - INFO - lr: 1.0536e-04 gnorm: 0.92 [14:17:31< 7:43:22] +[titan] 2025-07-10 03:52:37,098 - root - INFO - step: 64925 loss: 16.4103 memory: 44.58GiB(31.99%) tps: 84,249 tflops: 290.76 mfu: 29.40% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.9006 +[titan] 2025-07-10 03:52:37,098 - root - INFO - lr: 1.0534e-04 gnorm: 0.89 [14:17:35< 7:43:18] +[titan] 2025-07-10 03:52:41,017 - root - INFO - step: 64930 loss: 16.3066 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.7984 +[titan] 2025-07-10 03:52:41,017 - root - INFO - lr: 1.0532e-04 gnorm: 0.88 [14:17:39< 7:43:14] +[titan] 2025-07-10 03:52:44,921 - root - INFO - step: 64935 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.9730 +[titan] 2025-07-10 03:52:44,921 - root - INFO - lr: 1.0530e-04 gnorm: 0.92 [14:17:43< 7:43:10] +[titan] 2025-07-10 03:52:48,824 - root - INFO - step: 64940 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 03:52:48,824 - root - INFO - lr: 1.0528e-04 gnorm: 0.90 [14:17:47< 7:43:06] +[titan] 2025-07-10 03:52:52,748 - root - INFO - step: 64945 loss: 16.4270 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.9053 +[titan] 2025-07-10 03:52:52,749 - root - INFO - lr: 1.0526e-04 gnorm: 0.95 [14:17:51< 7:43:02] +[titan] 2025-07-10 03:52:55,882 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:52:56,683 - root - INFO - step: 64950 loss: 16.6033 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5556 global_avg_mtp_loss: 14.0477 +[titan] 2025-07-10 03:52:56,684 - root - INFO - lr: 1.0524e-04 gnorm: 0.90 [14:17:55< 7:42:58] +[titan] 2025-07-10 03:53:00,663 - root - INFO - step: 64955 loss: 16.3041 memory: 44.58GiB(31.99%) tps: 82,341 tflops: 284.17 mfu: 28.73% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8124 +[titan] 2025-07-10 03:53:00,664 - root - INFO - lr: 1.0522e-04 gnorm: 0.93 [14:17:59< 7:42:54] +[titan] 2025-07-10 03:53:04,611 - root - INFO - step: 64960 loss: 16.3711 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8624 +[titan] 2025-07-10 03:53:04,611 - root - INFO - lr: 1.0520e-04 gnorm: 0.91 [14:18:03< 7:42:50] +[titan] 2025-07-10 03:53:08,536 - root - INFO - step: 64965 loss: 16.4717 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.9460 +[titan] 2025-07-10 03:53:08,536 - root - INFO - lr: 1.0518e-04 gnorm: 0.92 [14:18:07< 7:42:46] +[titan] 2025-07-10 03:53:12,446 - root - INFO - step: 64970 loss: 16.6626 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.1051 +[titan] 2025-07-10 03:53:12,447 - root - INFO - lr: 1.0516e-04 gnorm: 0.91 [14:18:11< 7:42:42] +[titan] 2025-07-10 03:53:16,367 - root - INFO - step: 64975 loss: 16.3804 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.8626 +[titan] 2025-07-10 03:53:16,367 - root - INFO - lr: 1.0515e-04 gnorm: 0.86 [14:18:15< 7:42:38] +[titan] 2025-07-10 03:53:20,280 - root - INFO - step: 64980 loss: 16.5181 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.9871 +[titan] 2025-07-10 03:53:20,280 - root - INFO - lr: 1.0513e-04 gnorm: 0.86 [14:18:18< 7:42:34] +[titan] 2025-07-10 03:53:24,180 - root - INFO - step: 64985 loss: 16.3551 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 03:53:24,180 - root - INFO - lr: 1.0511e-04 gnorm: 0.85 [14:18:22< 7:42:30] +[titan] 2025-07-10 03:53:28,110 - root - INFO - step: 64990 loss: 16.5604 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 14.0187 +[titan] 2025-07-10 03:53:28,111 - root - INFO - lr: 1.0509e-04 gnorm: 0.91 [14:18:26< 7:42:26] +[titan] 2025-07-10 03:53:32,020 - root - INFO - step: 64995 loss: 16.7210 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5800 global_avg_mtp_loss: 14.1410 +[titan] 2025-07-10 03:53:32,021 - root - INFO - lr: 1.0507e-04 gnorm: 0.89 [14:18:30< 7:42:22] +[titan] 2025-07-10 03:53:35,143 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:53:35,932 - root - INFO - step: 65000 loss: 16.5130 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9760 +[titan] 2025-07-10 03:53:35,933 - root - INFO - lr: 1.0505e-04 gnorm: 0.89 [14:18:34< 7:42:18] +[titan] 2025-07-10 03:53:39,866 - root - INFO - step: 65005 loss: 16.5617 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0025 +[titan] 2025-07-10 03:53:39,866 - root - INFO - lr: 1.0503e-04 gnorm: 0.99 [14:18:38< 7:42:14] +[titan] 2025-07-10 03:53:43,794 - root - INFO - step: 65010 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8566 +[titan] 2025-07-10 03:53:43,794 - root - INFO - lr: 1.0501e-04 gnorm: 0.86 [14:18:42< 7:42:10] +[titan] 2025-07-10 03:53:47,725 - root - INFO - step: 65015 loss: 16.3953 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8836 +[titan] 2025-07-10 03:53:47,725 - root - INFO - lr: 1.0499e-04 gnorm: 0.86 [14:18:46< 7:42:06] +[titan] 2025-07-10 03:53:51,643 - root - INFO - step: 65020 loss: 16.6461 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.0753 +[titan] 2025-07-10 03:53:51,643 - root - INFO - lr: 1.0497e-04 gnorm: 0.88 [14:18:50< 7:42:02] +[titan] 2025-07-10 03:53:54,999 - root - INFO - Dumping profiler traces at step 65024 +[titan] 2025-07-10 03:53:55,031 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 03:53:55,826 - root - INFO - step: 65025 loss: 16.4776 memory: 44.58GiB(31.99%) tps: 78,348 tflops: 270.39 mfu: 27.34% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9528 +[titan] 2025-07-10 03:53:55,826 - root - INFO - lr: 1.0495e-04 gnorm: 0.91 [14:18:54< 7:41:58] +[titan] 2025-07-10 03:53:59,823 - root - INFO - step: 65030 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 81,989 tflops: 282.96 mfu: 28.61% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8963 +[titan] 2025-07-10 03:53:59,823 - root - INFO - lr: 1.0493e-04 gnorm: 0.89 [14:18:58< 7:41:54] +[titan] 2025-07-10 03:54:03,745 - root - INFO - step: 65035 loss: 16.2469 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.7557 +[titan] 2025-07-10 03:54:03,745 - root - INFO - lr: 1.0492e-04 gnorm: 0.93 [14:19:02< 7:41:50] +[titan] 2025-07-10 03:54:07,637 - root - INFO - step: 65040 loss: 16.6551 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0979 +[titan] 2025-07-10 03:54:07,637 - root - INFO - lr: 1.0490e-04 gnorm: 0.93 [14:19:06< 7:41:46] +[titan] 2025-07-10 03:54:11,539 - root - INFO - step: 65045 loss: 15.9832 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4240 global_avg_mtp_loss: 13.5592 +[titan] 2025-07-10 03:54:11,539 - root - INFO - lr: 1.0488e-04 gnorm: 0.93 [14:19:10< 7:41:42] +[titan] 2025-07-10 03:54:14,665 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:54:15,454 - root - INFO - step: 65050 loss: 16.4536 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9273 +[titan] 2025-07-10 03:54:15,454 - root - INFO - lr: 1.0486e-04 gnorm: 0.89 [14:19:14< 7:41:38] +[titan] 2025-07-10 03:54:19,385 - root - INFO - step: 65055 loss: 16.6661 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5722 global_avg_mtp_loss: 14.0939 +[titan] 2025-07-10 03:54:19,385 - root - INFO - lr: 1.0484e-04 gnorm: 0.86 [14:19:18< 7:41:34] +[titan] 2025-07-10 03:54:23,325 - root - INFO - step: 65060 loss: 16.2317 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7491 +[titan] 2025-07-10 03:54:23,325 - root - INFO - lr: 1.0482e-04 gnorm: 0.89 [14:19:21< 7:41:30] +[titan] 2025-07-10 03:54:27,243 - root - INFO - step: 65065 loss: 16.1877 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7121 +[titan] 2025-07-10 03:54:27,244 - root - INFO - lr: 1.0480e-04 gnorm: 0.85 [14:19:25< 7:41:27] +[titan] 2025-07-10 03:54:31,158 - root - INFO - step: 65070 loss: 16.4994 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5360 global_avg_mtp_loss: 13.9635 +[titan] 2025-07-10 03:54:31,159 - root - INFO - lr: 1.0478e-04 gnorm: 0.88 [14:19:29< 7:41:23] +[titan] 2025-07-10 03:54:35,082 - root - INFO - step: 65075 loss: 16.5078 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 13.9720 +[titan] 2025-07-10 03:54:35,083 - root - INFO - lr: 1.0476e-04 gnorm: 0.86 [14:19:33< 7:41:19] +[titan] 2025-07-10 03:54:39,000 - root - INFO - step: 65080 loss: 16.4491 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5376 global_avg_mtp_loss: 13.9115 +[titan] 2025-07-10 03:54:39,000 - root - INFO - lr: 1.0474e-04 gnorm: 0.90 [14:19:37< 7:41:15] +[titan] 2025-07-10 03:54:42,928 - root - INFO - step: 65085 loss: 16.1703 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7019 +[titan] 2025-07-10 03:54:42,928 - root - INFO - lr: 1.0472e-04 gnorm: 0.88 [14:19:41< 7:41:11] +[titan] 2025-07-10 03:54:46,842 - root - INFO - step: 65090 loss: 17.2669 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.7136 global_avg_mtp_loss: 14.5532 +[titan] 2025-07-10 03:54:46,842 - root - INFO - lr: 1.0470e-04 gnorm: 1.00 [14:19:45< 7:41:07] +[titan] 2025-07-10 03:54:50,786 - root - INFO - step: 65095 loss: 16.4146 memory: 44.58GiB(31.99%) tps: 83,082 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5184 global_avg_mtp_loss: 13.8961 +[titan] 2025-07-10 03:54:50,786 - root - INFO - lr: 1.0469e-04 gnorm: 1.13 [14:19:49< 7:41:03] +[titan] 2025-07-10 03:54:53,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:54:54,695 - root - INFO - step: 65100 loss: 16.2844 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.7787 +[titan] 2025-07-10 03:54:54,695 - root - INFO - lr: 1.0467e-04 gnorm: 0.85 [14:19:53< 7:40:59] +[titan] 2025-07-10 03:54:58,654 - root - INFO - step: 65105 loss: 16.3649 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8457 +[titan] 2025-07-10 03:54:58,654 - root - INFO - lr: 1.0465e-04 gnorm: 0.88 [14:19:57< 7:40:55] +[titan] 2025-07-10 03:55:02,562 - root - INFO - step: 65110 loss: 16.0771 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6198 +[titan] 2025-07-10 03:55:02,562 - root - INFO - lr: 1.0463e-04 gnorm: 0.86 [14:20:01< 7:40:51] +[titan] 2025-07-10 03:55:06,515 - root - INFO - step: 65115 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 03:55:06,516 - root - INFO - lr: 1.0461e-04 gnorm: 0.90 [14:20:05< 7:40:47] +[titan] 2025-07-10 03:55:10,458 - root - INFO - step: 65120 loss: 16.3705 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5057 global_avg_mtp_loss: 13.8648 +[titan] 2025-07-10 03:55:10,458 - root - INFO - lr: 1.0459e-04 gnorm: 0.88 [14:20:09< 7:40:43] +[titan] 2025-07-10 03:55:14,382 - root - INFO - step: 65125 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9275 +[titan] 2025-07-10 03:55:14,383 - root - INFO - lr: 1.0457e-04 gnorm: 0.91 [14:20:13< 7:40:39] +[titan] 2025-07-10 03:55:18,338 - root - INFO - step: 65130 loss: 16.4955 memory: 44.58GiB(31.99%) tps: 82,850 tflops: 285.93 mfu: 28.91% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9694 +[titan] 2025-07-10 03:55:18,338 - root - INFO - lr: 1.0455e-04 gnorm: 0.90 [14:20:16< 7:40:35] +[titan] 2025-07-10 03:55:22,254 - root - INFO - step: 65135 loss: 16.1848 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7143 +[titan] 2025-07-10 03:55:22,254 - root - INFO - lr: 1.0453e-04 gnorm: 0.94 [14:20:20< 7:40:31] +[titan] 2025-07-10 03:55:26,180 - root - INFO - step: 65140 loss: 16.5757 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5415 global_avg_mtp_loss: 14.0342 +[titan] 2025-07-10 03:55:26,180 - root - INFO - lr: 1.0451e-04 gnorm: 0.91 [14:20:24< 7:40:27] +[titan] 2025-07-10 03:55:30,091 - root - INFO - step: 65145 loss: 16.2304 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7500 +[titan] 2025-07-10 03:55:30,092 - root - INFO - lr: 1.0449e-04 gnorm: 0.91 [14:20:28< 7:40:23] +[titan] 2025-07-10 03:55:33,229 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:55:34,033 - root - INFO - step: 65150 loss: 16.4170 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8976 +[titan] 2025-07-10 03:55:34,033 - root - INFO - lr: 1.0447e-04 gnorm: 0.88 [14:20:32< 7:40:19] +[titan] 2025-07-10 03:55:37,940 - root - INFO - step: 65155 loss: 16.3819 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8721 +[titan] 2025-07-10 03:55:37,940 - root - INFO - lr: 1.0446e-04 gnorm: 0.87 [14:20:36< 7:40:15] +[titan] 2025-07-10 03:55:41,835 - root - INFO - step: 65160 loss: 16.2191 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.7293 +[titan] 2025-07-10 03:55:41,836 - root - INFO - lr: 1.0444e-04 gnorm: 0.88 [14:20:40< 7:40:11] +[titan] 2025-07-10 03:55:45,771 - root - INFO - step: 65165 loss: 16.2947 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.7913 +[titan] 2025-07-10 03:55:45,772 - root - INFO - lr: 1.0442e-04 gnorm: 0.86 [14:20:44< 7:40:07] +[titan] 2025-07-10 03:55:49,685 - root - INFO - step: 65170 loss: 16.3821 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.8539 +[titan] 2025-07-10 03:55:49,686 - root - INFO - lr: 1.0440e-04 gnorm: 0.88 [14:20:48< 7:40:03] +[titan] 2025-07-10 03:55:53,584 - root - INFO - step: 65175 loss: 16.3179 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 03:55:53,584 - root - INFO - lr: 1.0438e-04 gnorm: 0.88 [14:20:52< 7:39:59] +[titan] 2025-07-10 03:55:57,537 - root - INFO - step: 65180 loss: 16.2221 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7433 +[titan] 2025-07-10 03:55:57,537 - root - INFO - lr: 1.0436e-04 gnorm: 0.89 [14:20:56< 7:39:55] +[titan] 2025-07-10 03:56:01,486 - root - INFO - step: 65185 loss: 16.3090 memory: 44.58GiB(31.99%) tps: 82,982 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.8191 +[titan] 2025-07-10 03:56:01,486 - root - INFO - lr: 1.0434e-04 gnorm: 0.94 [14:21:00< 7:39:51] +[titan] 2025-07-10 03:56:05,400 - root - INFO - step: 65190 loss: 16.4492 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.9269 +[titan] 2025-07-10 03:56:05,400 - root - INFO - lr: 1.0432e-04 gnorm: 0.82 [14:21:04< 7:39:47] +[titan] 2025-07-10 03:56:09,314 - root - INFO - step: 65195 loss: 16.1367 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.6785 +[titan] 2025-07-10 03:56:09,314 - root - INFO - lr: 1.0430e-04 gnorm: 0.93 [14:21:07< 7:39:43] +[titan] 2025-07-10 03:56:12,469 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:56:13,260 - root - INFO - step: 65200 loss: 16.6177 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5639 global_avg_mtp_loss: 14.0538 +[titan] 2025-07-10 03:56:13,260 - root - INFO - lr: 1.0428e-04 gnorm: 0.91 [14:21:11< 7:39:39] +[titan] 2025-07-10 03:56:17,158 - root - INFO - step: 65205 loss: 16.3938 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8738 +[titan] 2025-07-10 03:56:17,158 - root - INFO - lr: 1.0426e-04 gnorm: 0.87 [14:21:15< 7:39:35] +[titan] 2025-07-10 03:56:21,061 - root - INFO - step: 65210 loss: 16.3349 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4905 global_avg_mtp_loss: 13.8443 +[titan] 2025-07-10 03:56:21,061 - root - INFO - lr: 1.0424e-04 gnorm: 0.94 [14:21:19< 7:39:31] +[titan] 2025-07-10 03:56:24,966 - root - INFO - step: 65215 loss: 16.3767 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8687 +[titan] 2025-07-10 03:56:24,966 - root - INFO - lr: 1.0423e-04 gnorm: 0.89 [14:21:23< 7:39:27] +[titan] 2025-07-10 03:56:28,877 - root - INFO - step: 65220 loss: 16.6021 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5521 global_avg_mtp_loss: 14.0501 +[titan] 2025-07-10 03:56:28,877 - root - INFO - lr: 1.0421e-04 gnorm: 0.91 [14:21:27< 7:39:23] +[titan] 2025-07-10 03:56:32,796 - root - INFO - step: 65225 loss: 16.7771 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5841 global_avg_mtp_loss: 14.1930 +[titan] 2025-07-10 03:56:32,796 - root - INFO - lr: 1.0419e-04 gnorm: 0.90 [14:21:31< 7:39:19] +[titan] 2025-07-10 03:56:36,716 - root - INFO - step: 65230 loss: 16.4259 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.9147 +[titan] 2025-07-10 03:56:36,716 - root - INFO - lr: 1.0417e-04 gnorm: 0.89 [14:21:35< 7:39:15] +[titan] 2025-07-10 03:56:40,622 - root - INFO - step: 65235 loss: 16.4652 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9420 +[titan] 2025-07-10 03:56:40,622 - root - INFO - lr: 1.0415e-04 gnorm: 0.92 [14:21:39< 7:39:11] +[titan] 2025-07-10 03:56:44,532 - root - INFO - step: 65240 loss: 16.1155 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6480 +[titan] 2025-07-10 03:56:44,533 - root - INFO - lr: 1.0413e-04 gnorm: 0.91 [14:21:43< 7:39:07] +[titan] 2025-07-10 03:56:48,434 - root - INFO - step: 65245 loss: 16.5064 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-10 03:56:48,434 - root - INFO - lr: 1.0411e-04 gnorm: 0.92 [14:21:47< 7:39:03] +[titan] 2025-07-10 03:56:51,554 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:56:52,349 - root - INFO - step: 65250 loss: 16.3079 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8058 +[titan] 2025-07-10 03:56:52,350 - root - INFO - lr: 1.0409e-04 gnorm: 0.91 [14:21:50< 7:38:59] +[titan] 2025-07-10 03:56:56,266 - root - INFO - step: 65255 loss: 16.4826 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.9556 +[titan] 2025-07-10 03:56:56,267 - root - INFO - lr: 1.0407e-04 gnorm: 0.87 [14:21:54< 7:38:55] +[titan] 2025-07-10 03:57:00,215 - root - INFO - step: 65260 loss: 16.3662 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8508 +[titan] 2025-07-10 03:57:00,215 - root - INFO - lr: 1.0405e-04 gnorm: 0.90 [14:21:58< 7:38:51] +[titan] 2025-07-10 03:57:04,164 - root - INFO - step: 65265 loss: 16.2977 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.7998 +[titan] 2025-07-10 03:57:04,164 - root - INFO - lr: 1.0403e-04 gnorm: 0.86 [14:22:02< 7:38:47] +[titan] 2025-07-10 03:57:08,132 - root - INFO - step: 65270 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 82,586 tflops: 285.02 mfu: 28.82% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7163 +[titan] 2025-07-10 03:57:08,132 - root - INFO - lr: 1.0402e-04 gnorm: 0.87 [14:22:06< 7:38:43] +[titan] 2025-07-10 03:57:12,107 - root - INFO - step: 65275 loss: 16.2152 memory: 44.58GiB(31.99%) tps: 82,449 tflops: 284.54 mfu: 28.77% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7311 +[titan] 2025-07-10 03:57:12,107 - root - INFO - lr: 1.0400e-04 gnorm: 0.89 [14:22:10< 7:38:39] +[titan] 2025-07-10 03:57:16,053 - root - INFO - step: 65280 loss: 16.5080 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9788 +[titan] 2025-07-10 03:57:16,053 - root - INFO - lr: 1.0398e-04 gnorm: 0.90 [14:22:14< 7:38:35] +[titan] 2025-07-10 03:57:19,998 - root - INFO - step: 65285 loss: 16.5057 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 13.9741 +[titan] 2025-07-10 03:57:19,998 - root - INFO - lr: 1.0396e-04 gnorm: 0.91 [14:22:18< 7:38:31] +[titan] 2025-07-10 03:57:24,006 - root - INFO - step: 65290 loss: 16.4386 memory: 44.58GiB(31.99%) tps: 81,767 tflops: 282.19 mfu: 28.53% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9185 +[titan] 2025-07-10 03:57:24,006 - root - INFO - lr: 1.0394e-04 gnorm: 0.89 [14:22:22< 7:38:27] +[titan] 2025-07-10 03:57:27,951 - root - INFO - step: 65295 loss: 16.3643 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8632 +[titan] 2025-07-10 03:57:27,951 - root - INFO - lr: 1.0392e-04 gnorm: 0.83 [14:22:26< 7:38:23] +[titan] 2025-07-10 03:57:31,080 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:57:31,886 - root - INFO - step: 65300 loss: 16.4681 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-10 03:57:31,887 - root - INFO - lr: 1.0390e-04 gnorm: 0.83 [14:22:30< 7:38:19] +[titan] 2025-07-10 03:57:35,805 - root - INFO - step: 65305 loss: 16.2140 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.7399 +[titan] 2025-07-10 03:57:35,806 - root - INFO - lr: 1.0388e-04 gnorm: 0.91 [14:22:34< 7:38:15] +[titan] 2025-07-10 03:57:39,714 - root - INFO - step: 65310 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5287 global_avg_mtp_loss: 13.9488 +[titan] 2025-07-10 03:57:39,714 - root - INFO - lr: 1.0386e-04 gnorm: 0.91 [14:22:38< 7:38:11] +[titan] 2025-07-10 03:57:43,636 - root - INFO - step: 65315 loss: 16.4224 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.9004 +[titan] 2025-07-10 03:57:43,636 - root - INFO - lr: 1.0384e-04 gnorm: 0.87 [14:22:42< 7:38:07] +[titan] 2025-07-10 03:57:47,581 - root - INFO - step: 65320 loss: 16.2942 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.7982 +[titan] 2025-07-10 03:57:47,581 - root - INFO - lr: 1.0382e-04 gnorm: 0.94 [14:22:46< 7:38:03] +[titan] 2025-07-10 03:57:51,504 - root - INFO - step: 65325 loss: 16.6017 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5560 global_avg_mtp_loss: 14.0457 +[titan] 2025-07-10 03:57:51,504 - root - INFO - lr: 1.0381e-04 gnorm: 0.91 [14:22:50< 7:37:59] +[titan] 2025-07-10 03:57:55,419 - root - INFO - step: 65330 loss: 16.5250 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9857 +[titan] 2025-07-10 03:57:55,420 - root - INFO - lr: 1.0379e-04 gnorm: 0.92 [14:22:54< 7:37:55] +[titan] 2025-07-10 03:57:59,371 - root - INFO - step: 65335 loss: 16.4323 memory: 44.58GiB(31.99%) tps: 82,938 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.5286 global_avg_mtp_loss: 13.9036 +[titan] 2025-07-10 03:57:59,371 - root - INFO - lr: 1.0377e-04 gnorm: 0.97 [14:22:58< 7:37:52] +[titan] 2025-07-10 03:58:03,319 - root - INFO - step: 65340 loss: 16.5522 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 14.0192 +[titan] 2025-07-10 03:58:03,319 - root - INFO - lr: 1.0375e-04 gnorm: 0.94 [14:23:01< 7:37:48] +[titan] 2025-07-10 03:58:07,211 - root - INFO - step: 65345 loss: 16.5484 memory: 44.58GiB(31.99%) tps: 84,198 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 14.0102 +[titan] 2025-07-10 03:58:07,211 - root - INFO - lr: 1.0373e-04 gnorm: 0.96 [14:23:05< 7:37:44] +[titan] 2025-07-10 03:58:10,337 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:58:11,124 - root - INFO - step: 65350 loss: 16.4692 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5344 global_avg_mtp_loss: 13.9348 +[titan] 2025-07-10 03:58:11,124 - root - INFO - lr: 1.0371e-04 gnorm: 0.86 [14:23:09< 7:37:40] +[titan] 2025-07-10 03:58:15,036 - root - INFO - step: 65355 loss: 16.1642 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 03:58:15,036 - root - INFO - lr: 1.0369e-04 gnorm: 0.94 [14:23:13< 7:37:36] +[titan] 2025-07-10 03:58:18,986 - root - INFO - step: 65360 loss: 16.1538 memory: 44.58GiB(31.99%) tps: 82,962 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.6894 +[titan] 2025-07-10 03:58:18,986 - root - INFO - lr: 1.0367e-04 gnorm: 0.93 [14:23:17< 7:37:32] +[titan] 2025-07-10 03:58:22,890 - root - INFO - step: 65365 loss: 16.2966 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8028 +[titan] 2025-07-10 03:58:22,890 - root - INFO - lr: 1.0365e-04 gnorm: 0.90 [14:23:21< 7:37:28] +[titan] 2025-07-10 03:58:26,804 - root - INFO - step: 65370 loss: 16.0546 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4607 global_avg_mtp_loss: 13.5939 +[titan] 2025-07-10 03:58:26,804 - root - INFO - lr: 1.0363e-04 gnorm: 0.89 [14:23:25< 7:37:24] +[titan] 2025-07-10 03:58:30,723 - root - INFO - step: 65375 loss: 16.1990 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7176 +[titan] 2025-07-10 03:58:30,723 - root - INFO - lr: 1.0361e-04 gnorm: 0.84 [14:23:29< 7:37:20] +[titan] 2025-07-10 03:58:34,653 - root - INFO - step: 65380 loss: 16.2523 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7655 +[titan] 2025-07-10 03:58:34,653 - root - INFO - lr: 1.0360e-04 gnorm: 0.84 [14:23:33< 7:37:16] +[titan] 2025-07-10 03:58:38,602 - root - INFO - step: 65385 loss: 16.4201 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.9020 +[titan] 2025-07-10 03:58:38,603 - root - INFO - lr: 1.0358e-04 gnorm: 0.88 [14:23:37< 7:37:12] +[titan] 2025-07-10 03:58:42,555 - root - INFO - step: 65390 loss: 16.2497 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7611 +[titan] 2025-07-10 03:58:42,555 - root - INFO - lr: 1.0356e-04 gnorm: 0.88 [14:23:41< 7:37:08] +[titan] 2025-07-10 03:58:46,476 - root - INFO - step: 65395 loss: 16.8333 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.6006 global_avg_mtp_loss: 14.2327 +[titan] 2025-07-10 03:58:46,476 - root - INFO - lr: 1.0354e-04 gnorm: 0.89 [14:23:45< 7:37:04] +[titan] 2025-07-10 03:58:49,613 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:58:50,409 - root - INFO - step: 65400 loss: 16.1938 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7203 +[titan] 2025-07-10 03:58:50,410 - root - INFO - lr: 1.0352e-04 gnorm: 0.90 [14:23:49< 7:37:00] +[titan] 2025-07-10 03:58:54,322 - root - INFO - step: 65405 loss: 16.0383 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.5722 +[titan] 2025-07-10 03:58:54,322 - root - INFO - lr: 1.0350e-04 gnorm: 0.96 [14:23:52< 7:36:56] +[titan] 2025-07-10 03:58:58,259 - root - INFO - step: 65410 loss: 16.0393 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.5846 +[titan] 2025-07-10 03:58:58,259 - root - INFO - lr: 1.0348e-04 gnorm: 0.85 [14:23:56< 7:36:52] +[titan] 2025-07-10 03:59:02,159 - root - INFO - step: 65415 loss: 16.2697 memory: 44.58GiB(31.99%) tps: 84,026 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.7753 +[titan] 2025-07-10 03:59:02,159 - root - INFO - lr: 1.0346e-04 gnorm: 0.85 [14:24:00< 7:36:48] +[titan] 2025-07-10 03:59:06,062 - root - INFO - step: 65420 loss: 16.4964 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.9654 +[titan] 2025-07-10 03:59:06,062 - root - INFO - lr: 1.0344e-04 gnorm: 0.94 [14:24:04< 7:36:44] +[titan] 2025-07-10 03:59:09,994 - root - INFO - step: 65425 loss: 16.2867 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.7885 +[titan] 2025-07-10 03:59:09,995 - root - INFO - lr: 1.0342e-04 gnorm: 0.93 [14:24:08< 7:36:40] +[titan] 2025-07-10 03:59:13,893 - root - INFO - step: 65430 loss: 16.3663 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8463 +[titan] 2025-07-10 03:59:13,893 - root - INFO - lr: 1.0340e-04 gnorm: 0.92 [14:24:12< 7:36:36] +[titan] 2025-07-10 03:59:17,829 - root - INFO - step: 65435 loss: 16.5153 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.9918 +[titan] 2025-07-10 03:59:17,829 - root - INFO - lr: 1.0339e-04 gnorm: 0.92 [14:24:16< 7:36:32] +[titan] 2025-07-10 03:59:21,738 - root - INFO - step: 65440 loss: 16.3141 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8188 +[titan] 2025-07-10 03:59:21,738 - root - INFO - lr: 1.0337e-04 gnorm: 0.96 [14:24:20< 7:36:28] +[titan] 2025-07-10 03:59:25,642 - root - INFO - step: 65445 loss: 16.1103 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.6536 +[titan] 2025-07-10 03:59:25,642 - root - INFO - lr: 1.0335e-04 gnorm: 0.98 [14:24:24< 7:36:24] +[titan] 2025-07-10 03:59:28,764 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 03:59:29,555 - root - INFO - step: 65450 loss: 16.3147 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8070 +[titan] 2025-07-10 03:59:29,555 - root - INFO - lr: 1.0333e-04 gnorm: 1.01 [14:24:28< 7:36:20] +[titan] 2025-07-10 03:59:33,457 - root - INFO - step: 65455 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 03:59:33,457 - root - INFO - lr: 1.0331e-04 gnorm: 0.91 [14:24:32< 7:36:16] +[titan] 2025-07-10 03:59:37,373 - root - INFO - step: 65460 loss: 16.3547 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8423 +[titan] 2025-07-10 03:59:37,373 - root - INFO - lr: 1.0329e-04 gnorm: 0.98 [14:24:35< 7:36:12] +[titan] 2025-07-10 03:59:41,281 - root - INFO - step: 65465 loss: 16.2762 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5159 global_avg_mtp_loss: 13.7602 +[titan] 2025-07-10 03:59:41,281 - root - INFO - lr: 1.0327e-04 gnorm: 0.91 [14:24:39< 7:36:08] +[titan] 2025-07-10 03:59:45,179 - root - INFO - step: 65470 loss: 16.2922 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.34% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.7904 +[titan] 2025-07-10 03:59:45,179 - root - INFO - lr: 1.0325e-04 gnorm: 0.86 [14:24:43< 7:36:04] +[titan] 2025-07-10 03:59:49,134 - root - INFO - step: 65475 loss: 16.5594 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5496 global_avg_mtp_loss: 14.0099 +[titan] 2025-07-10 03:59:49,134 - root - INFO - lr: 1.0323e-04 gnorm: 0.90 [14:24:47< 7:36:00] +[titan] 2025-07-10 03:59:53,054 - root - INFO - step: 65480 loss: 16.1296 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6673 +[titan] 2025-07-10 03:59:53,054 - root - INFO - lr: 1.0321e-04 gnorm: 0.89 [14:24:51< 7:35:56] +[titan] 2025-07-10 03:59:56,966 - root - INFO - step: 65485 loss: 16.3940 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.8752 +[titan] 2025-07-10 03:59:56,966 - root - INFO - lr: 1.0320e-04 gnorm: 0.94 [14:24:55< 7:35:52] +[titan] 2025-07-10 04:00:00,863 - root - INFO - step: 65490 loss: 16.3935 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.35% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 04:00:00,863 - root - INFO - lr: 1.0318e-04 gnorm: 0.88 [14:24:59< 7:35:48] +[titan] 2025-07-10 04:00:04,780 - root - INFO - step: 65495 loss: 16.4804 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9466 +[titan] 2025-07-10 04:00:04,781 - root - INFO - lr: 1.0316e-04 gnorm: 0.92 [14:25:03< 7:35:44] +[titan] 2025-07-10 04:00:07,897 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:00:08,691 - root - INFO - step: 65500 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.7903 +[titan] 2025-07-10 04:00:08,691 - root - INFO - lr: 1.0314e-04 gnorm: 0.95 [14:25:07< 7:35:40] +[titan] 2025-07-10 04:00:12,596 - root - INFO - step: 65505 loss: 16.4964 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5345 global_avg_mtp_loss: 13.9619 +[titan] 2025-07-10 04:00:12,597 - root - INFO - lr: 1.0312e-04 gnorm: 0.92 [14:25:11< 7:35:36] +[titan] 2025-07-10 04:00:16,507 - root - INFO - step: 65510 loss: 16.3219 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8201 +[titan] 2025-07-10 04:00:16,507 - root - INFO - lr: 1.0310e-04 gnorm: 0.95 [14:25:15< 7:35:32] +[titan] 2025-07-10 04:00:20,400 - root - INFO - step: 65515 loss: 16.2978 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 04:00:20,400 - root - INFO - lr: 1.0308e-04 gnorm: 0.88 [14:25:19< 7:35:28] +[titan] 2025-07-10 04:00:24,293 - root - INFO - step: 65520 loss: 16.5395 memory: 44.58GiB(31.99%) tps: 84,185 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9990 +[titan] 2025-07-10 04:00:24,293 - root - INFO - lr: 1.0306e-04 gnorm: 0.87 [14:25:22< 7:35:24] +[titan] 2025-07-10 04:00:28,232 - root - INFO - step: 65525 loss: 16.2381 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.7468 +[titan] 2025-07-10 04:00:28,232 - root - INFO - lr: 1.0304e-04 gnorm: 0.91 [14:25:26< 7:35:20] +[titan] 2025-07-10 04:00:32,124 - root - INFO - step: 65530 loss: 16.3886 memory: 44.58GiB(31.99%) tps: 84,201 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8768 +[titan] 2025-07-10 04:00:32,124 - root - INFO - lr: 1.0302e-04 gnorm: 0.88 [14:25:30< 7:35:16] +[titan] 2025-07-10 04:00:36,118 - root - INFO - step: 65535 loss: 16.4997 memory: 44.58GiB(31.99%) tps: 82,047 tflops: 283.16 mfu: 28.63% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 13.9599 +[titan] 2025-07-10 04:00:36,119 - root - INFO - lr: 1.0300e-04 gnorm: 0.87 [14:25:34< 7:35:12] +[titan] 2025-07-10 04:00:37,061 - root - INFO - Dumping profiler traces at step 65536 +[titan] 2025-07-10 04:00:37,092 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:00:40,237 - root - INFO - step: 65540 loss: 16.4420 memory: 44.58GiB(31.99%) tps: 79,569 tflops: 274.60 mfu: 27.77% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.9195 +[titan] 2025-07-10 04:00:40,237 - root - INFO - lr: 1.0299e-04 gnorm: 0.85 [14:25:38< 7:35:08] +[titan] 2025-07-10 04:00:44,150 - root - INFO - step: 65545 loss: 16.0522 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4458 global_avg_mtp_loss: 13.6064 +[titan] 2025-07-10 04:00:44,150 - root - INFO - lr: 1.0297e-04 gnorm: 0.87 [14:25:42< 7:35:04] +[titan] 2025-07-10 04:00:47,293 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:00:48,097 - root - INFO - step: 65550 loss: 16.4356 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.9195 +[titan] 2025-07-10 04:00:48,097 - root - INFO - lr: 1.0295e-04 gnorm: 0.92 [14:25:46< 7:35:00] +[titan] 2025-07-10 04:00:52,041 - root - INFO - step: 65555 loss: 16.6176 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5546 global_avg_mtp_loss: 14.0630 +[titan] 2025-07-10 04:00:52,041 - root - INFO - lr: 1.0293e-04 gnorm: 0.89 [14:25:50< 7:34:56] +[titan] 2025-07-10 04:00:55,953 - root - INFO - step: 65560 loss: 16.3388 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-10 04:00:55,953 - root - INFO - lr: 1.0291e-04 gnorm: 0.92 [14:25:54< 7:34:52] +[titan] 2025-07-10 04:00:59,901 - root - INFO - step: 65565 loss: 16.0982 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6391 +[titan] 2025-07-10 04:00:59,901 - root - INFO - lr: 1.0289e-04 gnorm: 0.92 [14:25:58< 7:34:48] +[titan] 2025-07-10 04:01:03,832 - root - INFO - step: 65570 loss: 16.1881 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7032 +[titan] 2025-07-10 04:01:03,832 - root - INFO - lr: 1.0287e-04 gnorm: 1.00 [14:26:02< 7:34:44] +[titan] 2025-07-10 04:01:07,779 - root - INFO - step: 65575 loss: 16.5117 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5355 global_avg_mtp_loss: 13.9762 +[titan] 2025-07-10 04:01:07,780 - root - INFO - lr: 1.0285e-04 gnorm: 0.87 [14:26:06< 7:34:40] +[titan] 2025-07-10 04:01:11,690 - root - INFO - step: 65580 loss: 16.5477 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 14.0063 +[titan] 2025-07-10 04:01:11,690 - root - INFO - lr: 1.0283e-04 gnorm: 0.85 [14:26:10< 7:34:36] +[titan] 2025-07-10 04:01:15,682 - root - INFO - step: 65585 loss: 16.3496 memory: 44.58GiB(31.99%) tps: 82,093 tflops: 283.32 mfu: 28.65% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8412 +[titan] 2025-07-10 04:01:15,682 - root - INFO - lr: 1.0281e-04 gnorm: 0.92 [14:26:14< 7:34:32] +[titan] 2025-07-10 04:01:19,614 - root - INFO - step: 65590 loss: 16.3213 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8172 +[titan] 2025-07-10 04:01:19,614 - root - INFO - lr: 1.0280e-04 gnorm: 0.86 [14:26:18< 7:34:28] +[titan] 2025-07-10 04:01:23,532 - root - INFO - step: 65595 loss: 16.4043 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 04:01:23,532 - root - INFO - lr: 1.0278e-04 gnorm: 0.92 [14:26:22< 7:34:24] +[titan] 2025-07-10 04:01:26,653 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:01:27,447 - root - INFO - step: 65600 loss: 16.2495 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.7528 +[titan] 2025-07-10 04:01:27,447 - root - INFO - lr: 1.0276e-04 gnorm: 0.91 [14:26:26< 7:34:20] +[titan] 2025-07-10 04:01:31,372 - root - INFO - step: 65605 loss: 16.3544 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8403 +[titan] 2025-07-10 04:01:31,372 - root - INFO - lr: 1.0274e-04 gnorm: 0.89 [14:26:29< 7:34:16] +[titan] 2025-07-10 04:01:35,292 - root - INFO - step: 65610 loss: 16.4590 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9307 +[titan] 2025-07-10 04:01:35,292 - root - INFO - lr: 1.0272e-04 gnorm: 0.87 [14:26:33< 7:34:13] +[titan] 2025-07-10 04:01:39,210 - root - INFO - step: 65615 loss: 16.5638 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 14.0268 +[titan] 2025-07-10 04:01:39,210 - root - INFO - lr: 1.0270e-04 gnorm: 0.88 [14:26:37< 7:34:09] +[titan] 2025-07-10 04:01:43,126 - root - INFO - step: 65620 loss: 16.4857 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 13.9361 +[titan] 2025-07-10 04:01:43,126 - root - INFO - lr: 1.0268e-04 gnorm: 0.91 [14:26:41< 7:34:05] +[titan] 2025-07-10 04:01:47,072 - root - INFO - step: 65625 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9745 +[titan] 2025-07-10 04:01:47,072 - root - INFO - lr: 1.0266e-04 gnorm: 0.91 [14:26:45< 7:34:01] +[titan] 2025-07-10 04:01:51,019 - root - INFO - step: 65630 loss: 16.1045 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6360 +[titan] 2025-07-10 04:01:51,019 - root - INFO - lr: 1.0264e-04 gnorm: 0.87 [14:26:49< 7:33:57] +[titan] 2025-07-10 04:01:54,945 - root - INFO - step: 65635 loss: 16.3469 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8322 +[titan] 2025-07-10 04:01:54,946 - root - INFO - lr: 1.0262e-04 gnorm: 0.86 [14:26:53< 7:33:53] +[titan] 2025-07-10 04:01:58,880 - root - INFO - step: 65640 loss: 16.1714 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7027 +[titan] 2025-07-10 04:01:58,880 - root - INFO - lr: 1.0261e-04 gnorm: 0.93 [14:26:57< 7:33:49] +[titan] 2025-07-10 04:02:02,785 - root - INFO - step: 65645 loss: 16.5435 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 14.0083 +[titan] 2025-07-10 04:02:02,785 - root - INFO - lr: 1.0259e-04 gnorm: 0.90 [14:27:01< 7:33:45] +[titan] 2025-07-10 04:02:05,893 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:02:06,693 - root - INFO - step: 65650 loss: 16.2380 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7506 +[titan] 2025-07-10 04:02:06,693 - root - INFO - lr: 1.0257e-04 gnorm: 0.88 [14:27:05< 7:33:41] +[titan] 2025-07-10 04:02:10,599 - root - INFO - step: 65655 loss: 16.3571 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8476 +[titan] 2025-07-10 04:02:10,599 - root - INFO - lr: 1.0255e-04 gnorm: 0.89 [14:27:09< 7:33:37] +[titan] 2025-07-10 04:02:14,546 - root - INFO - step: 65660 loss: 16.1737 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.6922 +[titan] 2025-07-10 04:02:14,547 - root - INFO - lr: 1.0253e-04 gnorm: 0.88 [14:27:13< 7:33:33] +[titan] 2025-07-10 04:02:18,461 - root - INFO - step: 65665 loss: 16.6227 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0682 +[titan] 2025-07-10 04:02:18,461 - root - INFO - lr: 1.0251e-04 gnorm: 0.87 [14:27:17< 7:33:29] +[titan] 2025-07-10 04:02:22,407 - root - INFO - step: 65670 loss: 16.1997 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7191 +[titan] 2025-07-10 04:02:22,408 - root - INFO - lr: 1.0249e-04 gnorm: 0.97 [14:27:21< 7:33:25] +[titan] 2025-07-10 04:02:26,359 - root - INFO - step: 65675 loss: 16.3875 memory: 44.58GiB(31.99%) tps: 82,920 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8664 +[titan] 2025-07-10 04:02:26,360 - root - INFO - lr: 1.0247e-04 gnorm: 0.89 [14:27:24< 7:33:21] +[titan] 2025-07-10 04:02:30,251 - root - INFO - step: 65680 loss: 16.5961 memory: 44.58GiB(31.99%) tps: 84,220 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 14.0597 +[titan] 2025-07-10 04:02:30,251 - root - INFO - lr: 1.0245e-04 gnorm: 0.84 [14:27:28< 7:33:17] +[titan] 2025-07-10 04:02:34,164 - root - INFO - step: 65685 loss: 16.2176 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7247 +[titan] 2025-07-10 04:02:34,164 - root - INFO - lr: 1.0243e-04 gnorm: 0.92 [14:27:32< 7:33:13] +[titan] 2025-07-10 04:02:38,086 - root - INFO - step: 65690 loss: 16.4166 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.9093 +[titan] 2025-07-10 04:02:38,086 - root - INFO - lr: 1.0242e-04 gnorm: 0.85 [14:27:36< 7:33:09] +[titan] 2025-07-10 04:02:42,031 - root - INFO - step: 65695 loss: 16.4205 memory: 44.58GiB(31.99%) tps: 83,069 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5293 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 04:02:42,031 - root - INFO - lr: 1.0240e-04 gnorm: 0.88 [14:27:40< 7:33:05] +[titan] 2025-07-10 04:02:45,204 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:02:45,997 - root - INFO - step: 65700 loss: 16.2343 memory: 44.58GiB(31.99%) tps: 82,633 tflops: 285.18 mfu: 28.84% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.7379 +[titan] 2025-07-10 04:02:45,997 - root - INFO - lr: 1.0238e-04 gnorm: 0.88 [14:27:44< 7:33:01] +[titan] 2025-07-10 04:02:49,924 - root - INFO - step: 65705 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.8117 +[titan] 2025-07-10 04:02:49,924 - root - INFO - lr: 1.0236e-04 gnorm: 0.89 [14:27:48< 7:32:57] +[titan] 2025-07-10 04:02:53,845 - root - INFO - step: 65710 loss: 16.1122 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6532 +[titan] 2025-07-10 04:02:53,845 - root - INFO - lr: 1.0234e-04 gnorm: 0.91 [14:27:52< 7:32:53] +[titan] 2025-07-10 04:02:57,761 - root - INFO - step: 65715 loss: 16.4839 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-10 04:02:57,761 - root - INFO - lr: 1.0232e-04 gnorm: 0.85 [14:27:56< 7:32:49] +[titan] 2025-07-10 04:03:01,727 - root - INFO - step: 65720 loss: 16.3897 memory: 44.58GiB(31.99%) tps: 82,629 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8830 +[titan] 2025-07-10 04:03:01,727 - root - INFO - lr: 1.0230e-04 gnorm: 0.89 [14:28:00< 7:32:45] +[titan] 2025-07-10 04:03:05,656 - root - INFO - step: 65725 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.9168 +[titan] 2025-07-10 04:03:05,656 - root - INFO - lr: 1.0228e-04 gnorm: 0.98 [14:28:04< 7:32:41] +[titan] 2025-07-10 04:03:09,641 - root - INFO - step: 65730 loss: 16.4347 memory: 44.58GiB(31.99%) tps: 82,234 tflops: 283.80 mfu: 28.70% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9106 +[titan] 2025-07-10 04:03:09,641 - root - INFO - lr: 1.0226e-04 gnorm: 0.86 [14:28:08< 7:32:37] +[titan] 2025-07-10 04:03:13,572 - root - INFO - step: 65735 loss: 16.2501 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7782 +[titan] 2025-07-10 04:03:13,573 - root - INFO - lr: 1.0225e-04 gnorm: 0.87 [14:28:12< 7:32:33] +[titan] 2025-07-10 04:03:17,506 - root - INFO - step: 65740 loss: 16.7013 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5855 global_avg_mtp_loss: 14.1159 +[titan] 2025-07-10 04:03:17,506 - root - INFO - lr: 1.0223e-04 gnorm: 0.85 [14:28:16< 7:32:29] +[titan] 2025-07-10 04:03:21,451 - root - INFO - step: 65745 loss: 16.5430 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 14.0034 +[titan] 2025-07-10 04:03:21,451 - root - INFO - lr: 1.0221e-04 gnorm: 0.87 [14:28:20< 7:32:25] +[titan] 2025-07-10 04:03:24,577 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:03:25,371 - root - INFO - step: 65750 loss: 16.3845 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8756 +[titan] 2025-07-10 04:03:25,371 - root - INFO - lr: 1.0219e-04 gnorm: 0.86 [14:28:23< 7:32:21] +[titan] 2025-07-10 04:03:29,322 - root - INFO - step: 65755 loss: 16.3616 memory: 44.58GiB(31.99%) tps: 82,953 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8508 +[titan] 2025-07-10 04:03:29,322 - root - INFO - lr: 1.0217e-04 gnorm: 0.89 [14:28:27< 7:32:17] +[titan] 2025-07-10 04:03:33,257 - root - INFO - step: 65760 loss: 16.2860 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.7886 +[titan] 2025-07-10 04:03:33,257 - root - INFO - lr: 1.0215e-04 gnorm: 0.95 [14:28:31< 7:32:13] +[titan] 2025-07-10 04:03:37,156 - root - INFO - step: 65765 loss: 16.7432 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1592 +[titan] 2025-07-10 04:03:37,156 - root - INFO - lr: 1.0213e-04 gnorm: 0.92 [14:28:35< 7:32:09] +[titan] 2025-07-10 04:03:41,074 - root - INFO - step: 65770 loss: 16.2859 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.7938 +[titan] 2025-07-10 04:03:41,074 - root - INFO - lr: 1.0211e-04 gnorm: 0.84 [14:28:39< 7:32:05] +[titan] 2025-07-10 04:03:45,011 - root - INFO - step: 65775 loss: 16.3910 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8857 +[titan] 2025-07-10 04:03:45,011 - root - INFO - lr: 1.0209e-04 gnorm: 0.86 [14:28:43< 7:32:01] +[titan] 2025-07-10 04:03:48,929 - root - INFO - step: 65780 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9473 +[titan] 2025-07-10 04:03:48,930 - root - INFO - lr: 1.0207e-04 gnorm: 0.86 [14:28:47< 7:31:57] +[titan] 2025-07-10 04:03:52,859 - root - INFO - step: 65785 loss: 16.3552 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8471 +[titan] 2025-07-10 04:03:52,859 - root - INFO - lr: 1.0206e-04 gnorm: 0.87 [14:28:51< 7:31:53] +[titan] 2025-07-10 04:03:56,803 - root - INFO - step: 65790 loss: 16.3406 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8421 +[titan] 2025-07-10 04:03:56,803 - root - INFO - lr: 1.0204e-04 gnorm: 0.85 [14:28:55< 7:31:49] +[titan] 2025-07-10 04:04:00,745 - root - INFO - step: 65795 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,148 tflops: 286.96 mfu: 29.01% global_avg_ntp_loss: 2.5348 global_avg_mtp_loss: 13.9483 +[titan] 2025-07-10 04:04:00,745 - root - INFO - lr: 1.0202e-04 gnorm: 0.86 [14:28:59< 7:31:45] +[titan] 2025-07-10 04:04:03,885 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:04:04,677 - root - INFO - step: 65800 loss: 16.2377 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7493 +[titan] 2025-07-10 04:04:04,677 - root - INFO - lr: 1.0200e-04 gnorm: 0.92 [14:29:03< 7:31:41] +[titan] 2025-07-10 04:04:08,586 - root - INFO - step: 65805 loss: 16.5647 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 14.0301 +[titan] 2025-07-10 04:04:08,586 - root - INFO - lr: 1.0198e-04 gnorm: 0.89 [14:29:07< 7:31:37] +[titan] 2025-07-10 04:04:12,496 - root - INFO - step: 65810 loss: 16.6110 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0617 +[titan] 2025-07-10 04:04:12,496 - root - INFO - lr: 1.0196e-04 gnorm: 0.87 [14:29:11< 7:31:33] +[titan] 2025-07-10 04:04:16,397 - root - INFO - step: 65815 loss: 16.3820 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.8793 +[titan] 2025-07-10 04:04:16,397 - root - INFO - lr: 1.0194e-04 gnorm: 0.85 [14:29:14< 7:31:29] +[titan] 2025-07-10 04:04:20,315 - root - INFO - step: 65820 loss: 16.2421 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 04:04:20,315 - root - INFO - lr: 1.0192e-04 gnorm: 0.86 [14:29:18< 7:31:25] +[titan] 2025-07-10 04:04:24,218 - root - INFO - step: 65825 loss: 16.2638 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 04:04:24,219 - root - INFO - lr: 1.0190e-04 gnorm: 0.84 [14:29:22< 7:31:21] +[titan] 2025-07-10 04:04:28,184 - root - INFO - step: 65830 loss: 16.2095 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.7263 +[titan] 2025-07-10 04:04:28,184 - root - INFO - lr: 1.0188e-04 gnorm: 0.86 [14:29:26< 7:31:17] +[titan] 2025-07-10 04:04:32,113 - root - INFO - step: 65835 loss: 16.4773 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.9547 +[titan] 2025-07-10 04:04:32,114 - root - INFO - lr: 1.0187e-04 gnorm: 0.89 [14:29:30< 7:31:13] +[titan] 2025-07-10 04:04:36,057 - root - INFO - step: 65840 loss: 16.3638 memory: 44.58GiB(31.99%) tps: 83,109 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8469 +[titan] 2025-07-10 04:04:36,057 - root - INFO - lr: 1.0185e-04 gnorm: 0.90 [14:29:34< 7:31:09] +[titan] 2025-07-10 04:04:39,972 - root - INFO - step: 65845 loss: 15.9961 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4503 global_avg_mtp_loss: 13.5458 +[titan] 2025-07-10 04:04:39,972 - root - INFO - lr: 1.0183e-04 gnorm: 0.88 [14:29:38< 7:31:05] +[titan] 2025-07-10 04:04:43,143 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:04:43,943 - root - INFO - step: 65850 loss: 16.5888 memory: 44.58GiB(31.99%) tps: 82,518 tflops: 284.78 mfu: 28.80% global_avg_ntp_loss: 2.5578 global_avg_mtp_loss: 14.0310 +[titan] 2025-07-10 04:04:43,943 - root - INFO - lr: 1.0181e-04 gnorm: 0.95 [14:29:42< 7:31:02] +[titan] 2025-07-10 04:04:47,873 - root - INFO - step: 65855 loss: 16.3484 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8412 +[titan] 2025-07-10 04:04:47,873 - root - INFO - lr: 1.0179e-04 gnorm: 0.87 [14:29:46< 7:30:58] +[titan] 2025-07-10 04:04:51,799 - root - INFO - step: 65860 loss: 16.3190 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 04:04:51,799 - root - INFO - lr: 1.0177e-04 gnorm: 0.85 [14:29:50< 7:30:54] +[titan] 2025-07-10 04:04:55,718 - root - INFO - step: 65865 loss: 16.2373 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7482 +[titan] 2025-07-10 04:04:55,719 - root - INFO - lr: 1.0175e-04 gnorm: 0.88 [14:29:54< 7:30:50] +[titan] 2025-07-10 04:04:59,649 - root - INFO - step: 65870 loss: 16.1225 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4578 global_avg_mtp_loss: 13.6647 +[titan] 2025-07-10 04:04:59,650 - root - INFO - lr: 1.0173e-04 gnorm: 0.98 [14:29:58< 7:30:46] +[titan] 2025-07-10 04:05:03,561 - root - INFO - step: 65875 loss: 16.6364 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0750 +[titan] 2025-07-10 04:05:03,561 - root - INFO - lr: 1.0171e-04 gnorm: 0.89 [14:30:02< 7:30:42] +[titan] 2025-07-10 04:05:07,479 - root - INFO - step: 65880 loss: 16.3424 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8324 +[titan] 2025-07-10 04:05:07,480 - root - INFO - lr: 1.0170e-04 gnorm: 0.96 [14:30:06< 7:30:38] +[titan] 2025-07-10 04:05:11,423 - root - INFO - step: 65885 loss: 16.0479 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4382 global_avg_mtp_loss: 13.6097 +[titan] 2025-07-10 04:05:11,423 - root - INFO - lr: 1.0168e-04 gnorm: 1.03 [14:30:10< 7:30:34] +[titan] 2025-07-10 04:05:15,341 - root - INFO - step: 65890 loss: 16.5692 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0235 +[titan] 2025-07-10 04:05:15,341 - root - INFO - lr: 1.0166e-04 gnorm: 0.89 [14:30:13< 7:30:30] +[titan] 2025-07-10 04:05:19,265 - root - INFO - step: 65895 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0019 +[titan] 2025-07-10 04:05:19,265 - root - INFO - lr: 1.0164e-04 gnorm: 0.93 [14:30:17< 7:30:26] +[titan] 2025-07-10 04:05:22,376 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:05:23,166 - root - INFO - step: 65900 loss: 16.4036 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8968 +[titan] 2025-07-10 04:05:23,166 - root - INFO - lr: 1.0162e-04 gnorm: 0.91 [14:30:21< 7:30:22] +[titan] 2025-07-10 04:05:27,061 - root - INFO - step: 65905 loss: 16.2797 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.7809 +[titan] 2025-07-10 04:05:27,061 - root - INFO - lr: 1.0160e-04 gnorm: 0.89 [14:30:25< 7:30:18] +[titan] 2025-07-10 04:05:30,992 - root - INFO - step: 65910 loss: 16.5103 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9736 +[titan] 2025-07-10 04:05:30,992 - root - INFO - lr: 1.0158e-04 gnorm: 0.87 [14:30:29< 7:30:14] +[titan] 2025-07-10 04:05:34,892 - root - INFO - step: 65915 loss: 16.1921 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.7229 +[titan] 2025-07-10 04:05:34,893 - root - INFO - lr: 1.0156e-04 gnorm: 0.91 [14:30:33< 7:30:10] +[titan] 2025-07-10 04:05:38,826 - root - INFO - step: 65920 loss: 16.2964 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 04:05:38,826 - root - INFO - lr: 1.0154e-04 gnorm: 0.88 [14:30:37< 7:30:06] +[titan] 2025-07-10 04:05:42,795 - root - INFO - step: 65925 loss: 16.4406 memory: 44.58GiB(31.99%) tps: 82,559 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 04:05:42,795 - root - INFO - lr: 1.0153e-04 gnorm: 0.90 [14:30:41< 7:30:02] +[titan] 2025-07-10 04:05:46,710 - root - INFO - step: 65930 loss: 16.2636 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.7617 +[titan] 2025-07-10 04:05:46,710 - root - INFO - lr: 1.0151e-04 gnorm: 0.98 [14:30:45< 7:29:58] +[titan] 2025-07-10 04:05:50,658 - root - INFO - step: 65935 loss: 16.2600 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7748 +[titan] 2025-07-10 04:05:50,658 - root - INFO - lr: 1.0149e-04 gnorm: 0.88 [14:30:49< 7:29:54] +[titan] 2025-07-10 04:05:54,602 - root - INFO - step: 65940 loss: 16.3904 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.8786 +[titan] 2025-07-10 04:05:54,603 - root - INFO - lr: 1.0147e-04 gnorm: 0.88 [14:30:53< 7:29:50] +[titan] 2025-07-10 04:05:58,533 - root - INFO - step: 65945 loss: 16.3385 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8373 +[titan] 2025-07-10 04:05:58,533 - root - INFO - lr: 1.0145e-04 gnorm: 0.93 [14:30:57< 7:29:46] +[titan] 2025-07-10 04:06:01,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:06:02,463 - root - INFO - step: 65950 loss: 16.2473 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.7549 +[titan] 2025-07-10 04:06:02,463 - root - INFO - lr: 1.0143e-04 gnorm: 0.88 [14:31:01< 7:29:42] +[titan] 2025-07-10 04:06:06,374 - root - INFO - step: 65955 loss: 16.4029 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5094 global_avg_mtp_loss: 13.8935 +[titan] 2025-07-10 04:06:06,375 - root - INFO - lr: 1.0141e-04 gnorm: 0.91 [14:31:04< 7:29:38] +[titan] 2025-07-10 04:06:10,274 - root - INFO - step: 65960 loss: 16.4136 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.8922 +[titan] 2025-07-10 04:06:10,274 - root - INFO - lr: 1.0139e-04 gnorm: 0.89 [14:31:08< 7:29:34] +[titan] 2025-07-10 04:06:14,196 - root - INFO - step: 65965 loss: 16.4804 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.9549 +[titan] 2025-07-10 04:06:14,197 - root - INFO - lr: 1.0137e-04 gnorm: 0.92 [14:31:12< 7:29:30] +[titan] 2025-07-10 04:06:18,137 - root - INFO - step: 65970 loss: 16.5760 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0186 +[titan] 2025-07-10 04:06:18,137 - root - INFO - lr: 1.0136e-04 gnorm: 0.92 [14:31:16< 7:29:26] +[titan] 2025-07-10 04:06:22,073 - root - INFO - step: 65975 loss: 16.1766 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7038 +[titan] 2025-07-10 04:06:22,073 - root - INFO - lr: 1.0134e-04 gnorm: 0.87 [14:31:20< 7:29:22] +[titan] 2025-07-10 04:06:25,972 - root - INFO - step: 65980 loss: 16.3304 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.8304 +[titan] 2025-07-10 04:06:25,972 - root - INFO - lr: 1.0132e-04 gnorm: 0.90 [14:31:24< 7:29:18] +[titan] 2025-07-10 04:06:29,894 - root - INFO - step: 65985 loss: 16.3305 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8299 +[titan] 2025-07-10 04:06:29,894 - root - INFO - lr: 1.0130e-04 gnorm: 0.94 [14:31:28< 7:29:14] +[titan] 2025-07-10 04:06:33,800 - root - INFO - step: 65990 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.8840 +[titan] 2025-07-10 04:06:33,801 - root - INFO - lr: 1.0128e-04 gnorm: 0.88 [14:31:32< 7:29:10] +[titan] 2025-07-10 04:06:37,765 - root - INFO - step: 65995 loss: 16.6053 memory: 44.58GiB(31.99%) tps: 82,652 tflops: 285.25 mfu: 28.84% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0489 +[titan] 2025-07-10 04:06:37,766 - root - INFO - lr: 1.0126e-04 gnorm: 0.87 [14:31:36< 7:29:06] +[titan] 2025-07-10 04:06:40,940 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:06:41,750 - root - INFO - step: 66000 loss: 16.3275 memory: 44.58GiB(31.99%) tps: 82,243 tflops: 283.83 mfu: 28.70% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8185 +[titan] 2025-07-10 04:06:41,750 - root - INFO - lr: 1.0124e-04 gnorm: 0.92 [14:31:40< 7:29:02] +[titan] 2025-07-10 04:06:45,687 - root - INFO - step: 66005 loss: 16.4233 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.9060 +[titan] 2025-07-10 04:06:45,687 - root - INFO - lr: 1.0122e-04 gnorm: 0.86 [14:31:44< 7:28:58] +[titan] 2025-07-10 04:06:49,608 - root - INFO - step: 66010 loss: 16.2719 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7767 +[titan] 2025-07-10 04:06:49,608 - root - INFO - lr: 1.0120e-04 gnorm: 0.89 [14:31:48< 7:28:54] +[titan] 2025-07-10 04:06:53,504 - root - INFO - step: 66015 loss: 16.5143 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9796 +[titan] 2025-07-10 04:06:53,504 - root - INFO - lr: 1.0119e-04 gnorm: 0.88 [14:31:52< 7:28:50] +[titan] 2025-07-10 04:06:57,417 - root - INFO - step: 66020 loss: 16.3040 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8015 +[titan] 2025-07-10 04:06:57,417 - root - INFO - lr: 1.0117e-04 gnorm: 0.93 [14:31:55< 7:28:46] +[titan] 2025-07-10 04:07:01,325 - root - INFO - step: 66025 loss: 16.4059 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 04:07:01,325 - root - INFO - lr: 1.0115e-04 gnorm: 0.89 [14:31:59< 7:28:42] +[titan] 2025-07-10 04:07:05,242 - root - INFO - step: 66030 loss: 16.3321 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 04:07:05,242 - root - INFO - lr: 1.0113e-04 gnorm: 0.91 [14:32:03< 7:28:38] +[titan] 2025-07-10 04:07:09,178 - root - INFO - step: 66035 loss: 16.5497 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5510 global_avg_mtp_loss: 13.9987 +[titan] 2025-07-10 04:07:09,179 - root - INFO - lr: 1.0111e-04 gnorm: 0.87 [14:32:07< 7:28:34] +[titan] 2025-07-10 04:07:13,087 - root - INFO - step: 66040 loss: 16.5364 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5443 global_avg_mtp_loss: 13.9921 +[titan] 2025-07-10 04:07:13,088 - root - INFO - lr: 1.0109e-04 gnorm: 0.87 [14:32:11< 7:28:30] +[titan] 2025-07-10 04:07:17,081 - root - INFO - step: 66045 loss: 16.6461 memory: 44.58GiB(31.99%) tps: 82,063 tflops: 283.21 mfu: 28.64% global_avg_ntp_loss: 2.5672 global_avg_mtp_loss: 14.0789 +[titan] 2025-07-10 04:07:17,081 - root - INFO - lr: 1.0107e-04 gnorm: 0.94 [14:32:15< 7:28:26] +[titan] 2025-07-10 04:07:19,588 - root - INFO - Dumping profiler traces at step 66048 +[titan] 2025-07-10 04:07:19,620 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:07:20,411 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:07:21,208 - root - INFO - step: 66050 loss: 16.3033 memory: 44.58GiB(31.99%) tps: 79,404 tflops: 274.04 mfu: 27.71% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8032 +[titan] 2025-07-10 04:07:21,208 - root - INFO - lr: 1.0105e-04 gnorm: 0.88 [14:32:19< 7:28:22] +[titan] 2025-07-10 04:07:25,119 - root - INFO - step: 66055 loss: 16.4342 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9042 +[titan] 2025-07-10 04:07:25,119 - root - INFO - lr: 1.0103e-04 gnorm: 0.89 [14:32:23< 7:28:18] +[titan] 2025-07-10 04:07:29,032 - root - INFO - step: 66060 loss: 16.5893 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5621 global_avg_mtp_loss: 14.0271 +[titan] 2025-07-10 04:07:29,032 - root - INFO - lr: 1.0102e-04 gnorm: 0.91 [14:32:27< 7:28:14] +[titan] 2025-07-10 04:07:32,981 - root - INFO - step: 66065 loss: 16.2939 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 04:07:32,981 - root - INFO - lr: 1.0100e-04 gnorm: 0.92 [14:32:31< 7:28:10] +[titan] 2025-07-10 04:07:36,919 - root - INFO - step: 66070 loss: 16.2879 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.8028 +[titan] 2025-07-10 04:07:36,919 - root - INFO - lr: 1.0098e-04 gnorm: 0.88 [14:32:35< 7:28:06] +[titan] 2025-07-10 04:07:40,862 - root - INFO - step: 66075 loss: 16.3887 memory: 44.58GiB(31.99%) tps: 83,110 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8719 +[titan] 2025-07-10 04:07:40,862 - root - INFO - lr: 1.0096e-04 gnorm: 0.86 [14:32:39< 7:28:02] +[titan] 2025-07-10 04:07:44,787 - root - INFO - step: 66080 loss: 16.3543 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 04:07:44,787 - root - INFO - lr: 1.0094e-04 gnorm: 0.82 [14:32:43< 7:27:59] +[titan] 2025-07-10 04:07:48,719 - root - INFO - step: 66085 loss: 16.1631 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.6922 +[titan] 2025-07-10 04:07:48,719 - root - INFO - lr: 1.0092e-04 gnorm: 0.87 [14:32:47< 7:27:55] +[titan] 2025-07-10 04:07:52,661 - root - INFO - step: 66090 loss: 16.2287 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7425 +[titan] 2025-07-10 04:07:52,661 - root - INFO - lr: 1.0090e-04 gnorm: 0.92 [14:32:51< 7:27:51] +[titan] 2025-07-10 04:07:56,593 - root - INFO - step: 66095 loss: 16.5456 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 14.0032 +[titan] 2025-07-10 04:07:56,593 - root - INFO - lr: 1.0088e-04 gnorm: 0.88 [14:32:55< 7:27:47] +[titan] 2025-07-10 04:07:59,779 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:08:00,588 - root - INFO - step: 66100 loss: 16.4509 memory: 44.58GiB(31.99%) tps: 82,031 tflops: 283.10 mfu: 28.63% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9249 +[titan] 2025-07-10 04:08:00,588 - root - INFO - lr: 1.0086e-04 gnorm: 0.86 [14:32:59< 7:27:43] +[titan] 2025-07-10 04:08:04,540 - root - INFO - step: 66105 loss: 16.5432 memory: 44.58GiB(31.99%) tps: 82,931 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 14.0048 +[titan] 2025-07-10 04:08:04,540 - root - INFO - lr: 1.0085e-04 gnorm: 0.88 [14:33:03< 7:27:39] +[titan] 2025-07-10 04:08:08,493 - root - INFO - step: 66110 loss: 16.2954 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.5007 global_avg_mtp_loss: 13.7947 +[titan] 2025-07-10 04:08:08,494 - root - INFO - lr: 1.0083e-04 gnorm: 0.88 [14:33:07< 7:27:35] +[titan] 2025-07-10 04:08:12,456 - root - INFO - step: 66115 loss: 16.4435 memory: 44.58GiB(31.99%) tps: 82,694 tflops: 285.39 mfu: 28.86% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.9301 +[titan] 2025-07-10 04:08:12,456 - root - INFO - lr: 1.0081e-04 gnorm: 0.85 [14:33:11< 7:27:31] +[titan] 2025-07-10 04:08:16,455 - root - INFO - step: 66120 loss: 16.4217 memory: 44.58GiB(31.99%) tps: 81,947 tflops: 282.81 mfu: 28.60% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9049 +[titan] 2025-07-10 04:08:16,456 - root - INFO - lr: 1.0079e-04 gnorm: 0.88 [14:33:15< 7:27:27] +[titan] 2025-07-10 04:08:20,375 - root - INFO - step: 66125 loss: 16.3669 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8619 +[titan] 2025-07-10 04:08:20,375 - root - INFO - lr: 1.0077e-04 gnorm: 0.88 [14:33:18< 7:27:23] +[titan] 2025-07-10 04:08:24,299 - root - INFO - step: 66130 loss: 16.3268 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.8042 +[titan] 2025-07-10 04:08:24,299 - root - INFO - lr: 1.0075e-04 gnorm: 0.88 [14:33:22< 7:27:19] +[titan] 2025-07-10 04:08:28,243 - root - INFO - step: 66135 loss: 16.4597 memory: 44.58GiB(31.99%) tps: 83,082 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9284 +[titan] 2025-07-10 04:08:28,243 - root - INFO - lr: 1.0073e-04 gnorm: 0.91 [14:33:26< 7:27:15] +[titan] 2025-07-10 04:08:32,162 - root - INFO - step: 66140 loss: 16.2856 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.7896 +[titan] 2025-07-10 04:08:32,162 - root - INFO - lr: 1.0071e-04 gnorm: 0.89 [14:33:30< 7:27:11] +[titan] 2025-07-10 04:08:36,055 - root - INFO - step: 66145 loss: 16.1934 memory: 44.58GiB(31.99%) tps: 84,185 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7140 +[titan] 2025-07-10 04:08:36,055 - root - INFO - lr: 1.0070e-04 gnorm: 0.98 [14:33:34< 7:27:07] +[titan] 2025-07-10 04:08:39,165 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:08:39,999 - root - INFO - step: 66150 loss: 16.3335 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8194 +[titan] 2025-07-10 04:08:39,999 - root - INFO - lr: 1.0068e-04 gnorm: 0.90 [14:33:38< 7:27:03] +[titan] 2025-07-10 04:08:43,958 - root - INFO - step: 66155 loss: 16.5148 memory: 44.58GiB(31.99%) tps: 82,759 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9797 +[titan] 2025-07-10 04:08:43,959 - root - INFO - lr: 1.0066e-04 gnorm: 0.86 [14:33:42< 7:26:59] +[titan] 2025-07-10 04:08:47,854 - root - INFO - step: 66160 loss: 16.2271 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7483 +[titan] 2025-07-10 04:08:47,855 - root - INFO - lr: 1.0064e-04 gnorm: 0.98 [14:33:46< 7:26:55] +[titan] 2025-07-10 04:08:51,816 - root - INFO - step: 66165 loss: 16.4064 memory: 44.58GiB(31.99%) tps: 82,717 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8890 +[titan] 2025-07-10 04:08:51,817 - root - INFO - lr: 1.0062e-04 gnorm: 0.92 [14:33:50< 7:26:51] +[titan] 2025-07-10 04:08:55,719 - root - INFO - step: 66170 loss: 16.2198 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.7464 +[titan] 2025-07-10 04:08:55,720 - root - INFO - lr: 1.0060e-04 gnorm: 0.87 [14:33:54< 7:26:47] +[titan] 2025-07-10 04:08:59,627 - root - INFO - step: 66175 loss: 16.1732 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.6900 +[titan] 2025-07-10 04:08:59,627 - root - INFO - lr: 1.0058e-04 gnorm: 0.86 [14:33:58< 7:26:43] +[titan] 2025-07-10 04:09:03,552 - root - INFO - step: 66180 loss: 16.4900 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9553 +[titan] 2025-07-10 04:09:03,552 - root - INFO - lr: 1.0056e-04 gnorm: 0.87 [14:34:02< 7:26:39] +[titan] 2025-07-10 04:09:07,514 - root - INFO - step: 66185 loss: 16.2927 memory: 44.58GiB(31.99%) tps: 82,705 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.8024 +[titan] 2025-07-10 04:09:07,515 - root - INFO - lr: 1.0054e-04 gnorm: 0.87 [14:34:06< 7:26:35] +[titan] 2025-07-10 04:09:11,463 - root - INFO - step: 66190 loss: 16.4168 memory: 44.58GiB(31.99%) tps: 83,003 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8931 +[titan] 2025-07-10 04:09:11,463 - root - INFO - lr: 1.0053e-04 gnorm: 0.91 [14:34:10< 7:26:31] +[titan] 2025-07-10 04:09:15,373 - root - INFO - step: 66195 loss: 16.3104 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8133 +[titan] 2025-07-10 04:09:15,373 - root - INFO - lr: 1.0051e-04 gnorm: 0.91 [14:34:13< 7:26:27] +[titan] 2025-07-10 04:09:18,495 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:09:19,284 - root - INFO - step: 66200 loss: 16.3417 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 04:09:19,285 - root - INFO - lr: 1.0049e-04 gnorm: 0.83 [14:34:17< 7:26:23] +[titan] 2025-07-10 04:09:23,184 - root - INFO - step: 66205 loss: 16.0805 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6219 +[titan] 2025-07-10 04:09:23,185 - root - INFO - lr: 1.0047e-04 gnorm: 0.90 [14:34:21< 7:26:19] +[titan] 2025-07-10 04:09:27,120 - root - INFO - step: 66210 loss: 16.3268 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8298 +[titan] 2025-07-10 04:09:27,121 - root - INFO - lr: 1.0045e-04 gnorm: 0.89 [14:34:25< 7:26:15] +[titan] 2025-07-10 04:09:31,040 - root - INFO - step: 66215 loss: 16.8383 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5924 global_avg_mtp_loss: 14.2459 +[titan] 2025-07-10 04:09:31,041 - root - INFO - lr: 1.0043e-04 gnorm: 0.86 [14:34:29< 7:26:11] +[titan] 2025-07-10 04:09:35,032 - root - INFO - step: 66220 loss: 16.4766 memory: 44.58GiB(31.99%) tps: 82,096 tflops: 283.33 mfu: 28.65% global_avg_ntp_loss: 2.5322 global_avg_mtp_loss: 13.9444 +[titan] 2025-07-10 04:09:35,032 - root - INFO - lr: 1.0041e-04 gnorm: 0.86 [14:34:33< 7:26:07] +[titan] 2025-07-10 04:09:38,983 - root - INFO - step: 66225 loss: 16.2878 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.7879 +[titan] 2025-07-10 04:09:38,983 - root - INFO - lr: 1.0039e-04 gnorm: 0.87 [14:34:37< 7:26:03] +[titan] 2025-07-10 04:09:42,893 - root - INFO - step: 66230 loss: 16.4292 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.8997 +[titan] 2025-07-10 04:09:42,893 - root - INFO - lr: 1.0038e-04 gnorm: 0.89 [14:34:41< 7:25:59] +[titan] 2025-07-10 04:09:46,876 - root - INFO - step: 66235 loss: 16.4984 memory: 44.58GiB(31.99%) tps: 82,269 tflops: 283.92 mfu: 28.71% global_avg_ntp_loss: 2.5462 global_avg_mtp_loss: 13.9522 +[titan] 2025-07-10 04:09:46,876 - root - INFO - lr: 1.0036e-04 gnorm: 0.98 [14:34:45< 7:25:55] +[titan] 2025-07-10 04:09:50,980 - root - INFO - step: 66240 loss: 16.0626 memory: 44.58GiB(31.99%) tps: 79,847 tflops: 275.57 mfu: 27.86% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6089 +[titan] 2025-07-10 04:09:50,981 - root - INFO - lr: 1.0034e-04 gnorm: 0.93 [14:34:49< 7:25:51] +[titan] 2025-07-10 04:09:54,946 - root - INFO - step: 66245 loss: 16.4595 memory: 44.58GiB(31.99%) tps: 82,645 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.9475 +[titan] 2025-07-10 04:09:54,946 - root - INFO - lr: 1.0032e-04 gnorm: 0.89 [14:34:53< 7:25:47] +[titan] 2025-07-10 04:09:58,086 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:09:58,878 - root - INFO - step: 66250 loss: 16.4378 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.9185 +[titan] 2025-07-10 04:09:58,878 - root - INFO - lr: 1.0030e-04 gnorm: 0.92 [14:34:57< 7:25:43] +[titan] 2025-07-10 04:10:02,781 - root - INFO - step: 66255 loss: 16.1947 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4749 global_avg_mtp_loss: 13.7198 +[titan] 2025-07-10 04:10:02,781 - root - INFO - lr: 1.0028e-04 gnorm: 0.91 [14:35:01< 7:25:39] +[titan] 2025-07-10 04:10:06,738 - root - INFO - step: 66260 loss: 16.6191 memory: 44.58GiB(31.99%) tps: 82,818 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0589 +[titan] 2025-07-10 04:10:06,738 - root - INFO - lr: 1.0026e-04 gnorm: 0.87 [14:35:05< 7:25:36] +[titan] 2025-07-10 04:10:10,662 - root - INFO - step: 66265 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.7920 +[titan] 2025-07-10 04:10:10,662 - root - INFO - lr: 1.0024e-04 gnorm: 0.89 [14:35:09< 7:25:32] +[titan] 2025-07-10 04:10:14,599 - root - INFO - step: 66270 loss: 16.4657 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.9328 +[titan] 2025-07-10 04:10:14,599 - root - INFO - lr: 1.0022e-04 gnorm: 0.88 [14:35:13< 7:25:28] +[titan] 2025-07-10 04:10:18,493 - root - INFO - step: 66275 loss: 16.5058 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9645 +[titan] 2025-07-10 04:10:18,494 - root - INFO - lr: 1.0021e-04 gnorm: 0.85 [14:35:17< 7:25:24] +[titan] 2025-07-10 04:10:22,420 - root - INFO - step: 66280 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8825 +[titan] 2025-07-10 04:10:22,420 - root - INFO - lr: 1.0019e-04 gnorm: 0.87 [14:35:20< 7:25:20] +[titan] 2025-07-10 04:10:26,344 - root - INFO - step: 66285 loss: 16.0860 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6236 +[titan] 2025-07-10 04:10:26,345 - root - INFO - lr: 1.0017e-04 gnorm: 0.88 [14:35:24< 7:25:16] +[titan] 2025-07-10 04:10:30,238 - root - INFO - step: 66290 loss: 16.3192 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.8077 +[titan] 2025-07-10 04:10:30,239 - root - INFO - lr: 1.0015e-04 gnorm: 0.89 [14:35:28< 7:25:12] +[titan] 2025-07-10 04:10:34,159 - root - INFO - step: 66295 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8296 +[titan] 2025-07-10 04:10:34,159 - root - INFO - lr: 1.0013e-04 gnorm: 0.93 [14:35:32< 7:25:08] +[titan] 2025-07-10 04:10:37,285 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:10:38,076 - root - INFO - step: 66300 loss: 16.4568 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.9380 +[titan] 2025-07-10 04:10:38,076 - root - INFO - lr: 1.0011e-04 gnorm: 0.89 [14:35:36< 7:25:04] +[titan] 2025-07-10 04:10:41,990 - root - INFO - step: 66305 loss: 15.8580 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4222 global_avg_mtp_loss: 13.4358 +[titan] 2025-07-10 04:10:41,991 - root - INFO - lr: 1.0009e-04 gnorm: 0.95 [14:35:40< 7:25:00] +[titan] 2025-07-10 04:10:45,899 - root - INFO - step: 66310 loss: 16.3812 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8670 +[titan] 2025-07-10 04:10:45,900 - root - INFO - lr: 1.0007e-04 gnorm: 0.92 [14:35:44< 7:24:56] +[titan] 2025-07-10 04:10:49,807 - root - INFO - step: 66315 loss: 16.5565 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 14.0129 +[titan] 2025-07-10 04:10:49,807 - root - INFO - lr: 1.0006e-04 gnorm: 0.87 [14:35:48< 7:24:52] +[titan] 2025-07-10 04:10:53,702 - root - INFO - step: 66320 loss: 16.3768 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8568 +[titan] 2025-07-10 04:10:53,702 - root - INFO - lr: 1.0004e-04 gnorm: 0.95 [14:35:52< 7:24:48] +[titan] 2025-07-10 04:10:57,618 - root - INFO - step: 66325 loss: 16.2827 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.8067 +[titan] 2025-07-10 04:10:57,618 - root - INFO - lr: 1.0002e-04 gnorm: 0.86 [14:35:56< 7:24:44] +[titan] 2025-07-10 04:11:01,529 - root - INFO - step: 66330 loss: 16.6675 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5796 global_avg_mtp_loss: 14.0879 +[titan] 2025-07-10 04:11:01,529 - root - INFO - lr: 1.0000e-04 gnorm: 0.97 [14:36:00< 7:24:40] +[titan] 2025-07-10 04:11:05,440 - root - INFO - step: 66335 loss: 16.3985 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8896 +[titan] 2025-07-10 04:11:05,441 - root - INFO - lr: 9.9981e-05 gnorm: 0.92 [14:36:03< 7:24:36] +[titan] 2025-07-10 04:11:09,361 - root - INFO - step: 66340 loss: 16.1016 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4524 global_avg_mtp_loss: 13.6492 +[titan] 2025-07-10 04:11:09,361 - root - INFO - lr: 9.9962e-05 gnorm: 0.94 [14:36:07< 7:24:32] +[titan] 2025-07-10 04:11:13,329 - root - INFO - step: 66345 loss: 16.5278 memory: 44.58GiB(31.99%) tps: 82,589 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9926 +[titan] 2025-07-10 04:11:13,329 - root - INFO - lr: 9.9943e-05 gnorm: 0.93 [14:36:11< 7:24:28] +[titan] 2025-07-10 04:11:16,440 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:11:17,236 - root - INFO - step: 66350 loss: 16.6765 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5832 global_avg_mtp_loss: 14.0932 +[titan] 2025-07-10 04:11:17,236 - root - INFO - lr: 9.9924e-05 gnorm: 0.94 [14:36:15< 7:24:24] +[titan] 2025-07-10 04:11:21,141 - root - INFO - step: 66355 loss: 16.3181 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8189 +[titan] 2025-07-10 04:11:21,141 - root - INFO - lr: 9.9906e-05 gnorm: 0.91 [14:36:19< 7:24:20] +[titan] 2025-07-10 04:11:25,082 - root - INFO - step: 66360 loss: 16.1822 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 04:11:25,082 - root - INFO - lr: 9.9887e-05 gnorm: 0.91 [14:36:23< 7:24:16] +[titan] 2025-07-10 04:11:28,988 - root - INFO - step: 66365 loss: 16.3363 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-10 04:11:28,988 - root - INFO - lr: 9.9868e-05 gnorm: 0.89 [14:36:27< 7:24:12] +[titan] 2025-07-10 04:11:32,908 - root - INFO - step: 66370 loss: 16.3840 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8771 +[titan] 2025-07-10 04:11:32,908 - root - INFO - lr: 9.9849e-05 gnorm: 0.88 [14:36:31< 7:24:08] +[titan] 2025-07-10 04:11:36,864 - root - INFO - step: 66375 loss: 16.1561 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.6860 +[titan] 2025-07-10 04:11:36,865 - root - INFO - lr: 9.9831e-05 gnorm: 0.99 [14:36:35< 7:24:04] +[titan] 2025-07-10 04:11:40,783 - root - INFO - step: 66380 loss: 16.5120 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9861 +[titan] 2025-07-10 04:11:40,783 - root - INFO - lr: 9.9812e-05 gnorm: 0.91 [14:36:39< 7:24:00] +[titan] 2025-07-10 04:11:44,686 - root - INFO - step: 66385 loss: 16.4363 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.9257 +[titan] 2025-07-10 04:11:44,686 - root - INFO - lr: 9.9793e-05 gnorm: 0.92 [14:36:43< 7:23:56] +[titan] 2025-07-10 04:11:48,649 - root - INFO - step: 66390 loss: 16.5045 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.5452 global_avg_mtp_loss: 13.9593 +[titan] 2025-07-10 04:11:48,650 - root - INFO - lr: 9.9774e-05 gnorm: 0.94 [14:36:47< 7:23:52] +[titan] 2025-07-10 04:11:52,554 - root - INFO - step: 66395 loss: 16.3861 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8709 +[titan] 2025-07-10 04:11:52,555 - root - INFO - lr: 9.9756e-05 gnorm: 0.87 [14:36:51< 7:23:48] +[titan] 2025-07-10 04:11:55,666 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:11:56,452 - root - INFO - step: 66400 loss: 16.5281 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9888 +[titan] 2025-07-10 04:11:56,452 - root - INFO - lr: 9.9737e-05 gnorm: 0.87 [14:36:54< 7:23:44] +[titan] 2025-07-10 04:12:00,365 - root - INFO - step: 66405 loss: 16.3598 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 04:12:00,365 - root - INFO - lr: 9.9718e-05 gnorm: 0.97 [14:36:58< 7:23:40] +[titan] 2025-07-10 04:12:04,272 - root - INFO - step: 66410 loss: 16.4531 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9216 +[titan] 2025-07-10 04:12:04,272 - root - INFO - lr: 9.9699e-05 gnorm: 0.94 [14:37:02< 7:23:36] +[titan] 2025-07-10 04:12:08,193 - root - INFO - step: 66415 loss: 16.0444 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.5834 +[titan] 2025-07-10 04:12:08,193 - root - INFO - lr: 9.9681e-05 gnorm: 0.90 [14:37:06< 7:23:32] +[titan] 2025-07-10 04:12:12,110 - root - INFO - step: 66420 loss: 16.2010 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7277 +[titan] 2025-07-10 04:12:12,110 - root - INFO - lr: 9.9662e-05 gnorm: 0.85 [14:37:10< 7:23:28] +[titan] 2025-07-10 04:12:16,016 - root - INFO - step: 66425 loss: 16.4795 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5394 global_avg_mtp_loss: 13.9401 +[titan] 2025-07-10 04:12:16,017 - root - INFO - lr: 9.9643e-05 gnorm: 0.86 [14:37:14< 7:23:24] +[titan] 2025-07-10 04:12:19,950 - root - INFO - step: 66430 loss: 16.4001 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.8688 +[titan] 2025-07-10 04:12:19,950 - root - INFO - lr: 9.9624e-05 gnorm: 0.89 [14:37:18< 7:23:20] +[titan] 2025-07-10 04:12:23,889 - root - INFO - step: 66435 loss: 16.3156 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8098 +[titan] 2025-07-10 04:12:23,889 - root - INFO - lr: 9.9606e-05 gnorm: 0.85 [14:37:22< 7:23:16] +[titan] 2025-07-10 04:12:27,795 - root - INFO - step: 66440 loss: 16.4119 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.9012 +[titan] 2025-07-10 04:12:27,795 - root - INFO - lr: 9.9587e-05 gnorm: 0.92 [14:37:26< 7:23:12] +[titan] 2025-07-10 04:12:31,706 - root - INFO - step: 66445 loss: 16.1668 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.6973 +[titan] 2025-07-10 04:12:31,706 - root - INFO - lr: 9.9568e-05 gnorm: 0.87 [14:37:30< 7:23:08] +[titan] 2025-07-10 04:12:34,860 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:12:35,648 - root - INFO - step: 66450 loss: 16.2399 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.7418 +[titan] 2025-07-10 04:12:35,648 - root - INFO - lr: 9.9549e-05 gnorm: 0.90 [14:37:34< 7:23:04] +[titan] 2025-07-10 04:12:39,563 - root - INFO - step: 66455 loss: 16.1000 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6403 +[titan] 2025-07-10 04:12:39,563 - root - INFO - lr: 9.9531e-05 gnorm: 0.87 [14:37:38< 7:23:00] +[titan] 2025-07-10 04:12:43,502 - root - INFO - step: 66460 loss: 16.2787 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7891 +[titan] 2025-07-10 04:12:43,502 - root - INFO - lr: 9.9512e-05 gnorm: 0.85 [14:37:42< 7:22:56] +[titan] 2025-07-10 04:12:47,434 - root - INFO - step: 66465 loss: 16.1978 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7192 +[titan] 2025-07-10 04:12:47,435 - root - INFO - lr: 9.9493e-05 gnorm: 0.93 [14:37:45< 7:22:52] +[titan] 2025-07-10 04:12:51,359 - root - INFO - step: 66470 loss: 16.5382 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5474 global_avg_mtp_loss: 13.9909 +[titan] 2025-07-10 04:12:51,360 - root - INFO - lr: 9.9475e-05 gnorm: 0.87 [14:37:49< 7:22:48] +[titan] 2025-07-10 04:12:55,287 - root - INFO - step: 66475 loss: 16.3889 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8751 +[titan] 2025-07-10 04:12:55,288 - root - INFO - lr: 9.9456e-05 gnorm: 0.90 [14:37:53< 7:22:44] +[titan] 2025-07-10 04:12:59,246 - root - INFO - step: 66480 loss: 16.4751 memory: 44.58GiB(31.99%) tps: 82,791 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9531 +[titan] 2025-07-10 04:12:59,246 - root - INFO - lr: 9.9437e-05 gnorm: 0.86 [14:37:57< 7:22:40] +[titan] 2025-07-10 04:13:03,163 - root - INFO - step: 66485 loss: 16.3411 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8326 +[titan] 2025-07-10 04:13:03,164 - root - INFO - lr: 9.9418e-05 gnorm: 0.91 [14:38:01< 7:22:36] +[titan] 2025-07-10 04:13:07,083 - root - INFO - step: 66490 loss: 16.3121 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8121 +[titan] 2025-07-10 04:13:07,083 - root - INFO - lr: 9.9400e-05 gnorm: 0.89 [14:38:05< 7:22:32] +[titan] 2025-07-10 04:13:11,011 - root - INFO - step: 66495 loss: 16.1167 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.6442 +[titan] 2025-07-10 04:13:11,012 - root - INFO - lr: 9.9381e-05 gnorm: 0.88 [14:38:09< 7:22:28] +[titan] 2025-07-10 04:13:14,153 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:13:14,943 - root - INFO - step: 66500 loss: 16.2561 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7757 +[titan] 2025-07-10 04:13:14,943 - root - INFO - lr: 9.9362e-05 gnorm: 0.87 [14:38:13< 7:22:24] +[titan] 2025-07-10 04:13:18,870 - root - INFO - step: 66505 loss: 16.5986 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5626 global_avg_mtp_loss: 14.0361 +[titan] 2025-07-10 04:13:18,871 - root - INFO - lr: 9.9343e-05 gnorm: 0.90 [14:38:17< 7:22:20] +[titan] 2025-07-10 04:13:22,818 - root - INFO - step: 66510 loss: 16.2783 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7927 +[titan] 2025-07-10 04:13:22,818 - root - INFO - lr: 9.9325e-05 gnorm: 0.87 [14:38:21< 7:22:16] +[titan] 2025-07-10 04:13:26,762 - root - INFO - step: 66515 loss: 16.1771 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.7091 +[titan] 2025-07-10 04:13:26,762 - root - INFO - lr: 9.9306e-05 gnorm: 0.87 [14:38:25< 7:22:12] +[titan] 2025-07-10 04:13:30,703 - root - INFO - step: 66520 loss: 16.3252 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8211 +[titan] 2025-07-10 04:13:30,703 - root - INFO - lr: 9.9287e-05 gnorm: 0.92 [14:38:29< 7:22:08] +[titan] 2025-07-10 04:13:34,644 - root - INFO - step: 66525 loss: 16.5026 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9614 +[titan] 2025-07-10 04:13:34,644 - root - INFO - lr: 9.9269e-05 gnorm: 0.94 [14:38:33< 7:22:04] +[titan] 2025-07-10 04:13:38,578 - root - INFO - step: 66530 loss: 16.3411 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 04:13:38,578 - root - INFO - lr: 9.9250e-05 gnorm: 0.89 [14:38:37< 7:22:00] +[titan] 2025-07-10 04:13:42,506 - root - INFO - step: 66535 loss: 16.9283 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.6160 global_avg_mtp_loss: 14.3123 +[titan] 2025-07-10 04:13:42,506 - root - INFO - lr: 9.9231e-05 gnorm: 0.93 [14:38:41< 7:21:57] +[titan] 2025-07-10 04:13:46,405 - root - INFO - step: 66540 loss: 16.1551 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.6863 +[titan] 2025-07-10 04:13:46,406 - root - INFO - lr: 9.9212e-05 gnorm: 0.92 [14:38:44< 7:21:53] +[titan] 2025-07-10 04:13:50,315 - root - INFO - step: 66545 loss: 16.5056 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9643 +[titan] 2025-07-10 04:13:50,315 - root - INFO - lr: 9.9194e-05 gnorm: 0.90 [14:38:48< 7:21:49] +[titan] 2025-07-10 04:13:53,465 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:13:54,252 - root - INFO - step: 66550 loss: 16.3704 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8570 +[titan] 2025-07-10 04:13:54,252 - root - INFO - lr: 9.9175e-05 gnorm: 0.91 [14:38:52< 7:21:45] +[titan] 2025-07-10 04:13:58,181 - root - INFO - step: 66555 loss: 16.4738 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-10 04:13:58,181 - root - INFO - lr: 9.9156e-05 gnorm: 0.88 [14:38:56< 7:21:41] +[titan] 2025-07-10 04:14:02,167 - root - INFO - step: 66560 loss: 16.4730 memory: 44.58GiB(31.99%) tps: 82,213 tflops: 283.73 mfu: 28.69% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9274 +[titan] 2025-07-10 04:14:02,167 - root - INFO - lr: 9.9138e-05 gnorm: 0.91 [14:39:00< 7:21:37] +[titan] 2025-07-10 04:14:02,324 - root - INFO - Dumping profiler traces at step 66560 +[titan] 2025-07-10 04:14:02,356 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:14:06,269 - root - INFO - step: 66565 loss: 16.0763 memory: 44.58GiB(31.99%) tps: 79,894 tflops: 275.73 mfu: 27.88% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6147 +[titan] 2025-07-10 04:14:06,269 - root - INFO - lr: 9.9119e-05 gnorm: 0.89 [14:39:04< 7:21:33] +[titan] 2025-07-10 04:14:10,199 - root - INFO - step: 66570 loss: 16.2275 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7409 +[titan] 2025-07-10 04:14:10,199 - root - INFO - lr: 9.9100e-05 gnorm: 0.86 [14:39:08< 7:21:29] +[titan] 2025-07-10 04:14:14,126 - root - INFO - step: 66575 loss: 16.6423 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0848 +[titan] 2025-07-10 04:14:14,126 - root - INFO - lr: 9.9082e-05 gnorm: 0.86 [14:39:12< 7:21:25] +[titan] 2025-07-10 04:14:18,045 - root - INFO - step: 66580 loss: 16.2630 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.7733 +[titan] 2025-07-10 04:14:18,045 - root - INFO - lr: 9.9063e-05 gnorm: 0.88 [14:39:16< 7:21:21] +[titan] 2025-07-10 04:14:22,002 - root - INFO - step: 66585 loss: 16.3620 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5253 global_avg_mtp_loss: 13.8367 +[titan] 2025-07-10 04:14:22,003 - root - INFO - lr: 9.9044e-05 gnorm: 0.88 [14:39:20< 7:21:17] +[titan] 2025-07-10 04:14:25,944 - root - INFO - step: 66590 loss: 16.0596 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4605 global_avg_mtp_loss: 13.5990 +[titan] 2025-07-10 04:14:25,944 - root - INFO - lr: 9.9026e-05 gnorm: 0.93 [14:39:24< 7:21:13] +[titan] 2025-07-10 04:14:29,875 - root - INFO - step: 66595 loss: 16.2358 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 04:14:29,876 - root - INFO - lr: 9.9007e-05 gnorm: 0.91 [14:39:28< 7:21:09] +[titan] 2025-07-10 04:14:33,011 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:14:33,806 - root - INFO - step: 66600 loss: 16.5782 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0289 +[titan] 2025-07-10 04:14:33,806 - root - INFO - lr: 9.8988e-05 gnorm: 0.86 [14:39:32< 7:21:05] +[titan] 2025-07-10 04:14:37,716 - root - INFO - step: 66605 loss: 16.4527 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9327 +[titan] 2025-07-10 04:14:37,716 - root - INFO - lr: 9.8969e-05 gnorm: 0.87 [14:39:36< 7:21:01] +[titan] 2025-07-10 04:14:41,636 - root - INFO - step: 66610 loss: 16.2952 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.7924 +[titan] 2025-07-10 04:14:41,636 - root - INFO - lr: 9.8951e-05 gnorm: 0.91 [14:39:40< 7:20:57] +[titan] 2025-07-10 04:14:45,592 - root - INFO - step: 66615 loss: 16.3270 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8263 +[titan] 2025-07-10 04:14:45,592 - root - INFO - lr: 9.8932e-05 gnorm: 0.97 [14:39:44< 7:20:53] +[titan] 2025-07-10 04:14:49,526 - root - INFO - step: 66620 loss: 16.1667 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6970 +[titan] 2025-07-10 04:14:49,526 - root - INFO - lr: 9.8913e-05 gnorm: 0.85 [14:39:48< 7:20:49] +[titan] 2025-07-10 04:14:53,465 - root - INFO - step: 66625 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8735 +[titan] 2025-07-10 04:14:53,465 - root - INFO - lr: 9.8895e-05 gnorm: 0.92 [14:39:51< 7:20:45] +[titan] 2025-07-10 04:14:57,366 - root - INFO - step: 66630 loss: 16.2371 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 04:14:57,367 - root - INFO - lr: 9.8876e-05 gnorm: 1.02 [14:39:55< 7:20:41] +[titan] 2025-07-10 04:15:01,290 - root - INFO - step: 66635 loss: 16.2315 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7469 +[titan] 2025-07-10 04:15:01,290 - root - INFO - lr: 9.8857e-05 gnorm: 0.87 [14:39:59< 7:20:37] +[titan] 2025-07-10 04:15:05,197 - root - INFO - step: 66640 loss: 16.5821 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 14.0380 +[titan] 2025-07-10 04:15:05,197 - root - INFO - lr: 9.8839e-05 gnorm: 0.93 [14:40:03< 7:20:33] +[titan] 2025-07-10 04:15:09,095 - root - INFO - step: 66645 loss: 16.4061 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.8927 +[titan] 2025-07-10 04:15:09,095 - root - INFO - lr: 9.8820e-05 gnorm: 0.92 [14:40:07< 7:20:29] +[titan] 2025-07-10 04:15:12,242 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:15:13,035 - root - INFO - step: 66650 loss: 16.3656 memory: 44.58GiB(31.99%) tps: 83,171 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8528 +[titan] 2025-07-10 04:15:13,035 - root - INFO - lr: 9.8801e-05 gnorm: 0.92 [14:40:11< 7:20:25] +[titan] 2025-07-10 04:15:16,943 - root - INFO - step: 66655 loss: 16.5768 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 14.0364 +[titan] 2025-07-10 04:15:16,943 - root - INFO - lr: 9.8783e-05 gnorm: 0.94 [14:40:15< 7:20:21] +[titan] 2025-07-10 04:15:20,879 - root - INFO - step: 66660 loss: 16.0956 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.6350 +[titan] 2025-07-10 04:15:20,880 - root - INFO - lr: 9.8764e-05 gnorm: 0.92 [14:40:19< 7:20:17] +[titan] 2025-07-10 04:15:24,790 - root - INFO - step: 66665 loss: 16.3701 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8640 +[titan] 2025-07-10 04:15:24,790 - root - INFO - lr: 9.8745e-05 gnorm: 0.93 [14:40:23< 7:20:13] +[titan] 2025-07-10 04:15:28,708 - root - INFO - step: 66670 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.8784 +[titan] 2025-07-10 04:15:28,709 - root - INFO - lr: 9.8727e-05 gnorm: 0.94 [14:40:27< 7:20:09] +[titan] 2025-07-10 04:15:32,608 - root - INFO - step: 66675 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8322 +[titan] 2025-07-10 04:15:32,608 - root - INFO - lr: 9.8708e-05 gnorm: 0.84 [14:40:31< 7:20:05] +[titan] 2025-07-10 04:15:36,513 - root - INFO - step: 66680 loss: 16.1098 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.6486 +[titan] 2025-07-10 04:15:36,513 - root - INFO - lr: 9.8689e-05 gnorm: 0.84 [14:40:35< 7:20:01] +[titan] 2025-07-10 04:15:40,415 - root - INFO - step: 66685 loss: 16.4387 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.9201 +[titan] 2025-07-10 04:15:40,415 - root - INFO - lr: 9.8671e-05 gnorm: 0.88 [14:40:38< 7:19:57] +[titan] 2025-07-10 04:15:44,327 - root - INFO - step: 66690 loss: 16.5409 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 14.0025 +[titan] 2025-07-10 04:15:44,327 - root - INFO - lr: 9.8652e-05 gnorm: 0.83 [14:40:42< 7:19:53] +[titan] 2025-07-10 04:15:48,232 - root - INFO - step: 66695 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.6639 +[titan] 2025-07-10 04:15:48,232 - root - INFO - lr: 9.8633e-05 gnorm: 0.90 [14:40:46< 7:19:49] +[titan] 2025-07-10 04:15:51,356 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:15:52,139 - root - INFO - step: 66700 loss: 16.2584 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 04:15:52,139 - root - INFO - lr: 9.8615e-05 gnorm: 0.85 [14:40:50< 7:19:45] +[titan] 2025-07-10 04:15:56,039 - root - INFO - step: 66705 loss: 16.4838 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9478 +[titan] 2025-07-10 04:15:56,040 - root - INFO - lr: 9.8596e-05 gnorm: 0.86 [14:40:54< 7:19:41] +[titan] 2025-07-10 04:15:59,982 - root - INFO - step: 66710 loss: 16.4668 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9462 +[titan] 2025-07-10 04:15:59,982 - root - INFO - lr: 9.8578e-05 gnorm: 0.86 [14:40:58< 7:19:37] +[titan] 2025-07-10 04:16:03,899 - root - INFO - step: 66715 loss: 16.2564 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7764 +[titan] 2025-07-10 04:16:03,900 - root - INFO - lr: 9.8559e-05 gnorm: 0.95 [14:41:02< 7:19:33] +[titan] 2025-07-10 04:16:07,814 - root - INFO - step: 66720 loss: 15.9920 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4393 global_avg_mtp_loss: 13.5527 +[titan] 2025-07-10 04:16:07,814 - root - INFO - lr: 9.8540e-05 gnorm: 0.89 [14:41:06< 7:19:29] +[titan] 2025-07-10 04:16:11,722 - root - INFO - step: 66725 loss: 16.1974 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4703 global_avg_mtp_loss: 13.7271 +[titan] 2025-07-10 04:16:11,723 - root - INFO - lr: 9.8522e-05 gnorm: 0.90 [14:41:10< 7:19:25] +[titan] 2025-07-10 04:16:15,659 - root - INFO - step: 66730 loss: 16.1568 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.6966 +[titan] 2025-07-10 04:16:15,659 - root - INFO - lr: 9.8503e-05 gnorm: 0.88 [14:41:14< 7:19:21] +[titan] 2025-07-10 04:16:19,572 - root - INFO - step: 66735 loss: 16.5818 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 14.0402 +[titan] 2025-07-10 04:16:19,572 - root - INFO - lr: 9.8484e-05 gnorm: 0.89 [14:41:18< 7:19:17] +[titan] 2025-07-10 04:16:23,487 - root - INFO - step: 66740 loss: 16.3482 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8372 +[titan] 2025-07-10 04:16:23,488 - root - INFO - lr: 9.8466e-05 gnorm: 0.86 [14:41:21< 7:19:13] +[titan] 2025-07-10 04:16:27,399 - root - INFO - step: 66745 loss: 16.5327 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5334 global_avg_mtp_loss: 13.9992 +[titan] 2025-07-10 04:16:27,399 - root - INFO - lr: 9.8447e-05 gnorm: 0.89 [14:41:25< 7:19:09] +[titan] 2025-07-10 04:16:30,533 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:16:31,323 - root - INFO - step: 66750 loss: 16.4492 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.9460 +[titan] 2025-07-10 04:16:31,323 - root - INFO - lr: 9.8428e-05 gnorm: 0.90 [14:41:29< 7:19:05] +[titan] 2025-07-10 04:16:35,311 - root - INFO - step: 66755 loss: 16.4196 memory: 44.58GiB(31.99%) tps: 82,188 tflops: 283.65 mfu: 28.68% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.9032 +[titan] 2025-07-10 04:16:35,311 - root - INFO - lr: 9.8410e-05 gnorm: 0.88 [14:41:33< 7:19:01] +[titan] 2025-07-10 04:16:39,255 - root - INFO - step: 66760 loss: 16.4442 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.9232 +[titan] 2025-07-10 04:16:39,255 - root - INFO - lr: 9.8391e-05 gnorm: 0.87 [14:41:37< 7:18:57] +[titan] 2025-07-10 04:16:43,249 - root - INFO - step: 66765 loss: 16.3795 memory: 44.58GiB(31.99%) tps: 82,049 tflops: 283.16 mfu: 28.63% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.8516 +[titan] 2025-07-10 04:16:43,249 - root - INFO - lr: 9.8372e-05 gnorm: 0.96 [14:41:41< 7:18:54] +[titan] 2025-07-10 04:16:47,148 - root - INFO - step: 66770 loss: 16.5262 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.9857 +[titan] 2025-07-10 04:16:47,148 - root - INFO - lr: 9.8354e-05 gnorm: 0.94 [14:41:45< 7:18:50] +[titan] 2025-07-10 04:16:51,061 - root - INFO - step: 66775 loss: 16.6636 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5641 global_avg_mtp_loss: 14.0995 +[titan] 2025-07-10 04:16:51,061 - root - INFO - lr: 9.8335e-05 gnorm: 0.89 [14:41:49< 7:18:46] +[titan] 2025-07-10 04:16:54,974 - root - INFO - step: 66780 loss: 16.0217 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.5730 +[titan] 2025-07-10 04:16:54,974 - root - INFO - lr: 9.8317e-05 gnorm: 0.87 [14:41:53< 7:18:42] +[titan] 2025-07-10 04:16:58,912 - root - INFO - step: 66785 loss: 16.2466 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7614 +[titan] 2025-07-10 04:16:58,912 - root - INFO - lr: 9.8298e-05 gnorm: 0.93 [14:41:57< 7:18:38] +[titan] 2025-07-10 04:17:02,818 - root - INFO - step: 66790 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5225 global_avg_mtp_loss: 13.9573 +[titan] 2025-07-10 04:17:02,818 - root - INFO - lr: 9.8279e-05 gnorm: 0.89 [14:42:01< 7:18:34] +[titan] 2025-07-10 04:17:06,759 - root - INFO - step: 66795 loss: 16.3127 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.8117 +[titan] 2025-07-10 04:17:06,760 - root - INFO - lr: 9.8261e-05 gnorm: 0.92 [14:42:05< 7:18:30] +[titan] 2025-07-10 04:17:09,871 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:17:10,680 - root - INFO - step: 66800 loss: 16.2708 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7785 +[titan] 2025-07-10 04:17:10,680 - root - INFO - lr: 9.8242e-05 gnorm: 1.11 [14:42:09< 7:18:26] +[titan] 2025-07-10 04:17:14,589 - root - INFO - step: 66805 loss: 16.1616 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.6879 +[titan] 2025-07-10 04:17:14,590 - root - INFO - lr: 9.8223e-05 gnorm: 0.89 [14:42:13< 7:18:22] +[titan] 2025-07-10 04:17:18,492 - root - INFO - step: 66810 loss: 16.1366 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.6638 +[titan] 2025-07-10 04:17:18,492 - root - INFO - lr: 9.8205e-05 gnorm: 0.92 [14:42:16< 7:18:18] +[titan] 2025-07-10 04:17:22,421 - root - INFO - step: 66815 loss: 16.4896 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9656 +[titan] 2025-07-10 04:17:22,421 - root - INFO - lr: 9.8186e-05 gnorm: 0.90 [14:42:20< 7:18:14] +[titan] 2025-07-10 04:17:26,376 - root - INFO - step: 66820 loss: 16.2811 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.8012 +[titan] 2025-07-10 04:17:26,376 - root - INFO - lr: 9.8168e-05 gnorm: 0.87 [14:42:24< 7:18:10] +[titan] 2025-07-10 04:17:30,291 - root - INFO - step: 66825 loss: 16.3526 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8505 +[titan] 2025-07-10 04:17:30,292 - root - INFO - lr: 9.8149e-05 gnorm: 0.86 [14:42:28< 7:18:06] +[titan] 2025-07-10 04:17:34,212 - root - INFO - step: 66830 loss: 16.4680 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9399 +[titan] 2025-07-10 04:17:34,213 - root - INFO - lr: 9.8130e-05 gnorm: 0.87 [14:42:32< 7:18:02] +[titan] 2025-07-10 04:17:38,147 - root - INFO - step: 66835 loss: 16.2738 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7866 +[titan] 2025-07-10 04:17:38,147 - root - INFO - lr: 9.8112e-05 gnorm: 0.88 [14:42:36< 7:17:58] +[titan] 2025-07-10 04:17:42,041 - root - INFO - step: 66840 loss: 16.3201 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8144 +[titan] 2025-07-10 04:17:42,041 - root - INFO - lr: 9.8093e-05 gnorm: 0.94 [14:42:40< 7:17:54] +[titan] 2025-07-10 04:17:45,935 - root - INFO - step: 66845 loss: 16.4963 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 13.9509 +[titan] 2025-07-10 04:17:45,935 - root - INFO - lr: 9.8075e-05 gnorm: 0.88 [14:42:44< 7:17:50] +[titan] 2025-07-10 04:17:49,081 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:17:49,882 - root - INFO - step: 66850 loss: 16.4252 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9076 +[titan] 2025-07-10 04:17:49,882 - root - INFO - lr: 9.8056e-05 gnorm: 0.89 [14:42:48< 7:17:46] +[titan] 2025-07-10 04:17:53,798 - root - INFO - step: 66855 loss: 16.3389 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8392 +[titan] 2025-07-10 04:17:53,798 - root - INFO - lr: 9.8037e-05 gnorm: 0.90 [14:42:52< 7:17:42] +[titan] 2025-07-10 04:17:57,742 - root - INFO - step: 66860 loss: 16.3903 memory: 44.58GiB(31.99%) tps: 83,086 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8771 +[titan] 2025-07-10 04:17:57,742 - root - INFO - lr: 9.8019e-05 gnorm: 0.89 [14:42:56< 7:17:38] +[titan] 2025-07-10 04:18:01,699 - root - INFO - step: 66865 loss: 16.2588 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 04:18:01,699 - root - INFO - lr: 9.8000e-05 gnorm: 0.86 [14:43:00< 7:17:34] +[titan] 2025-07-10 04:18:05,626 - root - INFO - step: 66870 loss: 16.3780 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8675 +[titan] 2025-07-10 04:18:05,626 - root - INFO - lr: 9.7982e-05 gnorm: 0.91 [14:43:04< 7:17:30] +[titan] 2025-07-10 04:18:09,547 - root - INFO - step: 66875 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8587 +[titan] 2025-07-10 04:18:09,548 - root - INFO - lr: 9.7963e-05 gnorm: 0.86 [14:43:08< 7:17:26] +[titan] 2025-07-10 04:18:13,467 - root - INFO - step: 66880 loss: 16.4061 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.9032 +[titan] 2025-07-10 04:18:13,467 - root - INFO - lr: 9.7944e-05 gnorm: 0.88 [14:43:11< 7:17:22] +[titan] 2025-07-10 04:18:17,391 - root - INFO - step: 66885 loss: 16.1171 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4638 global_avg_mtp_loss: 13.6534 +[titan] 2025-07-10 04:18:17,391 - root - INFO - lr: 9.7926e-05 gnorm: 0.91 [14:43:15< 7:17:18] +[titan] 2025-07-10 04:18:21,339 - root - INFO - step: 66890 loss: 15.9190 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.4317 global_avg_mtp_loss: 13.4872 +[titan] 2025-07-10 04:18:21,340 - root - INFO - lr: 9.7907e-05 gnorm: 0.90 [14:43:19< 7:17:14] +[titan] 2025-07-10 04:18:25,261 - root - INFO - step: 66895 loss: 16.4147 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.9044 +[titan] 2025-07-10 04:18:25,261 - root - INFO - lr: 9.7889e-05 gnorm: 0.85 [14:43:23< 7:17:10] +[titan] 2025-07-10 04:18:28,394 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:18:29,182 - root - INFO - step: 66900 loss: 16.3778 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8639 +[titan] 2025-07-10 04:18:29,182 - root - INFO - lr: 9.7870e-05 gnorm: 0.90 [14:43:27< 7:17:06] +[titan] 2025-07-10 04:18:33,135 - root - INFO - step: 66905 loss: 16.4095 memory: 44.58GiB(31.99%) tps: 82,900 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.8912 +[titan] 2025-07-10 04:18:33,136 - root - INFO - lr: 9.7852e-05 gnorm: 0.93 [14:43:31< 7:17:02] +[titan] 2025-07-10 04:18:37,085 - root - INFO - step: 66910 loss: 16.3945 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.8809 +[titan] 2025-07-10 04:18:37,085 - root - INFO - lr: 9.7833e-05 gnorm: 0.92 [14:43:35< 7:16:58] +[titan] 2025-07-10 04:18:41,031 - root - INFO - step: 66915 loss: 16.4100 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8877 +[titan] 2025-07-10 04:18:41,032 - root - INFO - lr: 9.7814e-05 gnorm: 0.88 [14:43:39< 7:16:54] +[titan] 2025-07-10 04:18:45,004 - root - INFO - step: 66920 loss: 16.3420 memory: 44.58GiB(31.99%) tps: 82,490 tflops: 284.69 mfu: 28.79% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8403 +[titan] 2025-07-10 04:18:45,004 - root - INFO - lr: 9.7796e-05 gnorm: 0.93 [14:43:43< 7:16:50] +[titan] 2025-07-10 04:18:48,938 - root - INFO - step: 66925 loss: 16.2006 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7307 +[titan] 2025-07-10 04:18:48,939 - root - INFO - lr: 9.7777e-05 gnorm: 0.87 [14:43:47< 7:16:46] +[titan] 2025-07-10 04:18:52,854 - root - INFO - step: 66930 loss: 16.0524 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.5929 +[titan] 2025-07-10 04:18:52,854 - root - INFO - lr: 9.7759e-05 gnorm: 0.93 [14:43:51< 7:16:42] +[titan] 2025-07-10 04:18:56,794 - root - INFO - step: 66935 loss: 16.3287 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.8208 +[titan] 2025-07-10 04:18:56,794 - root - INFO - lr: 9.7740e-05 gnorm: 0.92 [14:43:55< 7:16:38] +[titan] 2025-07-10 04:19:00,720 - root - INFO - step: 66940 loss: 16.6184 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5726 global_avg_mtp_loss: 14.0458 +[titan] 2025-07-10 04:19:00,720 - root - INFO - lr: 9.7721e-05 gnorm: 0.92 [14:43:59< 7:16:34] +[titan] 2025-07-10 04:19:04,639 - root - INFO - step: 66945 loss: 16.3490 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8413 +[titan] 2025-07-10 04:19:04,639 - root - INFO - lr: 9.7703e-05 gnorm: 0.87 [14:44:03< 7:16:30] +[titan] 2025-07-10 04:19:07,751 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:19:08,540 - root - INFO - step: 66950 loss: 16.1124 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6521 +[titan] 2025-07-10 04:19:08,540 - root - INFO - lr: 9.7684e-05 gnorm: 0.90 [14:44:07< 7:16:26] +[titan] 2025-07-10 04:19:12,465 - root - INFO - step: 66955 loss: 16.4318 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.9136 +[titan] 2025-07-10 04:19:12,465 - root - INFO - lr: 9.7666e-05 gnorm: 0.94 [14:44:10< 7:16:22] +[titan] 2025-07-10 04:19:16,408 - root - INFO - step: 66960 loss: 16.1742 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.6933 +[titan] 2025-07-10 04:19:16,409 - root - INFO - lr: 9.7647e-05 gnorm: 0.88 [14:44:14< 7:16:18] +[titan] 2025-07-10 04:19:20,308 - root - INFO - step: 66965 loss: 16.3197 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8248 +[titan] 2025-07-10 04:19:20,308 - root - INFO - lr: 9.7629e-05 gnorm: 0.88 [14:44:18< 7:16:14] +[titan] 2025-07-10 04:19:24,217 - root - INFO - step: 66970 loss: 16.4156 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.9081 +[titan] 2025-07-10 04:19:24,217 - root - INFO - lr: 9.7610e-05 gnorm: 0.89 [14:44:22< 7:16:10] +[titan] 2025-07-10 04:19:28,115 - root - INFO - step: 66975 loss: 16.2299 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7540 +[titan] 2025-07-10 04:19:28,115 - root - INFO - lr: 9.7592e-05 gnorm: 0.93 [14:44:26< 7:16:06] +[titan] 2025-07-10 04:19:32,033 - root - INFO - step: 66980 loss: 16.2724 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.7729 +[titan] 2025-07-10 04:19:32,033 - root - INFO - lr: 9.7573e-05 gnorm: 0.89 [14:44:30< 7:16:02] +[titan] 2025-07-10 04:19:35,941 - root - INFO - step: 66985 loss: 16.1442 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.6740 +[titan] 2025-07-10 04:19:35,941 - root - INFO - lr: 9.7554e-05 gnorm: 0.88 [14:44:34< 7:15:58] +[titan] 2025-07-10 04:19:39,834 - root - INFO - step: 66990 loss: 16.3336 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.8370 +[titan] 2025-07-10 04:19:39,835 - root - INFO - lr: 9.7536e-05 gnorm: 0.99 [14:44:38< 7:15:54] +[titan] 2025-07-10 04:19:43,748 - root - INFO - step: 66995 loss: 16.3977 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8907 +[titan] 2025-07-10 04:19:43,748 - root - INFO - lr: 9.7517e-05 gnorm: 0.97 [14:44:42< 7:15:50] +[titan] 2025-07-10 04:19:46,884 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:19:47,678 - root - INFO - step: 67000 loss: 16.3523 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.8393 +[titan] 2025-07-10 04:19:47,678 - root - INFO - lr: 9.7499e-05 gnorm: 0.92 [14:44:46< 7:15:46] +[titan] 2025-07-10 04:19:51,572 - root - INFO - step: 67005 loss: 16.4225 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.9091 +[titan] 2025-07-10 04:19:51,573 - root - INFO - lr: 9.7480e-05 gnorm: 0.90 [14:44:50< 7:15:42] +[titan] 2025-07-10 04:19:55,542 - root - INFO - step: 67010 loss: 16.4092 memory: 44.58GiB(31.99%) tps: 82,558 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8898 +[titan] 2025-07-10 04:19:55,542 - root - INFO - lr: 9.7462e-05 gnorm: 0.88 [14:44:54< 7:15:38] +[titan] 2025-07-10 04:19:59,514 - root - INFO - step: 67015 loss: 15.9140 memory: 44.58GiB(31.99%) tps: 82,510 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.4399 global_avg_mtp_loss: 13.4741 +[titan] 2025-07-10 04:19:59,514 - root - INFO - lr: 9.7443e-05 gnorm: 0.91 [14:44:57< 7:15:35] +[titan] 2025-07-10 04:20:03,434 - root - INFO - step: 67020 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.9218 +[titan] 2025-07-10 04:20:03,434 - root - INFO - lr: 9.7425e-05 gnorm: 0.92 [14:45:01< 7:15:31] +[titan] 2025-07-10 04:20:07,360 - root - INFO - step: 67025 loss: 16.3838 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8641 +[titan] 2025-07-10 04:20:07,360 - root - INFO - lr: 9.7406e-05 gnorm: 0.93 [14:45:05< 7:15:27] +[titan] 2025-07-10 04:20:11,332 - root - INFO - step: 67030 loss: 16.3575 memory: 44.58GiB(31.99%) tps: 82,495 tflops: 284.70 mfu: 28.79% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8542 +[titan] 2025-07-10 04:20:11,332 - root - INFO - lr: 9.7387e-05 gnorm: 0.92 [14:45:09< 7:15:23] +[titan] 2025-07-10 04:20:15,253 - root - INFO - step: 67035 loss: 16.3007 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.7920 +[titan] 2025-07-10 04:20:15,253 - root - INFO - lr: 9.7369e-05 gnorm: 0.95 [14:45:13< 7:15:19] +[titan] 2025-07-10 04:20:19,208 - root - INFO - step: 67040 loss: 16.5377 memory: 44.58GiB(31.99%) tps: 82,864 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 14.0006 +[titan] 2025-07-10 04:20:19,208 - root - INFO - lr: 9.7350e-05 gnorm: 0.90 [14:45:17< 7:15:15] +[titan] 2025-07-10 04:20:23,124 - root - INFO - step: 67045 loss: 16.2425 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.7469 +[titan] 2025-07-10 04:20:23,124 - root - INFO - lr: 9.7332e-05 gnorm: 0.91 [14:45:21< 7:15:11] +[titan] 2025-07-10 04:20:26,268 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:20:27,104 - root - INFO - step: 67050 loss: 16.3310 memory: 44.58GiB(31.99%) tps: 82,333 tflops: 284.15 mfu: 28.73% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8330 +[titan] 2025-07-10 04:20:27,105 - root - INFO - lr: 9.7313e-05 gnorm: 0.90 [14:45:25< 7:15:07] +[titan] 2025-07-10 04:20:31,013 - root - INFO - step: 67055 loss: 16.8003 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.6059 global_avg_mtp_loss: 14.1944 +[titan] 2025-07-10 04:20:31,014 - root - INFO - lr: 9.7295e-05 gnorm: 0.89 [14:45:29< 7:15:03] +[titan] 2025-07-10 04:20:34,922 - root - INFO - step: 67060 loss: 16.3634 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8583 +[titan] 2025-07-10 04:20:34,922 - root - INFO - lr: 9.7276e-05 gnorm: 0.88 [14:45:33< 7:14:59] +[titan] 2025-07-10 04:20:38,869 - root - INFO - step: 67065 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8243 +[titan] 2025-07-10 04:20:38,869 - root - INFO - lr: 9.7258e-05 gnorm: 0.91 [14:45:37< 7:14:55] +[titan] 2025-07-10 04:20:42,880 - root - INFO - step: 67070 loss: 16.3245 memory: 44.58GiB(31.99%) tps: 81,718 tflops: 282.02 mfu: 28.52% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8243 +[titan] 2025-07-10 04:20:42,880 - root - INFO - lr: 9.7239e-05 gnorm: 0.88 [14:45:41< 7:14:51] +[titan] 2025-07-10 04:20:44,593 - root - INFO - Dumping profiler traces at step 67072 +[titan] 2025-07-10 04:20:44,626 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:20:47,004 - root - INFO - step: 67075 loss: 16.2946 memory: 44.58GiB(31.99%) tps: 79,464 tflops: 274.24 mfu: 27.73% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.7989 +[titan] 2025-07-10 04:20:47,004 - root - INFO - lr: 9.7221e-05 gnorm: 0.89 [14:45:45< 7:14:47] +[titan] 2025-07-10 04:20:50,914 - root - INFO - step: 67080 loss: 16.3630 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8522 +[titan] 2025-07-10 04:20:50,915 - root - INFO - lr: 9.7202e-05 gnorm: 0.90 [14:45:49< 7:14:43] +[titan] 2025-07-10 04:20:54,824 - root - INFO - step: 67085 loss: 16.0837 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6156 +[titan] 2025-07-10 04:20:54,824 - root - INFO - lr: 9.7184e-05 gnorm: 0.91 [14:45:53< 7:14:39] +[titan] 2025-07-10 04:20:58,739 - root - INFO - step: 67090 loss: 16.3408 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 04:20:58,739 - root - INFO - lr: 9.7165e-05 gnorm: 0.90 [14:45:57< 7:14:35] +[titan] 2025-07-10 04:21:02,653 - root - INFO - step: 67095 loss: 16.2458 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7549 +[titan] 2025-07-10 04:21:02,653 - root - INFO - lr: 9.7147e-05 gnorm: 0.85 [14:46:01< 7:14:31] +[titan] 2025-07-10 04:21:05,797 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:21:06,589 - root - INFO - step: 67100 loss: 16.6583 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5592 global_avg_mtp_loss: 14.0990 +[titan] 2025-07-10 04:21:06,589 - root - INFO - lr: 9.7128e-05 gnorm: 0.89 [14:46:05< 7:14:27] +[titan] 2025-07-10 04:21:10,544 - root - INFO - step: 67105 loss: 15.9593 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.4403 global_avg_mtp_loss: 13.5190 +[titan] 2025-07-10 04:21:10,544 - root - INFO - lr: 9.7110e-05 gnorm: 0.90 [14:46:09< 7:14:23] +[titan] 2025-07-10 04:21:14,458 - root - INFO - step: 67110 loss: 16.3859 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.8614 +[titan] 2025-07-10 04:21:14,458 - root - INFO - lr: 9.7091e-05 gnorm: 0.89 [14:46:12< 7:14:19] +[titan] 2025-07-10 04:21:18,398 - root - INFO - step: 67115 loss: 16.3070 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8019 +[titan] 2025-07-10 04:21:18,398 - root - INFO - lr: 9.7073e-05 gnorm: 0.88 [14:46:16< 7:14:15] +[titan] 2025-07-10 04:21:22,318 - root - INFO - step: 67120 loss: 16.1054 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.6385 +[titan] 2025-07-10 04:21:22,318 - root - INFO - lr: 9.7054e-05 gnorm: 0.91 [14:46:20< 7:14:11] +[titan] 2025-07-10 04:21:26,225 - root - INFO - step: 67125 loss: 16.1048 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.6375 +[titan] 2025-07-10 04:21:26,226 - root - INFO - lr: 9.7036e-05 gnorm: 0.88 [14:46:24< 7:14:07] +[titan] 2025-07-10 04:21:30,155 - root - INFO - step: 67130 loss: 16.2586 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7636 +[titan] 2025-07-10 04:21:30,156 - root - INFO - lr: 9.7017e-05 gnorm: 0.92 [14:46:28< 7:14:03] +[titan] 2025-07-10 04:21:34,067 - root - INFO - step: 67135 loss: 16.3527 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8465 +[titan] 2025-07-10 04:21:34,068 - root - INFO - lr: 9.6999e-05 gnorm: 0.97 [14:46:32< 7:13:59] +[titan] 2025-07-10 04:21:38,009 - root - INFO - step: 67140 loss: 16.3856 memory: 44.58GiB(31.99%) tps: 83,140 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.8540 +[titan] 2025-07-10 04:21:38,009 - root - INFO - lr: 9.6980e-05 gnorm: 0.92 [14:46:36< 7:13:55] +[titan] 2025-07-10 04:21:41,971 - root - INFO - step: 67145 loss: 16.3734 memory: 44.58GiB(31.99%) tps: 82,710 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8765 +[titan] 2025-07-10 04:21:41,972 - root - INFO - lr: 9.6962e-05 gnorm: 0.90 [14:46:40< 7:13:51] +[titan] 2025-07-10 04:21:45,083 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:21:45,882 - root - INFO - step: 67150 loss: 16.3466 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-10 04:21:45,882 - root - INFO - lr: 9.6943e-05 gnorm: 0.86 [14:46:44< 7:13:47] +[titan] 2025-07-10 04:21:49,820 - root - INFO - step: 67155 loss: 16.2627 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.7675 +[titan] 2025-07-10 04:21:49,820 - root - INFO - lr: 9.6925e-05 gnorm: 0.87 [14:46:48< 7:13:43] +[titan] 2025-07-10 04:21:53,721 - root - INFO - step: 67160 loss: 16.5620 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 14.0204 +[titan] 2025-07-10 04:21:53,721 - root - INFO - lr: 9.6906e-05 gnorm: 0.88 [14:46:52< 7:13:39] +[titan] 2025-07-10 04:21:57,644 - root - INFO - step: 67165 loss: 16.2503 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7669 +[titan] 2025-07-10 04:21:57,644 - root - INFO - lr: 9.6888e-05 gnorm: 0.93 [14:46:56< 7:13:35] +[titan] 2025-07-10 04:22:01,553 - root - INFO - step: 67170 loss: 16.4366 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 04:22:01,554 - root - INFO - lr: 9.6869e-05 gnorm: 0.90 [14:47:00< 7:13:31] +[titan] 2025-07-10 04:22:05,477 - root - INFO - step: 67175 loss: 16.1534 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.6812 +[titan] 2025-07-10 04:22:05,477 - root - INFO - lr: 9.6851e-05 gnorm: 0.94 [14:47:03< 7:13:27] +[titan] 2025-07-10 04:22:09,518 - root - INFO - step: 67180 loss: 16.6506 memory: 44.58GiB(31.99%) tps: 81,090 tflops: 279.85 mfu: 28.30% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-10 04:22:09,518 - root - INFO - lr: 9.6832e-05 gnorm: 0.93 [14:47:07< 7:13:23] +[titan] 2025-07-10 04:22:13,433 - root - INFO - step: 67185 loss: 16.4750 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-10 04:22:13,434 - root - INFO - lr: 9.6814e-05 gnorm: 0.95 [14:47:11< 7:13:19] +[titan] 2025-07-10 04:22:17,379 - root - INFO - step: 67190 loss: 16.2866 memory: 44.58GiB(31.99%) tps: 83,049 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.7853 +[titan] 2025-07-10 04:22:17,380 - root - INFO - lr: 9.6795e-05 gnorm: 0.98 [14:47:15< 7:13:15] +[titan] 2025-07-10 04:22:21,295 - root - INFO - step: 67195 loss: 16.2717 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.7802 +[titan] 2025-07-10 04:22:21,295 - root - INFO - lr: 9.6777e-05 gnorm: 0.93 [14:47:19< 7:13:11] +[titan] 2025-07-10 04:22:24,453 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:22:25,260 - root - INFO - step: 67200 loss: 16.4724 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9459 +[titan] 2025-07-10 04:22:25,260 - root - INFO - lr: 9.6758e-05 gnorm: 0.93 [14:47:23< 7:13:08] +[titan] 2025-07-10 04:22:29,217 - root - INFO - step: 67205 loss: 16.4458 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.9168 +[titan] 2025-07-10 04:22:29,218 - root - INFO - lr: 9.6740e-05 gnorm: 0.86 [14:47:27< 7:13:04] +[titan] 2025-07-10 04:22:33,136 - root - INFO - step: 67210 loss: 16.4630 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9385 +[titan] 2025-07-10 04:22:33,136 - root - INFO - lr: 9.6721e-05 gnorm: 0.91 [14:47:31< 7:13:00] +[titan] 2025-07-10 04:22:37,085 - root - INFO - step: 67215 loss: 16.4418 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9289 +[titan] 2025-07-10 04:22:37,085 - root - INFO - lr: 9.6703e-05 gnorm: 0.87 [14:47:35< 7:12:56] +[titan] 2025-07-10 04:22:40,992 - root - INFO - step: 67220 loss: 16.5959 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5575 global_avg_mtp_loss: 14.0384 +[titan] 2025-07-10 04:22:40,993 - root - INFO - lr: 9.6684e-05 gnorm: 0.87 [14:47:39< 7:12:52] +[titan] 2025-07-10 04:22:44,974 - root - INFO - step: 67225 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 82,314 tflops: 284.08 mfu: 28.72% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 04:22:44,974 - root - INFO - lr: 9.6666e-05 gnorm: 0.90 [14:47:43< 7:12:48] +[titan] 2025-07-10 04:22:48,904 - root - INFO - step: 67230 loss: 16.4317 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9110 +[titan] 2025-07-10 04:22:48,904 - root - INFO - lr: 9.6647e-05 gnorm: 0.94 [14:47:47< 7:12:44] +[titan] 2025-07-10 04:22:52,816 - root - INFO - step: 67235 loss: 16.4917 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9531 +[titan] 2025-07-10 04:22:52,816 - root - INFO - lr: 9.6629e-05 gnorm: 0.86 [14:47:51< 7:12:40] +[titan] 2025-07-10 04:22:56,776 - root - INFO - step: 67240 loss: 16.2804 memory: 44.58GiB(31.99%) tps: 82,737 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 04:22:56,777 - root - INFO - lr: 9.6610e-05 gnorm: 0.94 [14:47:55< 7:12:36] +[titan] 2025-07-10 04:23:00,716 - root - INFO - step: 67245 loss: 16.2469 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 04:23:00,716 - root - INFO - lr: 9.6592e-05 gnorm: 0.88 [14:47:59< 7:12:32] +[titan] 2025-07-10 04:23:03,852 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:23:04,644 - root - INFO - step: 67250 loss: 16.4001 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.8791 +[titan] 2025-07-10 04:23:04,644 - root - INFO - lr: 9.6573e-05 gnorm: 0.89 [14:48:03< 7:12:28] +[titan] 2025-07-10 04:23:08,573 - root - INFO - step: 67255 loss: 16.3041 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8060 +[titan] 2025-07-10 04:23:08,574 - root - INFO - lr: 9.6555e-05 gnorm: 0.90 [14:48:07< 7:12:24] +[titan] 2025-07-10 04:23:12,535 - root - INFO - step: 67260 loss: 16.2420 memory: 44.58GiB(31.99%) tps: 82,725 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7668 +[titan] 2025-07-10 04:23:12,535 - root - INFO - lr: 9.6536e-05 gnorm: 0.89 [14:48:10< 7:12:20] +[titan] 2025-07-10 04:23:16,452 - root - INFO - step: 67265 loss: 16.3381 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.8409 +[titan] 2025-07-10 04:23:16,452 - root - INFO - lr: 9.6518e-05 gnorm: 0.87 [14:48:14< 7:12:16] +[titan] 2025-07-10 04:23:20,357 - root - INFO - step: 67270 loss: 16.3185 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8245 +[titan] 2025-07-10 04:23:20,357 - root - INFO - lr: 9.6500e-05 gnorm: 0.87 [14:48:18< 7:12:12] +[titan] 2025-07-10 04:23:24,267 - root - INFO - step: 67275 loss: 16.4052 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8913 +[titan] 2025-07-10 04:23:24,267 - root - INFO - lr: 9.6481e-05 gnorm: 0.91 [14:48:22< 7:12:08] +[titan] 2025-07-10 04:23:28,174 - root - INFO - step: 67280 loss: 16.5078 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5300 global_avg_mtp_loss: 13.9778 +[titan] 2025-07-10 04:23:28,175 - root - INFO - lr: 9.6463e-05 gnorm: 0.93 [14:48:26< 7:12:04] +[titan] 2025-07-10 04:23:32,072 - root - INFO - step: 67285 loss: 16.4739 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 13.9349 +[titan] 2025-07-10 04:23:32,072 - root - INFO - lr: 9.6444e-05 gnorm: 0.92 [14:48:30< 7:12:00] +[titan] 2025-07-10 04:23:36,011 - root - INFO - step: 67290 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.9328 +[titan] 2025-07-10 04:23:36,011 - root - INFO - lr: 9.6426e-05 gnorm: 0.94 [14:48:34< 7:11:56] +[titan] 2025-07-10 04:23:39,935 - root - INFO - step: 67295 loss: 16.5201 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9792 +[titan] 2025-07-10 04:23:39,936 - root - INFO - lr: 9.6407e-05 gnorm: 0.92 [14:48:38< 7:11:52] +[titan] 2025-07-10 04:23:43,054 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:23:43,840 - root - INFO - step: 67300 loss: 16.3735 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8671 +[titan] 2025-07-10 04:23:43,840 - root - INFO - lr: 9.6389e-05 gnorm: 0.93 [14:48:42< 7:11:48] +[titan] 2025-07-10 04:23:47,738 - root - INFO - step: 67305 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-10 04:23:47,738 - root - INFO - lr: 9.6370e-05 gnorm: 0.90 [14:48:46< 7:11:44] +[titan] 2025-07-10 04:23:51,680 - root - INFO - step: 67310 loss: 16.5067 memory: 44.58GiB(31.99%) tps: 83,127 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5294 global_avg_mtp_loss: 13.9773 +[titan] 2025-07-10 04:23:51,680 - root - INFO - lr: 9.6352e-05 gnorm: 0.88 [14:48:50< 7:11:40] +[titan] 2025-07-10 04:23:55,598 - root - INFO - step: 67315 loss: 16.4408 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-10 04:23:55,598 - root - INFO - lr: 9.6334e-05 gnorm: 0.92 [14:48:54< 7:11:36] +[titan] 2025-07-10 04:23:59,516 - root - INFO - step: 67320 loss: 16.4620 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.9329 +[titan] 2025-07-10 04:23:59,516 - root - INFO - lr: 9.6315e-05 gnorm: 0.92 [14:48:57< 7:11:32] +[titan] 2025-07-10 04:24:03,433 - root - INFO - step: 67325 loss: 16.6313 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5526 global_avg_mtp_loss: 14.0788 +[titan] 2025-07-10 04:24:03,433 - root - INFO - lr: 9.6297e-05 gnorm: 0.88 [14:49:01< 7:11:28] +[titan] 2025-07-10 04:24:07,344 - root - INFO - step: 67330 loss: 16.5246 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 13.9822 +[titan] 2025-07-10 04:24:07,345 - root - INFO - lr: 9.6278e-05 gnorm: 0.89 [14:49:05< 7:11:24] +[titan] 2025-07-10 04:24:11,243 - root - INFO - step: 67335 loss: 16.4950 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9610 +[titan] 2025-07-10 04:24:11,243 - root - INFO - lr: 9.6260e-05 gnorm: 0.91 [14:49:09< 7:11:20] +[titan] 2025-07-10 04:24:15,157 - root - INFO - step: 67340 loss: 16.2401 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.7438 +[titan] 2025-07-10 04:24:15,158 - root - INFO - lr: 9.6241e-05 gnorm: 0.90 [14:49:13< 7:11:16] +[titan] 2025-07-10 04:24:19,070 - root - INFO - step: 67345 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9552 +[titan] 2025-07-10 04:24:19,070 - root - INFO - lr: 9.6223e-05 gnorm: 0.89 [14:49:17< 7:11:12] +[titan] 2025-07-10 04:24:22,217 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:24:23,007 - root - INFO - step: 67350 loss: 16.5581 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0108 +[titan] 2025-07-10 04:24:23,007 - root - INFO - lr: 9.6204e-05 gnorm: 0.91 [14:49:21< 7:11:08] +[titan] 2025-07-10 04:24:26,927 - root - INFO - step: 67355 loss: 16.3367 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.8343 +[titan] 2025-07-10 04:24:26,927 - root - INFO - lr: 9.6186e-05 gnorm: 0.89 [14:49:25< 7:11:04] +[titan] 2025-07-10 04:24:30,858 - root - INFO - step: 67360 loss: 16.2485 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7697 +[titan] 2025-07-10 04:24:30,859 - root - INFO - lr: 9.6168e-05 gnorm: 0.91 [14:49:29< 7:11:00] +[titan] 2025-07-10 04:24:34,775 - root - INFO - step: 67365 loss: 16.4936 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5450 global_avg_mtp_loss: 13.9486 +[titan] 2025-07-10 04:24:34,775 - root - INFO - lr: 9.6149e-05 gnorm: 0.91 [14:49:33< 7:10:56] +[titan] 2025-07-10 04:24:38,695 - root - INFO - step: 67370 loss: 16.3080 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8134 +[titan] 2025-07-10 04:24:38,696 - root - INFO - lr: 9.6131e-05 gnorm: 0.89 [14:49:37< 7:10:52] +[titan] 2025-07-10 04:24:42,619 - root - INFO - step: 67375 loss: 16.4807 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9469 +[titan] 2025-07-10 04:24:42,619 - root - INFO - lr: 9.6112e-05 gnorm: 0.95 [14:49:41< 7:10:48] +[titan] 2025-07-10 04:24:46,543 - root - INFO - step: 67380 loss: 16.2780 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7897 +[titan] 2025-07-10 04:24:46,543 - root - INFO - lr: 9.6094e-05 gnorm: 0.91 [14:49:44< 7:10:44] +[titan] 2025-07-10 04:24:50,466 - root - INFO - step: 67385 loss: 16.5467 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 14.0200 +[titan] 2025-07-10 04:24:50,466 - root - INFO - lr: 9.6075e-05 gnorm: 0.84 [14:49:48< 7:10:40] +[titan] 2025-07-10 04:24:54,373 - root - INFO - step: 67390 loss: 16.1768 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7000 +[titan] 2025-07-10 04:24:54,373 - root - INFO - lr: 9.6057e-05 gnorm: 0.91 [14:49:52< 7:10:36] +[titan] 2025-07-10 04:24:58,288 - root - INFO - step: 67395 loss: 16.6244 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.0546 +[titan] 2025-07-10 04:24:58,289 - root - INFO - lr: 9.6039e-05 gnorm: 0.90 [14:49:56< 7:10:32] +[titan] 2025-07-10 04:25:01,415 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:25:02,203 - root - INFO - step: 67400 loss: 16.7835 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5902 global_avg_mtp_loss: 14.1933 +[titan] 2025-07-10 04:25:02,203 - root - INFO - lr: 9.6020e-05 gnorm: 0.91 [14:50:00< 7:10:28] +[titan] 2025-07-10 04:25:06,102 - root - INFO - step: 67405 loss: 16.2230 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7327 +[titan] 2025-07-10 04:25:06,102 - root - INFO - lr: 9.6002e-05 gnorm: 0.91 [14:50:04< 7:10:24] +[titan] 2025-07-10 04:25:10,036 - root - INFO - step: 67410 loss: 16.2146 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 04:25:10,036 - root - INFO - lr: 9.5983e-05 gnorm: 0.87 [14:50:08< 7:10:20] +[titan] 2025-07-10 04:25:13,970 - root - INFO - step: 67415 loss: 16.4754 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.9497 +[titan] 2025-07-10 04:25:13,970 - root - INFO - lr: 9.5965e-05 gnorm: 0.91 [14:50:12< 7:10:16] +[titan] 2025-07-10 04:25:17,886 - root - INFO - step: 67420 loss: 16.2926 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.8008 +[titan] 2025-07-10 04:25:17,886 - root - INFO - lr: 9.5947e-05 gnorm: 0.90 [14:50:16< 7:10:12] +[titan] 2025-07-10 04:25:21,827 - root - INFO - step: 67425 loss: 16.3950 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8934 +[titan] 2025-07-10 04:25:21,827 - root - INFO - lr: 9.5928e-05 gnorm: 0.88 [14:50:20< 7:10:08] +[titan] 2025-07-10 04:25:25,756 - root - INFO - step: 67430 loss: 16.4062 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.8964 +[titan] 2025-07-10 04:25:25,757 - root - INFO - lr: 9.5910e-05 gnorm: 0.89 [14:50:24< 7:10:04] +[titan] 2025-07-10 04:25:29,711 - root - INFO - step: 67435 loss: 16.3624 memory: 44.58GiB(31.99%) tps: 82,866 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.8638 +[titan] 2025-07-10 04:25:29,711 - root - INFO - lr: 9.5891e-05 gnorm: 0.89 [14:50:28< 7:10:00] +[titan] 2025-07-10 04:25:33,622 - root - INFO - step: 67440 loss: 16.4821 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.9683 +[titan] 2025-07-10 04:25:33,622 - root - INFO - lr: 9.5873e-05 gnorm: 0.92 [14:50:32< 7:09:56] +[titan] 2025-07-10 04:25:37,540 - root - INFO - step: 67445 loss: 16.3399 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.8400 +[titan] 2025-07-10 04:25:37,540 - root - INFO - lr: 9.5855e-05 gnorm: 0.85 [14:50:35< 7:09:52] +[titan] 2025-07-10 04:25:40,680 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:25:41,470 - root - INFO - step: 67450 loss: 16.5243 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9904 +[titan] 2025-07-10 04:25:41,470 - root - INFO - lr: 9.5836e-05 gnorm: 0.92 [14:50:39< 7:09:49] +[titan] 2025-07-10 04:25:45,409 - root - INFO - step: 67455 loss: 16.0277 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4383 global_avg_mtp_loss: 13.5894 +[titan] 2025-07-10 04:25:45,410 - root - INFO - lr: 9.5818e-05 gnorm: 0.93 [14:50:43< 7:09:45] +[titan] 2025-07-10 04:25:49,323 - root - INFO - step: 67460 loss: 16.2948 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7995 +[titan] 2025-07-10 04:25:49,323 - root - INFO - lr: 9.5799e-05 gnorm: 0.88 [14:50:47< 7:09:41] +[titan] 2025-07-10 04:25:53,238 - root - INFO - step: 67465 loss: 16.6001 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 14.0540 +[titan] 2025-07-10 04:25:53,238 - root - INFO - lr: 9.5781e-05 gnorm: 0.86 [14:50:51< 7:09:37] +[titan] 2025-07-10 04:25:57,173 - root - INFO - step: 67470 loss: 16.3794 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8646 +[titan] 2025-07-10 04:25:57,173 - root - INFO - lr: 9.5763e-05 gnorm: 0.89 [14:50:55< 7:09:33] +[titan] 2025-07-10 04:26:01,083 - root - INFO - step: 67475 loss: 16.1958 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7207 +[titan] 2025-07-10 04:26:01,083 - root - INFO - lr: 9.5744e-05 gnorm: 0.94 [14:50:59< 7:09:29] +[titan] 2025-07-10 04:26:05,000 - root - INFO - step: 67480 loss: 16.2688 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7908 +[titan] 2025-07-10 04:26:05,001 - root - INFO - lr: 9.5726e-05 gnorm: 0.90 [14:51:03< 7:09:25] +[titan] 2025-07-10 04:26:08,925 - root - INFO - step: 67485 loss: 16.4097 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8873 +[titan] 2025-07-10 04:26:08,925 - root - INFO - lr: 9.5707e-05 gnorm: 0.93 [14:51:07< 7:09:21] +[titan] 2025-07-10 04:26:12,827 - root - INFO - step: 67490 loss: 16.2288 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7416 +[titan] 2025-07-10 04:26:12,827 - root - INFO - lr: 9.5689e-05 gnorm: 0.90 [14:51:11< 7:09:17] +[titan] 2025-07-10 04:26:16,739 - root - INFO - step: 67495 loss: 16.6128 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5530 global_avg_mtp_loss: 14.0598 +[titan] 2025-07-10 04:26:16,739 - root - INFO - lr: 9.5671e-05 gnorm: 0.90 [14:51:15< 7:09:13] +[titan] 2025-07-10 04:26:19,850 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:26:20,639 - root - INFO - step: 67500 loss: 16.3292 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5007 global_avg_mtp_loss: 13.8284 +[titan] 2025-07-10 04:26:20,639 - root - INFO - lr: 9.5652e-05 gnorm: 0.89 [14:51:19< 7:09:09] +[titan] 2025-07-10 04:26:24,536 - root - INFO - step: 67505 loss: 16.4203 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.9049 +[titan] 2025-07-10 04:26:24,536 - root - INFO - lr: 9.5634e-05 gnorm: 0.85 [14:51:22< 7:09:05] +[titan] 2025-07-10 04:26:28,465 - root - INFO - step: 67510 loss: 16.3194 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.8221 +[titan] 2025-07-10 04:26:28,465 - root - INFO - lr: 9.5616e-05 gnorm: 0.96 [14:51:26< 7:09:01] +[titan] 2025-07-10 04:26:32,388 - root - INFO - step: 67515 loss: 16.4366 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9084 +[titan] 2025-07-10 04:26:32,389 - root - INFO - lr: 9.5597e-05 gnorm: 0.88 [14:51:30< 7:08:57] +[titan] 2025-07-10 04:26:36,301 - root - INFO - step: 67520 loss: 16.2325 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.7403 +[titan] 2025-07-10 04:26:36,301 - root - INFO - lr: 9.5579e-05 gnorm: 0.96 [14:51:34< 7:08:53] +[titan] 2025-07-10 04:26:40,224 - root - INFO - step: 67525 loss: 16.2512 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7577 +[titan] 2025-07-10 04:26:40,224 - root - INFO - lr: 9.5560e-05 gnorm: 0.88 [14:51:38< 7:08:49] +[titan] 2025-07-10 04:26:44,163 - root - INFO - step: 67530 loss: 16.7018 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5785 global_avg_mtp_loss: 14.1233 +[titan] 2025-07-10 04:26:44,164 - root - INFO - lr: 9.5542e-05 gnorm: 0.89 [14:51:42< 7:08:45] +[titan] 2025-07-10 04:26:48,074 - root - INFO - step: 67535 loss: 16.2594 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.7560 +[titan] 2025-07-10 04:26:48,074 - root - INFO - lr: 9.5524e-05 gnorm: 0.89 [14:51:46< 7:08:41] +[titan] 2025-07-10 04:26:52,004 - root - INFO - step: 67540 loss: 16.5465 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5370 global_avg_mtp_loss: 14.0095 +[titan] 2025-07-10 04:26:52,004 - root - INFO - lr: 9.5505e-05 gnorm: 0.93 [14:51:50< 7:08:37] +[titan] 2025-07-10 04:26:55,959 - root - INFO - step: 67545 loss: 16.2090 memory: 44.58GiB(31.99%) tps: 82,864 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7365 +[titan] 2025-07-10 04:26:55,959 - root - INFO - lr: 9.5487e-05 gnorm: 0.89 [14:51:54< 7:08:33] +[titan] 2025-07-10 04:26:59,092 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:26:59,885 - root - INFO - step: 67550 loss: 16.3806 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8717 +[titan] 2025-07-10 04:26:59,885 - root - INFO - lr: 9.5469e-05 gnorm: 0.97 [14:51:58< 7:08:29] +[titan] 2025-07-10 04:27:03,846 - root - INFO - step: 67555 loss: 16.5684 memory: 44.58GiB(31.99%) tps: 82,733 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0115 +[titan] 2025-07-10 04:27:03,846 - root - INFO - lr: 9.5450e-05 gnorm: 0.96 [14:52:02< 7:08:25] +[titan] 2025-07-10 04:27:07,740 - root - INFO - step: 67560 loss: 16.3289 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8343 +[titan] 2025-07-10 04:27:07,741 - root - INFO - lr: 9.5432e-05 gnorm: 0.88 [14:52:06< 7:08:21] +[titan] 2025-07-10 04:27:11,650 - root - INFO - step: 67565 loss: 16.4114 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.8740 +[titan] 2025-07-10 04:27:11,650 - root - INFO - lr: 9.5414e-05 gnorm: 0.94 [14:52:10< 7:08:17] +[titan] 2025-07-10 04:27:15,546 - root - INFO - step: 67570 loss: 16.4698 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9383 +[titan] 2025-07-10 04:27:15,546 - root - INFO - lr: 9.5395e-05 gnorm: 0.87 [14:52:13< 7:08:13] +[titan] 2025-07-10 04:27:19,455 - root - INFO - step: 67575 loss: 16.2416 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7598 +[titan] 2025-07-10 04:27:19,456 - root - INFO - lr: 9.5377e-05 gnorm: 0.93 [14:52:17< 7:08:09] +[titan] 2025-07-10 04:27:23,378 - root - INFO - step: 67580 loss: 16.0525 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6085 +[titan] 2025-07-10 04:27:23,378 - root - INFO - lr: 9.5359e-05 gnorm: 0.99 [14:52:21< 7:08:05] +[titan] 2025-07-10 04:27:26,751 - root - INFO - Dumping profiler traces at step 67584 +[titan] 2025-07-10 04:27:26,784 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:27:27,583 - root - INFO - step: 67585 loss: 16.4492 memory: 44.58GiB(31.99%) tps: 77,926 tflops: 268.94 mfu: 27.19% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.9230 +[titan] 2025-07-10 04:27:27,583 - root - INFO - lr: 9.5340e-05 gnorm: 0.93 [14:52:26< 7:08:01] +[titan] 2025-07-10 04:27:31,512 - root - INFO - step: 67590 loss: 16.9299 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.6485 global_avg_mtp_loss: 14.2813 +[titan] 2025-07-10 04:27:31,512 - root - INFO - lr: 9.5322e-05 gnorm: 0.95 [14:52:29< 7:07:57] +[titan] 2025-07-10 04:27:35,424 - root - INFO - step: 67595 loss: 16.0888 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6238 +[titan] 2025-07-10 04:27:35,425 - root - INFO - lr: 9.5303e-05 gnorm: 0.89 [14:52:33< 7:07:53] +[titan] 2025-07-10 04:27:38,563 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:27:39,367 - root - INFO - step: 67600 loss: 16.2555 memory: 44.58GiB(31.99%) tps: 83,126 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7761 +[titan] 2025-07-10 04:27:39,367 - root - INFO - lr: 9.5285e-05 gnorm: 0.90 [14:52:37< 7:07:49] +[titan] 2025-07-10 04:27:43,267 - root - INFO - step: 67605 loss: 16.4953 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9719 +[titan] 2025-07-10 04:27:43,267 - root - INFO - lr: 9.5267e-05 gnorm: 0.93 [14:52:41< 7:07:45] +[titan] 2025-07-10 04:27:47,170 - root - INFO - step: 67610 loss: 16.5451 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 13.9956 +[titan] 2025-07-10 04:27:47,170 - root - INFO - lr: 9.5248e-05 gnorm: 0.88 [14:52:45< 7:07:41] +[titan] 2025-07-10 04:27:51,077 - root - INFO - step: 67615 loss: 16.1954 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.7139 +[titan] 2025-07-10 04:27:51,078 - root - INFO - lr: 9.5230e-05 gnorm: 0.93 [14:52:49< 7:07:37] +[titan] 2025-07-10 04:27:54,977 - root - INFO - step: 67620 loss: 16.4628 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9323 +[titan] 2025-07-10 04:27:54,977 - root - INFO - lr: 9.5212e-05 gnorm: 0.87 [14:52:53< 7:07:33] +[titan] 2025-07-10 04:27:58,942 - root - INFO - step: 67625 loss: 16.5998 memory: 44.58GiB(31.99%) tps: 82,656 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0516 +[titan] 2025-07-10 04:27:58,942 - root - INFO - lr: 9.5193e-05 gnorm: 0.90 [14:52:57< 7:07:29] +[titan] 2025-07-10 04:28:02,862 - root - INFO - step: 67630 loss: 16.2585 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7633 +[titan] 2025-07-10 04:28:02,862 - root - INFO - lr: 9.5175e-05 gnorm: 0.89 [14:53:01< 7:07:25] +[titan] 2025-07-10 04:28:06,783 - root - INFO - step: 67635 loss: 16.2349 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.7524 +[titan] 2025-07-10 04:28:06,783 - root - INFO - lr: 9.5157e-05 gnorm: 0.86 [14:53:05< 7:07:21] +[titan] 2025-07-10 04:28:10,688 - root - INFO - step: 67640 loss: 16.4953 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9616 +[titan] 2025-07-10 04:28:10,688 - root - INFO - lr: 9.5138e-05 gnorm: 0.91 [14:53:09< 7:07:17] +[titan] 2025-07-10 04:28:14,635 - root - INFO - step: 67645 loss: 16.2351 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7510 +[titan] 2025-07-10 04:28:14,636 - root - INFO - lr: 9.5120e-05 gnorm: 0.87 [14:53:13< 7:07:13] +[titan] 2025-07-10 04:28:17,746 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:28:18,535 - root - INFO - step: 67650 loss: 16.5267 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9985 +[titan] 2025-07-10 04:28:18,536 - root - INFO - lr: 9.5102e-05 gnorm: 0.92 [14:53:16< 7:07:09] +[titan] 2025-07-10 04:28:22,441 - root - INFO - step: 67655 loss: 16.5634 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 14.0255 +[titan] 2025-07-10 04:28:22,441 - root - INFO - lr: 9.5083e-05 gnorm: 0.91 [14:53:20< 7:07:05] +[titan] 2025-07-10 04:28:26,348 - root - INFO - step: 67660 loss: 16.6768 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5622 global_avg_mtp_loss: 14.1146 +[titan] 2025-07-10 04:28:26,349 - root - INFO - lr: 9.5065e-05 gnorm: 0.95 [14:53:24< 7:07:01] +[titan] 2025-07-10 04:28:30,247 - root - INFO - step: 67665 loss: 16.4690 memory: 44.58GiB(31.99%) tps: 84,052 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9469 +[titan] 2025-07-10 04:28:30,248 - root - INFO - lr: 9.5047e-05 gnorm: 1.03 [14:53:28< 7:06:57] +[titan] 2025-07-10 04:28:34,164 - root - INFO - step: 67670 loss: 16.2076 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7271 +[titan] 2025-07-10 04:28:34,164 - root - INFO - lr: 9.5029e-05 gnorm: 0.93 [14:53:32< 7:06:53] +[titan] 2025-07-10 04:28:38,089 - root - INFO - step: 67675 loss: 16.4026 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8828 +[titan] 2025-07-10 04:28:38,089 - root - INFO - lr: 9.5010e-05 gnorm: 0.85 [14:53:36< 7:06:49] +[titan] 2025-07-10 04:28:42,029 - root - INFO - step: 67680 loss: 16.3513 memory: 44.58GiB(31.99%) tps: 83,171 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8435 +[titan] 2025-07-10 04:28:42,029 - root - INFO - lr: 9.4992e-05 gnorm: 0.89 [14:53:40< 7:06:45] +[titan] 2025-07-10 04:28:45,986 - root - INFO - step: 67685 loss: 16.0463 memory: 44.58GiB(31.99%) tps: 82,823 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4656 global_avg_mtp_loss: 13.5807 +[titan] 2025-07-10 04:28:45,986 - root - INFO - lr: 9.4974e-05 gnorm: 0.90 [14:53:44< 7:06:42] +[titan] 2025-07-10 04:28:49,920 - root - INFO - step: 67690 loss: 16.3329 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8336 +[titan] 2025-07-10 04:28:49,920 - root - INFO - lr: 9.4955e-05 gnorm: 0.92 [14:53:48< 7:06:38] +[titan] 2025-07-10 04:28:53,853 - root - INFO - step: 67695 loss: 16.1044 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4646 global_avg_mtp_loss: 13.6398 +[titan] 2025-07-10 04:28:53,853 - root - INFO - lr: 9.4937e-05 gnorm: 0.93 [14:53:52< 7:06:34] +[titan] 2025-07-10 04:28:56,979 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:28:57,767 - root - INFO - step: 67700 loss: 16.3644 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.8527 +[titan] 2025-07-10 04:28:57,767 - root - INFO - lr: 9.4919e-05 gnorm: 0.89 [14:53:56< 7:06:30] +[titan] 2025-07-10 04:29:01,673 - root - INFO - step: 67705 loss: 16.3159 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.8262 +[titan] 2025-07-10 04:29:01,674 - root - INFO - lr: 9.4900e-05 gnorm: 0.88 [14:54:00< 7:06:26] +[titan] 2025-07-10 04:29:05,592 - root - INFO - step: 67710 loss: 16.6254 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0720 +[titan] 2025-07-10 04:29:05,592 - root - INFO - lr: 9.4882e-05 gnorm: 0.88 [14:54:04< 7:06:22] +[titan] 2025-07-10 04:29:09,510 - root - INFO - step: 67715 loss: 16.2509 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4638 global_avg_mtp_loss: 13.7871 +[titan] 2025-07-10 04:29:09,510 - root - INFO - lr: 9.4864e-05 gnorm: 0.91 [14:54:07< 7:06:18] +[titan] 2025-07-10 04:29:13,433 - root - INFO - step: 67720 loss: 16.2759 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7834 +[titan] 2025-07-10 04:29:13,433 - root - INFO - lr: 9.4845e-05 gnorm: 0.97 [14:54:11< 7:06:14] +[titan] 2025-07-10 04:29:17,361 - root - INFO - step: 67725 loss: 16.3634 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8548 +[titan] 2025-07-10 04:29:17,361 - root - INFO - lr: 9.4827e-05 gnorm: 0.91 [14:54:15< 7:06:10] +[titan] 2025-07-10 04:29:21,317 - root - INFO - step: 67730 loss: 16.1611 memory: 44.58GiB(31.99%) tps: 82,845 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.6830 +[titan] 2025-07-10 04:29:21,317 - root - INFO - lr: 9.4809e-05 gnorm: 0.91 [14:54:19< 7:06:06] +[titan] 2025-07-10 04:29:25,270 - root - INFO - step: 67735 loss: 16.3841 memory: 44.58GiB(31.99%) tps: 82,898 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8745 +[titan] 2025-07-10 04:29:25,270 - root - INFO - lr: 9.4791e-05 gnorm: 0.87 [14:54:23< 7:06:02] +[titan] 2025-07-10 04:29:29,229 - root - INFO - step: 67740 loss: 16.5832 memory: 44.58GiB(31.99%) tps: 82,781 tflops: 285.69 mfu: 28.89% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 14.0442 +[titan] 2025-07-10 04:29:29,229 - root - INFO - lr: 9.4772e-05 gnorm: 0.89 [14:54:27< 7:05:58] +[titan] 2025-07-10 04:29:33,147 - root - INFO - step: 67745 loss: 16.4143 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.8906 +[titan] 2025-07-10 04:29:33,147 - root - INFO - lr: 9.4754e-05 gnorm: 0.92 [14:54:31< 7:05:54] +[titan] 2025-07-10 04:29:36,261 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:29:37,053 - root - INFO - step: 67750 loss: 16.5567 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5431 global_avg_mtp_loss: 14.0137 +[titan] 2025-07-10 04:29:37,053 - root - INFO - lr: 9.4736e-05 gnorm: 0.90 [14:54:35< 7:05:50] +[titan] 2025-07-10 04:29:40,999 - root - INFO - step: 67755 loss: 16.4446 memory: 44.58GiB(31.99%) tps: 83,043 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5269 global_avg_mtp_loss: 13.9177 +[titan] 2025-07-10 04:29:40,999 - root - INFO - lr: 9.4717e-05 gnorm: 0.91 [14:54:39< 7:05:46] +[titan] 2025-07-10 04:29:44,951 - root - INFO - step: 67760 loss: 16.3978 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.8698 +[titan] 2025-07-10 04:29:44,951 - root - INFO - lr: 9.4699e-05 gnorm: 0.91 [14:54:43< 7:05:42] +[titan] 2025-07-10 04:29:48,879 - root - INFO - step: 67765 loss: 16.2712 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7862 +[titan] 2025-07-10 04:29:48,880 - root - INFO - lr: 9.4681e-05 gnorm: 0.88 [14:54:47< 7:05:38] +[titan] 2025-07-10 04:29:52,799 - root - INFO - step: 67770 loss: 16.3325 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8244 +[titan] 2025-07-10 04:29:52,799 - root - INFO - lr: 9.4663e-05 gnorm: 0.97 [14:54:51< 7:05:34] +[titan] 2025-07-10 04:29:56,764 - root - INFO - step: 67775 loss: 16.4937 memory: 44.58GiB(31.99%) tps: 82,651 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9631 +[titan] 2025-07-10 04:29:56,764 - root - INFO - lr: 9.4644e-05 gnorm: 0.91 [14:54:55< 7:05:30] +[titan] 2025-07-10 04:30:00,690 - root - INFO - step: 67780 loss: 16.3467 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-10 04:30:00,690 - root - INFO - lr: 9.4626e-05 gnorm: 0.90 [14:54:59< 7:05:26] +[titan] 2025-07-10 04:30:04,620 - root - INFO - step: 67785 loss: 16.1557 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.6778 +[titan] 2025-07-10 04:30:04,620 - root - INFO - lr: 9.4608e-05 gnorm: 0.94 [14:55:03< 7:05:22] +[titan] 2025-07-10 04:30:08,544 - root - INFO - step: 67790 loss: 16.1122 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6500 +[titan] 2025-07-10 04:30:08,544 - root - INFO - lr: 9.4589e-05 gnorm: 0.93 [14:55:06< 7:05:18] +[titan] 2025-07-10 04:30:12,484 - root - INFO - step: 67795 loss: 16.3293 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.8285 +[titan] 2025-07-10 04:30:12,484 - root - INFO - lr: 9.4571e-05 gnorm: 0.91 [14:55:10< 7:05:14] +[titan] 2025-07-10 04:30:15,632 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:30:16,418 - root - INFO - step: 67800 loss: 16.2010 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7184 +[titan] 2025-07-10 04:30:16,418 - root - INFO - lr: 9.4553e-05 gnorm: 0.94 [14:55:14< 7:05:10] +[titan] 2025-07-10 04:30:20,319 - root - INFO - step: 67805 loss: 16.3317 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8322 +[titan] 2025-07-10 04:30:20,319 - root - INFO - lr: 9.4535e-05 gnorm: 0.95 [14:55:18< 7:05:06] +[titan] 2025-07-10 04:30:24,234 - root - INFO - step: 67810 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5398 global_avg_mtp_loss: 13.9730 +[titan] 2025-07-10 04:30:24,235 - root - INFO - lr: 9.4516e-05 gnorm: 0.93 [14:55:22< 7:05:02] +[titan] 2025-07-10 04:30:28,156 - root - INFO - step: 67815 loss: 16.3439 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8405 +[titan] 2025-07-10 04:30:28,156 - root - INFO - lr: 9.4498e-05 gnorm: 0.89 [14:55:26< 7:04:58] +[titan] 2025-07-10 04:30:32,067 - root - INFO - step: 67820 loss: 16.2731 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7770 +[titan] 2025-07-10 04:30:32,067 - root - INFO - lr: 9.4480e-05 gnorm: 0.89 [14:55:30< 7:04:54] +[titan] 2025-07-10 04:30:35,967 - root - INFO - step: 67825 loss: 16.6548 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0961 +[titan] 2025-07-10 04:30:35,968 - root - INFO - lr: 9.4462e-05 gnorm: 0.91 [14:55:34< 7:04:50] +[titan] 2025-07-10 04:30:39,913 - root - INFO - step: 67830 loss: 16.3395 memory: 44.58GiB(31.99%) tps: 83,063 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8376 +[titan] 2025-07-10 04:30:39,913 - root - INFO - lr: 9.4443e-05 gnorm: 0.92 [14:55:38< 7:04:46] +[titan] 2025-07-10 04:30:43,836 - root - INFO - step: 67835 loss: 16.2335 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.7431 +[titan] 2025-07-10 04:30:43,837 - root - INFO - lr: 9.4425e-05 gnorm: 0.98 [14:55:42< 7:04:42] +[titan] 2025-07-10 04:30:47,756 - root - INFO - step: 67840 loss: 16.4720 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9485 +[titan] 2025-07-10 04:30:47,756 - root - INFO - lr: 9.4407e-05 gnorm: 0.90 [14:55:46< 7:04:38] +[titan] 2025-07-10 04:30:51,657 - root - INFO - step: 67845 loss: 16.5409 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 13.9986 +[titan] 2025-07-10 04:30:51,658 - root - INFO - lr: 9.4388e-05 gnorm: 0.94 [14:55:50< 7:04:34] +[titan] 2025-07-10 04:30:54,811 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:30:55,598 - root - INFO - step: 67850 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.8108 +[titan] 2025-07-10 04:30:55,598 - root - INFO - lr: 9.4370e-05 gnorm: 0.89 [14:55:53< 7:04:30] +[titan] 2025-07-10 04:30:59,551 - root - INFO - step: 67855 loss: 16.4404 memory: 44.58GiB(31.99%) tps: 82,896 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.9312 +[titan] 2025-07-10 04:30:59,552 - root - INFO - lr: 9.4352e-05 gnorm: 0.91 [14:55:57< 7:04:26] +[titan] 2025-07-10 04:31:03,497 - root - INFO - step: 67860 loss: 16.1986 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7051 +[titan] 2025-07-10 04:31:03,498 - root - INFO - lr: 9.4334e-05 gnorm: 0.88 [14:56:01< 7:04:22] +[titan] 2025-07-10 04:31:07,422 - root - INFO - step: 67865 loss: 16.6257 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0678 +[titan] 2025-07-10 04:31:07,422 - root - INFO - lr: 9.4315e-05 gnorm: 0.88 [14:56:05< 7:04:18] +[titan] 2025-07-10 04:31:11,331 - root - INFO - step: 67870 loss: 16.3627 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-10 04:31:11,331 - root - INFO - lr: 9.4297e-05 gnorm: 0.93 [14:56:09< 7:04:14] +[titan] 2025-07-10 04:31:15,241 - root - INFO - step: 67875 loss: 16.3968 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8922 +[titan] 2025-07-10 04:31:15,241 - root - INFO - lr: 9.4279e-05 gnorm: 0.89 [14:56:13< 7:04:10] +[titan] 2025-07-10 04:31:19,165 - root - INFO - step: 67880 loss: 16.3099 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.7993 +[titan] 2025-07-10 04:31:19,165 - root - INFO - lr: 9.4261e-05 gnorm: 0.87 [14:56:17< 7:04:06] +[titan] 2025-07-10 04:31:23,071 - root - INFO - step: 67885 loss: 16.4683 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5299 global_avg_mtp_loss: 13.9384 +[titan] 2025-07-10 04:31:23,071 - root - INFO - lr: 9.4242e-05 gnorm: 0.99 [14:56:21< 7:04:02] +[titan] 2025-07-10 04:31:26,987 - root - INFO - step: 67890 loss: 16.0467 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.5950 +[titan] 2025-07-10 04:31:26,987 - root - INFO - lr: 9.4224e-05 gnorm: 0.96 [14:56:25< 7:03:58] +[titan] 2025-07-10 04:31:30,901 - root - INFO - step: 67895 loss: 16.4159 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.8965 +[titan] 2025-07-10 04:31:30,901 - root - INFO - lr: 9.4206e-05 gnorm: 0.94 [14:56:29< 7:03:54] +[titan] 2025-07-10 04:31:34,027 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:31:34,856 - root - INFO - step: 67900 loss: 16.5246 memory: 44.58GiB(31.99%) tps: 82,866 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 13.9769 +[titan] 2025-07-10 04:31:34,856 - root - INFO - lr: 9.4188e-05 gnorm: 0.93 [14:56:33< 7:03:50] +[titan] 2025-07-10 04:31:38,820 - root - INFO - step: 67905 loss: 16.3487 memory: 44.58GiB(31.99%) tps: 82,664 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.5015 global_avg_mtp_loss: 13.8473 +[titan] 2025-07-10 04:31:38,820 - root - INFO - lr: 9.4170e-05 gnorm: 0.88 [14:56:37< 7:03:47] +[titan] 2025-07-10 04:31:42,734 - root - INFO - step: 67910 loss: 16.5813 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 14.0411 +[titan] 2025-07-10 04:31:42,735 - root - INFO - lr: 9.4151e-05 gnorm: 0.93 [14:56:41< 7:03:43] +[titan] 2025-07-10 04:31:46,654 - root - INFO - step: 67915 loss: 16.1916 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7051 +[titan] 2025-07-10 04:31:46,654 - root - INFO - lr: 9.4133e-05 gnorm: 0.88 [14:56:45< 7:03:39] +[titan] 2025-07-10 04:31:50,571 - root - INFO - step: 67920 loss: 16.3035 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.8106 +[titan] 2025-07-10 04:31:50,571 - root - INFO - lr: 9.4115e-05 gnorm: 0.94 [14:56:48< 7:03:35] +[titan] 2025-07-10 04:31:54,515 - root - INFO - step: 67925 loss: 16.3773 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8749 +[titan] 2025-07-10 04:31:54,515 - root - INFO - lr: 9.4097e-05 gnorm: 0.91 [14:56:52< 7:03:31] +[titan] 2025-07-10 04:31:58,463 - root - INFO - step: 67930 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 04:31:58,463 - root - INFO - lr: 9.4078e-05 gnorm: 0.89 [14:56:56< 7:03:27] +[titan] 2025-07-10 04:32:02,372 - root - INFO - step: 67935 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8424 +[titan] 2025-07-10 04:32:02,372 - root - INFO - lr: 9.4060e-05 gnorm: 0.90 [14:57:00< 7:03:23] +[titan] 2025-07-10 04:32:06,349 - root - INFO - step: 67940 loss: 16.2154 memory: 44.58GiB(31.99%) tps: 82,408 tflops: 284.40 mfu: 28.76% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7417 +[titan] 2025-07-10 04:32:06,349 - root - INFO - lr: 9.4042e-05 gnorm: 0.89 [14:57:04< 7:03:19] +[titan] 2025-07-10 04:32:10,265 - root - INFO - step: 67945 loss: 16.2731 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7854 +[titan] 2025-07-10 04:32:10,265 - root - INFO - lr: 9.4024e-05 gnorm: 0.86 [14:57:08< 7:03:15] +[titan] 2025-07-10 04:32:13,381 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:32:14,170 - root - INFO - step: 67950 loss: 16.4443 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.9189 +[titan] 2025-07-10 04:32:14,170 - root - INFO - lr: 9.4005e-05 gnorm: 0.89 [14:57:12< 7:03:11] +[titan] 2025-07-10 04:32:18,098 - root - INFO - step: 67955 loss: 15.9957 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.5437 +[titan] 2025-07-10 04:32:18,098 - root - INFO - lr: 9.3987e-05 gnorm: 0.93 [14:57:16< 7:03:07] +[titan] 2025-07-10 04:32:22,019 - root - INFO - step: 67960 loss: 16.5149 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9784 +[titan] 2025-07-10 04:32:22,019 - root - INFO - lr: 9.3969e-05 gnorm: 0.90 [14:57:20< 7:03:03] +[titan] 2025-07-10 04:32:25,929 - root - INFO - step: 67965 loss: 16.1617 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.6922 +[titan] 2025-07-10 04:32:25,929 - root - INFO - lr: 9.3951e-05 gnorm: 0.92 [14:57:24< 7:02:59] +[titan] 2025-07-10 04:32:29,835 - root - INFO - step: 67970 loss: 16.4805 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9491 +[titan] 2025-07-10 04:32:29,836 - root - INFO - lr: 9.3933e-05 gnorm: 0.94 [14:57:28< 7:02:55] +[titan] 2025-07-10 04:32:33,755 - root - INFO - step: 67975 loss: 16.4279 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.9108 +[titan] 2025-07-10 04:32:33,755 - root - INFO - lr: 9.3914e-05 gnorm: 0.90 [14:57:32< 7:02:51] +[titan] 2025-07-10 04:32:37,666 - root - INFO - step: 67980 loss: 16.5232 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5409 global_avg_mtp_loss: 13.9823 +[titan] 2025-07-10 04:32:37,666 - root - INFO - lr: 9.3896e-05 gnorm: 0.92 [14:57:36< 7:02:47] +[titan] 2025-07-10 04:32:41,578 - root - INFO - step: 67985 loss: 16.5592 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5509 global_avg_mtp_loss: 14.0084 +[titan] 2025-07-10 04:32:41,578 - root - INFO - lr: 9.3878e-05 gnorm: 0.97 [14:57:39< 7:02:43] +[titan] 2025-07-10 04:32:45,494 - root - INFO - step: 67990 loss: 16.3685 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8599 +[titan] 2025-07-10 04:32:45,494 - root - INFO - lr: 9.3860e-05 gnorm: 0.87 [14:57:43< 7:02:39] +[titan] 2025-07-10 04:32:49,432 - root - INFO - step: 67995 loss: 16.2238 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7451 +[titan] 2025-07-10 04:32:49,433 - root - INFO - lr: 9.3842e-05 gnorm: 0.89 [14:57:47< 7:02:35] +[titan] 2025-07-10 04:32:52,558 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:32:53,354 - root - INFO - step: 68000 loss: 16.2083 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7378 +[titan] 2025-07-10 04:32:53,354 - root - INFO - lr: 9.3823e-05 gnorm: 0.89 [14:57:51< 7:02:31] +[titan] 2025-07-10 04:32:57,270 - root - INFO - step: 68005 loss: 16.1494 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4679 global_avg_mtp_loss: 13.6815 +[titan] 2025-07-10 04:32:57,271 - root - INFO - lr: 9.3805e-05 gnorm: 0.93 [14:57:55< 7:02:27] +[titan] 2025-07-10 04:33:01,171 - root - INFO - step: 68010 loss: 16.8854 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.6170 global_avg_mtp_loss: 14.2684 +[titan] 2025-07-10 04:33:01,172 - root - INFO - lr: 9.3787e-05 gnorm: 0.94 [14:57:59< 7:02:23] +[titan] 2025-07-10 04:33:05,088 - root - INFO - step: 68015 loss: 16.2862 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.7917 +[titan] 2025-07-10 04:33:05,088 - root - INFO - lr: 9.3769e-05 gnorm: 0.94 [14:58:03< 7:02:19] +[titan] 2025-07-10 04:33:09,017 - root - INFO - step: 68020 loss: 16.3999 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8954 +[titan] 2025-07-10 04:33:09,017 - root - INFO - lr: 9.3751e-05 gnorm: 0.87 [14:58:07< 7:02:15] +[titan] 2025-07-10 04:33:12,971 - root - INFO - step: 68025 loss: 16.2623 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7707 +[titan] 2025-07-10 04:33:12,971 - root - INFO - lr: 9.3732e-05 gnorm: 0.89 [14:58:11< 7:02:11] +[titan] 2025-07-10 04:33:16,944 - root - INFO - step: 68030 loss: 16.4321 memory: 44.58GiB(31.99%) tps: 82,482 tflops: 284.66 mfu: 28.78% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9089 +[titan] 2025-07-10 04:33:16,944 - root - INFO - lr: 9.3714e-05 gnorm: 0.90 [14:58:15< 7:02:07] +[titan] 2025-07-10 04:33:20,851 - root - INFO - step: 68035 loss: 16.5499 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 14.0160 +[titan] 2025-07-10 04:33:20,851 - root - INFO - lr: 9.3696e-05 gnorm: 0.92 [14:58:19< 7:02:03] +[titan] 2025-07-10 04:33:24,765 - root - INFO - step: 68040 loss: 16.2309 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7460 +[titan] 2025-07-10 04:33:24,765 - root - INFO - lr: 9.3678e-05 gnorm: 0.90 [14:58:23< 7:01:59] +[titan] 2025-07-10 04:33:28,677 - root - INFO - step: 68045 loss: 16.0494 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.5879 +[titan] 2025-07-10 04:33:28,678 - root - INFO - lr: 9.3660e-05 gnorm: 1.02 [14:58:27< 7:01:55] +[titan] 2025-07-10 04:33:31,809 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:33:32,601 - root - INFO - step: 68050 loss: 16.4831 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9552 +[titan] 2025-07-10 04:33:32,602 - root - INFO - lr: 9.3641e-05 gnorm: 0.84 [14:58:30< 7:01:51] +[titan] 2025-07-10 04:33:36,523 - root - INFO - step: 68055 loss: 16.2580 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.7691 +[titan] 2025-07-10 04:33:36,523 - root - INFO - lr: 9.3623e-05 gnorm: 0.94 [14:58:34< 7:01:47] +[titan] 2025-07-10 04:33:40,458 - root - INFO - step: 68060 loss: 16.4793 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.9463 +[titan] 2025-07-10 04:33:40,459 - root - INFO - lr: 9.3605e-05 gnorm: 0.89 [14:58:38< 7:01:43] +[titan] 2025-07-10 04:33:44,360 - root - INFO - step: 68065 loss: 16.2416 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 04:33:44,360 - root - INFO - lr: 9.3587e-05 gnorm: 0.96 [14:58:42< 7:01:39] +[titan] 2025-07-10 04:33:48,276 - root - INFO - step: 68070 loss: 16.5050 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9707 +[titan] 2025-07-10 04:33:48,276 - root - INFO - lr: 9.3569e-05 gnorm: 0.91 [14:58:46< 7:01:35] +[titan] 2025-07-10 04:33:52,194 - root - INFO - step: 68075 loss: 16.4979 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.9706 +[titan] 2025-07-10 04:33:52,194 - root - INFO - lr: 9.3551e-05 gnorm: 0.91 [14:58:50< 7:01:31] +[titan] 2025-07-10 04:33:56,090 - root - INFO - step: 68080 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.9225 +[titan] 2025-07-10 04:33:56,090 - root - INFO - lr: 9.3532e-05 gnorm: 0.87 [14:58:54< 7:01:27] +[titan] 2025-07-10 04:34:00,010 - root - INFO - step: 68085 loss: 16.2826 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.7805 +[titan] 2025-07-10 04:34:00,011 - root - INFO - lr: 9.3514e-05 gnorm: 0.86 [14:58:58< 7:01:23] +[titan] 2025-07-10 04:34:03,901 - root - INFO - step: 68090 loss: 16.4821 memory: 44.58GiB(31.99%) tps: 84,222 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9583 +[titan] 2025-07-10 04:34:03,902 - root - INFO - lr: 9.3496e-05 gnorm: 1.09 [14:59:02< 7:01:19] +[titan] 2025-07-10 04:34:07,890 - root - INFO - step: 68095 loss: 16.5515 memory: 44.58GiB(31.99%) tps: 82,167 tflops: 283.57 mfu: 28.67% global_avg_ntp_loss: 2.5420 global_avg_mtp_loss: 14.0094 +[titan] 2025-07-10 04:34:07,890 - root - INFO - lr: 9.3478e-05 gnorm: 0.91 [14:59:06< 7:01:15] +[titan] 2025-07-10 04:34:08,823 - root - INFO - Dumping profiler traces at step 68096 +[titan] 2025-07-10 04:34:08,854 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:34:11,204 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:34:11,998 - root - INFO - step: 68100 loss: 16.1398 memory: 44.58GiB(31.99%) tps: 79,778 tflops: 275.33 mfu: 27.84% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.6657 +[titan] 2025-07-10 04:34:11,998 - root - INFO - lr: 9.3460e-05 gnorm: 0.94 [14:59:10< 7:01:11] +[titan] 2025-07-10 04:34:15,888 - root - INFO - step: 68105 loss: 16.4005 memory: 44.58GiB(31.99%) tps: 84,233 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 04:34:15,888 - root - INFO - lr: 9.3442e-05 gnorm: 0.90 [14:59:14< 7:01:07] +[titan] 2025-07-10 04:34:19,806 - root - INFO - step: 68110 loss: 16.3760 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8552 +[titan] 2025-07-10 04:34:19,806 - root - INFO - lr: 9.3423e-05 gnorm: 0.86 [14:59:18< 7:01:03] +[titan] 2025-07-10 04:34:23,713 - root - INFO - step: 68115 loss: 16.3119 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.7994 +[titan] 2025-07-10 04:34:23,713 - root - INFO - lr: 9.3405e-05 gnorm: 0.93 [14:59:22< 7:00:59] +[titan] 2025-07-10 04:34:27,621 - root - INFO - step: 68120 loss: 16.5220 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9905 +[titan] 2025-07-10 04:34:27,621 - root - INFO - lr: 9.3387e-05 gnorm: 0.99 [14:59:25< 7:00:55] +[titan] 2025-07-10 04:34:31,518 - root - INFO - step: 68125 loss: 16.1771 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7064 +[titan] 2025-07-10 04:34:31,518 - root - INFO - lr: 9.3369e-05 gnorm: 0.92 [14:59:29< 7:00:51] +[titan] 2025-07-10 04:34:35,433 - root - INFO - step: 68130 loss: 16.4006 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8931 +[titan] 2025-07-10 04:34:35,433 - root - INFO - lr: 9.3351e-05 gnorm: 0.92 [14:59:33< 7:00:47] +[titan] 2025-07-10 04:34:39,373 - root - INFO - step: 68135 loss: 16.3829 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5120 global_avg_mtp_loss: 13.8709 +[titan] 2025-07-10 04:34:39,373 - root - INFO - lr: 9.3333e-05 gnorm: 0.88 [14:59:37< 7:00:44] +[titan] 2025-07-10 04:34:43,297 - root - INFO - step: 68140 loss: 16.5406 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 13.9950 +[titan] 2025-07-10 04:34:43,298 - root - INFO - lr: 9.3314e-05 gnorm: 0.92 [14:59:41< 7:00:40] +[titan] 2025-07-10 04:34:47,217 - root - INFO - step: 68145 loss: 16.3878 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-10 04:34:47,218 - root - INFO - lr: 9.3296e-05 gnorm: 0.94 [14:59:45< 7:00:36] +[titan] 2025-07-10 04:34:50,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:34:51,164 - root - INFO - step: 68150 loss: 16.2869 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7934 +[titan] 2025-07-10 04:34:51,164 - root - INFO - lr: 9.3278e-05 gnorm: 0.92 [14:59:49< 7:00:32] +[titan] 2025-07-10 04:34:55,063 - root - INFO - step: 68155 loss: 16.0904 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6384 +[titan] 2025-07-10 04:34:55,063 - root - INFO - lr: 9.3260e-05 gnorm: 0.94 [14:59:53< 7:00:28] +[titan] 2025-07-10 04:34:59,006 - root - INFO - step: 68160 loss: 16.3918 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.8658 +[titan] 2025-07-10 04:34:59,006 - root - INFO - lr: 9.3242e-05 gnorm: 0.89 [14:59:57< 7:00:24] +[titan] 2025-07-10 04:35:02,933 - root - INFO - step: 68165 loss: 16.4547 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9220 +[titan] 2025-07-10 04:35:02,933 - root - INFO - lr: 9.3224e-05 gnorm: 0.87 [15:00:01< 7:00:20] +[titan] 2025-07-10 04:35:06,831 - root - INFO - step: 68170 loss: 16.5033 memory: 44.58GiB(31.99%) tps: 84,084 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-10 04:35:06,831 - root - INFO - lr: 9.3206e-05 gnorm: 0.98 [15:00:05< 7:00:16] +[titan] 2025-07-10 04:35:10,756 - root - INFO - step: 68175 loss: 16.3757 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.8558 +[titan] 2025-07-10 04:35:10,756 - root - INFO - lr: 9.3187e-05 gnorm: 0.88 [15:00:09< 7:00:12] +[titan] 2025-07-10 04:35:14,677 - root - INFO - step: 68180 loss: 16.4722 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.9575 +[titan] 2025-07-10 04:35:14,678 - root - INFO - lr: 9.3169e-05 gnorm: 0.89 [15:00:13< 7:00:08] +[titan] 2025-07-10 04:35:18,577 - root - INFO - step: 68185 loss: 16.4884 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9544 +[titan] 2025-07-10 04:35:18,577 - root - INFO - lr: 9.3151e-05 gnorm: 1.00 [15:00:16< 7:00:04] +[titan] 2025-07-10 04:35:22,532 - root - INFO - step: 68190 loss: 16.2086 memory: 44.58GiB(31.99%) tps: 82,858 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7333 +[titan] 2025-07-10 04:35:22,532 - root - INFO - lr: 9.3133e-05 gnorm: 1.02 [15:00:20< 7:00:00] +[titan] 2025-07-10 04:35:26,436 - root - INFO - step: 68195 loss: 16.5023 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 13.9719 +[titan] 2025-07-10 04:35:26,436 - root - INFO - lr: 9.3115e-05 gnorm: 0.92 [15:00:24< 6:59:56] +[titan] 2025-07-10 04:35:29,548 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:35:30,340 - root - INFO - step: 68200 loss: 16.4513 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9345 +[titan] 2025-07-10 04:35:30,340 - root - INFO - lr: 9.3097e-05 gnorm: 0.91 [15:00:28< 6:59:52] +[titan] 2025-07-10 04:35:34,237 - root - INFO - step: 68205 loss: 16.6044 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0479 +[titan] 2025-07-10 04:35:34,237 - root - INFO - lr: 9.3079e-05 gnorm: 0.94 [15:00:32< 6:59:48] +[titan] 2025-07-10 04:35:38,133 - root - INFO - step: 68210 loss: 16.5652 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 14.0401 +[titan] 2025-07-10 04:35:38,133 - root - INFO - lr: 9.3061e-05 gnorm: 0.95 [15:00:36< 6:59:44] +[titan] 2025-07-10 04:35:42,041 - root - INFO - step: 68215 loss: 16.2432 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7652 +[titan] 2025-07-10 04:35:42,042 - root - INFO - lr: 9.3042e-05 gnorm: 0.92 [15:00:40< 6:59:40] +[titan] 2025-07-10 04:35:45,960 - root - INFO - step: 68220 loss: 16.1947 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7123 +[titan] 2025-07-10 04:35:45,961 - root - INFO - lr: 9.3024e-05 gnorm: 0.90 [15:00:44< 6:59:36] +[titan] 2025-07-10 04:35:49,874 - root - INFO - step: 68225 loss: 16.2728 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7893 +[titan] 2025-07-10 04:35:49,874 - root - INFO - lr: 9.3006e-05 gnorm: 0.92 [15:00:48< 6:59:32] +[titan] 2025-07-10 04:35:53,773 - root - INFO - step: 68230 loss: 16.5487 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 14.0074 +[titan] 2025-07-10 04:35:53,774 - root - INFO - lr: 9.2988e-05 gnorm: 0.97 [15:00:52< 6:59:28] +[titan] 2025-07-10 04:35:57,686 - root - INFO - step: 68235 loss: 16.3912 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.8769 +[titan] 2025-07-10 04:35:57,686 - root - INFO - lr: 9.2970e-05 gnorm: 0.92 [15:00:56< 6:59:24] +[titan] 2025-07-10 04:36:01,599 - root - INFO - step: 68240 loss: 16.1816 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7107 +[titan] 2025-07-10 04:36:01,599 - root - INFO - lr: 9.2952e-05 gnorm: 0.97 [15:00:59< 6:59:20] +[titan] 2025-07-10 04:36:05,508 - root - INFO - step: 68245 loss: 16.1492 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4626 global_avg_mtp_loss: 13.6866 +[titan] 2025-07-10 04:36:05,508 - root - INFO - lr: 9.2934e-05 gnorm: 0.90 [15:01:03< 6:59:16] +[titan] 2025-07-10 04:36:08,634 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:36:09,429 - root - INFO - step: 68250 loss: 16.2966 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8058 +[titan] 2025-07-10 04:36:09,429 - root - INFO - lr: 9.2916e-05 gnorm: 0.86 [15:01:07< 6:59:12] +[titan] 2025-07-10 04:36:13,334 - root - INFO - step: 68255 loss: 16.2277 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.7644 +[titan] 2025-07-10 04:36:13,335 - root - INFO - lr: 9.2897e-05 gnorm: 0.92 [15:01:11< 6:59:08] +[titan] 2025-07-10 04:36:17,251 - root - INFO - step: 68260 loss: 16.1744 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7089 +[titan] 2025-07-10 04:36:17,251 - root - INFO - lr: 9.2879e-05 gnorm: 0.85 [15:01:15< 6:59:04] +[titan] 2025-07-10 04:36:21,153 - root - INFO - step: 68265 loss: 16.1738 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.6870 +[titan] 2025-07-10 04:36:21,153 - root - INFO - lr: 9.2861e-05 gnorm: 0.91 [15:01:19< 6:59:00] +[titan] 2025-07-10 04:36:25,059 - root - INFO - step: 68270 loss: 16.2970 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 04:36:25,060 - root - INFO - lr: 9.2843e-05 gnorm: 0.95 [15:01:23< 6:58:56] +[titan] 2025-07-10 04:36:28,970 - root - INFO - step: 68275 loss: 16.1458 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.6821 +[titan] 2025-07-10 04:36:28,970 - root - INFO - lr: 9.2825e-05 gnorm: 0.94 [15:01:27< 6:58:52] +[titan] 2025-07-10 04:36:32,884 - root - INFO - step: 68280 loss: 15.8861 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4179 global_avg_mtp_loss: 13.4682 +[titan] 2025-07-10 04:36:32,884 - root - INFO - lr: 9.2807e-05 gnorm: 0.99 [15:01:31< 6:58:48] +[titan] 2025-07-10 04:36:36,777 - root - INFO - step: 68285 loss: 16.4338 memory: 44.58GiB(31.99%) tps: 84,172 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 04:36:36,778 - root - INFO - lr: 9.2789e-05 gnorm: 0.93 [15:01:35< 6:58:44] +[titan] 2025-07-10 04:36:40,697 - root - INFO - step: 68290 loss: 16.3297 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 04:36:40,697 - root - INFO - lr: 9.2771e-05 gnorm: 0.92 [15:01:39< 6:58:40] +[titan] 2025-07-10 04:36:44,617 - root - INFO - step: 68295 loss: 16.2835 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.7870 +[titan] 2025-07-10 04:36:44,617 - root - INFO - lr: 9.2753e-05 gnorm: 0.87 [15:01:42< 6:58:36] +[titan] 2025-07-10 04:36:47,726 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:36:48,516 - root - INFO - step: 68300 loss: 15.8470 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4209 global_avg_mtp_loss: 13.4261 +[titan] 2025-07-10 04:36:48,516 - root - INFO - lr: 9.2735e-05 gnorm: 0.93 [15:01:46< 6:58:32] +[titan] 2025-07-10 04:36:52,429 - root - INFO - step: 68305 loss: 16.3874 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8716 +[titan] 2025-07-10 04:36:52,430 - root - INFO - lr: 9.2716e-05 gnorm: 0.98 [15:01:50< 6:58:28] +[titan] 2025-07-10 04:36:56,365 - root - INFO - step: 68310 loss: 16.5230 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9996 +[titan] 2025-07-10 04:36:56,366 - root - INFO - lr: 9.2698e-05 gnorm: 0.89 [15:01:54< 6:58:24] +[titan] 2025-07-10 04:37:00,284 - root - INFO - step: 68315 loss: 16.4251 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.9137 +[titan] 2025-07-10 04:37:00,284 - root - INFO - lr: 9.2680e-05 gnorm: 0.96 [15:01:58< 6:58:20] +[titan] 2025-07-10 04:37:04,186 - root - INFO - step: 68320 loss: 16.0557 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.5973 +[titan] 2025-07-10 04:37:04,186 - root - INFO - lr: 9.2662e-05 gnorm: 0.94 [15:02:02< 6:58:16] +[titan] 2025-07-10 04:37:08,109 - root - INFO - step: 68325 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.7659 +[titan] 2025-07-10 04:37:08,109 - root - INFO - lr: 9.2644e-05 gnorm: 0.94 [15:02:06< 6:58:12] +[titan] 2025-07-10 04:37:12,040 - root - INFO - step: 68330 loss: 16.2139 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7356 +[titan] 2025-07-10 04:37:12,040 - root - INFO - lr: 9.2626e-05 gnorm: 0.84 [15:02:10< 6:58:08] +[titan] 2025-07-10 04:37:15,941 - root - INFO - step: 68335 loss: 16.1575 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.6833 +[titan] 2025-07-10 04:37:15,941 - root - INFO - lr: 9.2608e-05 gnorm: 0.87 [15:02:14< 6:58:04] +[titan] 2025-07-10 04:37:19,859 - root - INFO - step: 68340 loss: 16.5655 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 14.0272 +[titan] 2025-07-10 04:37:19,859 - root - INFO - lr: 9.2590e-05 gnorm: 0.85 [15:02:18< 6:58:00] +[titan] 2025-07-10 04:37:23,819 - root - INFO - step: 68345 loss: 16.2668 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 04:37:23,819 - root - INFO - lr: 9.2572e-05 gnorm: 0.97 [15:02:22< 6:57:56] +[titan] 2025-07-10 04:37:26,948 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:37:27,735 - root - INFO - step: 68350 loss: 16.4273 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.9119 +[titan] 2025-07-10 04:37:27,736 - root - INFO - lr: 9.2554e-05 gnorm: 0.88 [15:02:26< 6:57:52] +[titan] 2025-07-10 04:37:31,666 - root - INFO - step: 68355 loss: 16.1156 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.6428 +[titan] 2025-07-10 04:37:31,667 - root - INFO - lr: 9.2536e-05 gnorm: 0.94 [15:02:30< 6:57:48] +[titan] 2025-07-10 04:37:35,555 - root - INFO - step: 68360 loss: 16.1796 memory: 44.58GiB(31.99%) tps: 84,274 tflops: 290.84 mfu: 29.41% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.7121 +[titan] 2025-07-10 04:37:35,555 - root - INFO - lr: 9.2518e-05 gnorm: 0.90 [15:02:33< 6:57:44] +[titan] 2025-07-10 04:37:39,450 - root - INFO - step: 68365 loss: 16.3450 memory: 44.58GiB(31.99%) tps: 84,129 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 04:37:39,451 - root - INFO - lr: 9.2500e-05 gnorm: 0.93 [15:02:37< 6:57:40] +[titan] 2025-07-10 04:37:43,355 - root - INFO - step: 68370 loss: 16.4938 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9602 +[titan] 2025-07-10 04:37:43,355 - root - INFO - lr: 9.2481e-05 gnorm: 0.89 [15:02:41< 6:57:36] +[titan] 2025-07-10 04:37:47,246 - root - INFO - step: 68375 loss: 16.2273 memory: 44.58GiB(31.99%) tps: 84,226 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7433 +[titan] 2025-07-10 04:37:47,246 - root - INFO - lr: 9.2463e-05 gnorm: 0.86 [15:02:45< 6:57:32] +[titan] 2025-07-10 04:37:51,176 - root - INFO - step: 68380 loss: 16.5103 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9825 +[titan] 2025-07-10 04:37:51,177 - root - INFO - lr: 9.2445e-05 gnorm: 0.94 [15:02:49< 6:57:28] +[titan] 2025-07-10 04:37:55,091 - root - INFO - step: 68385 loss: 16.1377 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.6575 +[titan] 2025-07-10 04:37:55,092 - root - INFO - lr: 9.2427e-05 gnorm: 0.92 [15:02:53< 6:57:24] +[titan] 2025-07-10 04:37:59,007 - root - INFO - step: 68390 loss: 16.6298 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5508 global_avg_mtp_loss: 14.0790 +[titan] 2025-07-10 04:37:59,007 - root - INFO - lr: 9.2409e-05 gnorm: 0.94 [15:02:57< 6:57:20] +[titan] 2025-07-10 04:38:02,925 - root - INFO - step: 68395 loss: 16.4061 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8958 +[titan] 2025-07-10 04:38:02,925 - root - INFO - lr: 9.2391e-05 gnorm: 0.91 [15:03:01< 6:57:16] +[titan] 2025-07-10 04:38:06,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:38:06,835 - root - INFO - step: 68400 loss: 16.4341 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 13.8880 +[titan] 2025-07-10 04:38:06,836 - root - INFO - lr: 9.2373e-05 gnorm: 0.91 [15:03:05< 6:57:12] +[titan] 2025-07-10 04:38:10,743 - root - INFO - step: 68405 loss: 16.1381 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6683 +[titan] 2025-07-10 04:38:10,743 - root - INFO - lr: 9.2355e-05 gnorm: 0.94 [15:03:09< 6:57:08] +[titan] 2025-07-10 04:38:14,678 - root - INFO - step: 68410 loss: 16.2302 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7414 +[titan] 2025-07-10 04:38:14,678 - root - INFO - lr: 9.2337e-05 gnorm: 0.89 [15:03:13< 6:57:04] +[titan] 2025-07-10 04:38:18,578 - root - INFO - step: 68415 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8342 +[titan] 2025-07-10 04:38:18,579 - root - INFO - lr: 9.2319e-05 gnorm: 0.87 [15:03:16< 6:57:00] +[titan] 2025-07-10 04:38:22,502 - root - INFO - step: 68420 loss: 16.3066 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8122 +[titan] 2025-07-10 04:38:22,502 - root - INFO - lr: 9.2301e-05 gnorm: 0.87 [15:03:20< 6:56:56] +[titan] 2025-07-10 04:38:26,427 - root - INFO - step: 68425 loss: 16.1829 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7185 +[titan] 2025-07-10 04:38:26,428 - root - INFO - lr: 9.2283e-05 gnorm: 0.96 [15:03:24< 6:56:53] +[titan] 2025-07-10 04:38:30,342 - root - INFO - step: 68430 loss: 16.3575 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8511 +[titan] 2025-07-10 04:38:30,343 - root - INFO - lr: 9.2265e-05 gnorm: 0.85 [15:03:28< 6:56:49] +[titan] 2025-07-10 04:38:34,268 - root - INFO - step: 68435 loss: 16.1206 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.6646 +[titan] 2025-07-10 04:38:34,268 - root - INFO - lr: 9.2247e-05 gnorm: 0.96 [15:03:32< 6:56:45] +[titan] 2025-07-10 04:38:38,242 - root - INFO - step: 68440 loss: 16.4362 memory: 44.58GiB(31.99%) tps: 82,469 tflops: 284.61 mfu: 28.78% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.9226 +[titan] 2025-07-10 04:38:38,242 - root - INFO - lr: 9.2229e-05 gnorm: 0.91 [15:03:36< 6:56:41] +[titan] 2025-07-10 04:38:42,156 - root - INFO - step: 68445 loss: 16.3898 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.8695 +[titan] 2025-07-10 04:38:42,156 - root - INFO - lr: 9.2211e-05 gnorm: 0.94 [15:03:40< 6:56:37] +[titan] 2025-07-10 04:38:45,299 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:38:46,100 - root - INFO - step: 68450 loss: 16.3240 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8275 +[titan] 2025-07-10 04:38:46,100 - root - INFO - lr: 9.2193e-05 gnorm: 0.92 [15:03:44< 6:56:33] +[titan] 2025-07-10 04:38:50,034 - root - INFO - step: 68455 loss: 16.1643 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6977 +[titan] 2025-07-10 04:38:50,034 - root - INFO - lr: 9.2175e-05 gnorm: 0.94 [15:03:48< 6:56:29] +[titan] 2025-07-10 04:38:53,947 - root - INFO - step: 68460 loss: 16.2727 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7899 +[titan] 2025-07-10 04:38:53,947 - root - INFO - lr: 9.2157e-05 gnorm: 0.91 [15:03:52< 6:56:25] +[titan] 2025-07-10 04:38:57,898 - root - INFO - step: 68465 loss: 16.1397 memory: 44.58GiB(31.99%) tps: 82,954 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.6686 +[titan] 2025-07-10 04:38:57,898 - root - INFO - lr: 9.2139e-05 gnorm: 0.92 [15:03:56< 6:56:21] +[titan] 2025-07-10 04:39:01,795 - root - INFO - step: 68470 loss: 16.4334 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.9123 +[titan] 2025-07-10 04:39:01,796 - root - INFO - lr: 9.2120e-05 gnorm: 0.90 [15:04:00< 6:56:17] +[titan] 2025-07-10 04:39:05,741 - root - INFO - step: 68475 loss: 16.4963 memory: 44.58GiB(31.99%) tps: 83,063 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.9610 +[titan] 2025-07-10 04:39:05,741 - root - INFO - lr: 9.2102e-05 gnorm: 0.90 [15:04:04< 6:56:13] +[titan] 2025-07-10 04:39:09,670 - root - INFO - step: 68480 loss: 16.4515 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9255 +[titan] 2025-07-10 04:39:09,670 - root - INFO - lr: 9.2084e-05 gnorm: 0.94 [15:04:08< 6:56:09] +[titan] 2025-07-10 04:39:13,599 - root - INFO - step: 68485 loss: 16.4557 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9254 +[titan] 2025-07-10 04:39:13,600 - root - INFO - lr: 9.2066e-05 gnorm: 0.90 [15:04:11< 6:56:05] +[titan] 2025-07-10 04:39:17,504 - root - INFO - step: 68490 loss: 16.2735 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7833 +[titan] 2025-07-10 04:39:17,504 - root - INFO - lr: 9.2048e-05 gnorm: 0.92 [15:04:15< 6:56:01] +[titan] 2025-07-10 04:39:21,409 - root - INFO - step: 68495 loss: 16.4193 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8972 +[titan] 2025-07-10 04:39:21,409 - root - INFO - lr: 9.2030e-05 gnorm: 0.90 [15:04:19< 6:55:57] +[titan] 2025-07-10 04:39:24,542 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:39:25,335 - root - INFO - step: 68500 loss: 16.3114 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.8274 +[titan] 2025-07-10 04:39:25,335 - root - INFO - lr: 9.2012e-05 gnorm: 0.90 [15:04:23< 6:55:53] +[titan] 2025-07-10 04:39:29,244 - root - INFO - step: 68505 loss: 16.3901 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8702 +[titan] 2025-07-10 04:39:29,245 - root - INFO - lr: 9.1994e-05 gnorm: 0.86 [15:04:27< 6:55:49] +[titan] 2025-07-10 04:39:33,173 - root - INFO - step: 68510 loss: 16.2018 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7154 +[titan] 2025-07-10 04:39:33,173 - root - INFO - lr: 9.1976e-05 gnorm: 0.86 [15:04:31< 6:55:45] +[titan] 2025-07-10 04:39:37,141 - root - INFO - step: 68515 loss: 16.1298 memory: 44.58GiB(31.99%) tps: 82,600 tflops: 285.07 mfu: 28.82% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.6493 +[titan] 2025-07-10 04:39:37,141 - root - INFO - lr: 9.1958e-05 gnorm: 0.91 [15:04:35< 6:55:41] +[titan] 2025-07-10 04:39:41,066 - root - INFO - step: 68520 loss: 16.3805 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-10 04:39:41,066 - root - INFO - lr: 9.1940e-05 gnorm: 0.89 [15:04:39< 6:55:37] +[titan] 2025-07-10 04:39:44,974 - root - INFO - step: 68525 loss: 16.7490 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.6024 global_avg_mtp_loss: 14.1466 +[titan] 2025-07-10 04:39:44,974 - root - INFO - lr: 9.1922e-05 gnorm: 0.91 [15:04:43< 6:55:33] +[titan] 2025-07-10 04:39:48,899 - root - INFO - step: 68530 loss: 16.4278 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.9065 +[titan] 2025-07-10 04:39:48,899 - root - INFO - lr: 9.1904e-05 gnorm: 0.87 [15:04:47< 6:55:29] +[titan] 2025-07-10 04:39:52,805 - root - INFO - step: 68535 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8357 +[titan] 2025-07-10 04:39:52,806 - root - INFO - lr: 9.1886e-05 gnorm: 0.92 [15:04:51< 6:55:25] +[titan] 2025-07-10 04:39:56,720 - root - INFO - step: 68540 loss: 16.1815 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7125 +[titan] 2025-07-10 04:39:56,720 - root - INFO - lr: 9.1868e-05 gnorm: 0.90 [15:04:55< 6:55:21] +[titan] 2025-07-10 04:40:00,629 - root - INFO - step: 68545 loss: 16.1931 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.7282 +[titan] 2025-07-10 04:40:00,630 - root - INFO - lr: 9.1850e-05 gnorm: 0.93 [15:04:58< 6:55:17] +[titan] 2025-07-10 04:40:03,765 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:40:04,563 - root - INFO - step: 68550 loss: 16.4480 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.9298 +[titan] 2025-07-10 04:40:04,563 - root - INFO - lr: 9.1832e-05 gnorm: 0.94 [15:05:02< 6:55:13] +[titan] 2025-07-10 04:40:08,477 - root - INFO - step: 68555 loss: 16.4183 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.9069 +[titan] 2025-07-10 04:40:08,477 - root - INFO - lr: 9.1814e-05 gnorm: 0.86 [15:05:06< 6:55:09] +[titan] 2025-07-10 04:40:12,413 - root - INFO - step: 68560 loss: 16.5686 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5440 global_avg_mtp_loss: 14.0246 +[titan] 2025-07-10 04:40:12,413 - root - INFO - lr: 9.1796e-05 gnorm: 0.96 [15:05:10< 6:55:05] +[titan] 2025-07-10 04:40:16,345 - root - INFO - step: 68565 loss: 16.1289 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.6516 +[titan] 2025-07-10 04:40:16,345 - root - INFO - lr: 9.1778e-05 gnorm: 1.08 [15:05:14< 6:55:01] +[titan] 2025-07-10 04:40:20,295 - root - INFO - step: 68570 loss: 16.4680 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9344 +[titan] 2025-07-10 04:40:20,295 - root - INFO - lr: 9.1760e-05 gnorm: 0.95 [15:05:18< 6:54:57] +[titan] 2025-07-10 04:40:24,208 - root - INFO - step: 68575 loss: 16.3630 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8629 +[titan] 2025-07-10 04:40:24,208 - root - INFO - lr: 9.1742e-05 gnorm: 0.96 [15:05:22< 6:54:53] +[titan] 2025-07-10 04:40:28,125 - root - INFO - step: 68580 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9014 +[titan] 2025-07-10 04:40:28,125 - root - INFO - lr: 9.1724e-05 gnorm: 0.96 [15:05:26< 6:54:49] +[titan] 2025-07-10 04:40:32,053 - root - INFO - step: 68585 loss: 16.2818 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.7808 +[titan] 2025-07-10 04:40:32,053 - root - INFO - lr: 9.1706e-05 gnorm: 0.89 [15:05:30< 6:54:45] +[titan] 2025-07-10 04:40:36,012 - root - INFO - step: 68590 loss: 16.1474 memory: 44.58GiB(31.99%) tps: 82,767 tflops: 285.64 mfu: 28.88% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.6815 +[titan] 2025-07-10 04:40:36,012 - root - INFO - lr: 9.1688e-05 gnorm: 0.88 [15:05:34< 6:54:41] +[titan] 2025-07-10 04:40:39,928 - root - INFO - step: 68595 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8170 +[titan] 2025-07-10 04:40:39,928 - root - INFO - lr: 9.1670e-05 gnorm: 0.90 [15:05:38< 6:54:37] +[titan] 2025-07-10 04:40:43,048 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:40:43,840 - root - INFO - step: 68600 loss: 16.3540 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 04:40:43,840 - root - INFO - lr: 9.1652e-05 gnorm: 0.85 [15:05:42< 6:54:33] +[titan] 2025-07-10 04:40:47,869 - root - INFO - step: 68605 loss: 16.4069 memory: 44.58GiB(31.99%) tps: 81,330 tflops: 280.68 mfu: 28.38% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.8887 +[titan] 2025-07-10 04:40:47,869 - root - INFO - lr: 9.1634e-05 gnorm: 0.91 [15:05:46< 6:54:29] +[titan] 2025-07-10 04:40:50,393 - root - INFO - Dumping profiler traces at step 68608 +[titan] 2025-07-10 04:40:50,425 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:40:51,996 - root - INFO - step: 68610 loss: 16.3357 memory: 44.58GiB(31.99%) tps: 79,418 tflops: 274.08 mfu: 27.71% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8418 +[titan] 2025-07-10 04:40:51,996 - root - INFO - lr: 9.1616e-05 gnorm: 0.94 [15:05:50< 6:54:26] +[titan] 2025-07-10 04:40:55,897 - root - INFO - step: 68615 loss: 16.5042 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9781 +[titan] 2025-07-10 04:40:55,898 - root - INFO - lr: 9.1598e-05 gnorm: 0.87 [15:05:54< 6:54:22] +[titan] 2025-07-10 04:40:59,819 - root - INFO - step: 68620 loss: 16.3705 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.8658 +[titan] 2025-07-10 04:40:59,820 - root - INFO - lr: 9.1580e-05 gnorm: 0.94 [15:05:58< 6:54:18] +[titan] 2025-07-10 04:41:03,721 - root - INFO - step: 68625 loss: 16.3817 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8789 +[titan] 2025-07-10 04:41:03,721 - root - INFO - lr: 9.1562e-05 gnorm: 0.88 [15:06:02< 6:54:14] +[titan] 2025-07-10 04:41:07,636 - root - INFO - step: 68630 loss: 16.2204 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 04:41:07,636 - root - INFO - lr: 9.1544e-05 gnorm: 0.90 [15:06:05< 6:54:10] +[titan] 2025-07-10 04:41:11,554 - root - INFO - step: 68635 loss: 15.8177 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4167 global_avg_mtp_loss: 13.4010 +[titan] 2025-07-10 04:41:11,554 - root - INFO - lr: 9.1526e-05 gnorm: 0.89 [15:06:09< 6:54:06] +[titan] 2025-07-10 04:41:15,476 - root - INFO - step: 68640 loss: 16.2761 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7884 +[titan] 2025-07-10 04:41:15,476 - root - INFO - lr: 9.1509e-05 gnorm: 0.93 [15:06:13< 6:54:02] +[titan] 2025-07-10 04:41:19,408 - root - INFO - step: 68645 loss: 16.5770 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5551 global_avg_mtp_loss: 14.0219 +[titan] 2025-07-10 04:41:19,408 - root - INFO - lr: 9.1491e-05 gnorm: 0.95 [15:06:17< 6:53:58] +[titan] 2025-07-10 04:41:22,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:41:23,352 - root - INFO - step: 68650 loss: 16.5314 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.5353 global_avg_mtp_loss: 13.9961 +[titan] 2025-07-10 04:41:23,353 - root - INFO - lr: 9.1473e-05 gnorm: 0.91 [15:06:21< 6:53:54] +[titan] 2025-07-10 04:41:27,263 - root - INFO - step: 68655 loss: 16.3589 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8487 +[titan] 2025-07-10 04:41:27,263 - root - INFO - lr: 9.1455e-05 gnorm: 0.93 [15:06:25< 6:53:50] +[titan] 2025-07-10 04:41:31,156 - root - INFO - step: 68660 loss: 16.3859 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8737 +[titan] 2025-07-10 04:41:31,156 - root - INFO - lr: 9.1437e-05 gnorm: 0.86 [15:06:29< 6:53:46] +[titan] 2025-07-10 04:41:35,062 - root - INFO - step: 68665 loss: 16.4474 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-10 04:41:35,062 - root - INFO - lr: 9.1419e-05 gnorm: 0.90 [15:06:33< 6:53:42] +[titan] 2025-07-10 04:41:39,022 - root - INFO - step: 68670 loss: 16.3155 memory: 44.58GiB(31.99%) tps: 82,754 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.8131 +[titan] 2025-07-10 04:41:39,022 - root - INFO - lr: 9.1401e-05 gnorm: 0.94 [15:06:37< 6:53:38] +[titan] 2025-07-10 04:41:42,910 - root - INFO - step: 68675 loss: 16.3107 memory: 44.58GiB(31.99%) tps: 84,276 tflops: 290.85 mfu: 29.41% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.8230 +[titan] 2025-07-10 04:41:42,911 - root - INFO - lr: 9.1383e-05 gnorm: 0.86 [15:06:41< 6:53:34] +[titan] 2025-07-10 04:41:46,844 - root - INFO - step: 68680 loss: 16.5130 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 13.9735 +[titan] 2025-07-10 04:41:46,845 - root - INFO - lr: 9.1365e-05 gnorm: 0.93 [15:06:45< 6:53:30] +[titan] 2025-07-10 04:41:50,781 - root - INFO - step: 68685 loss: 16.2850 memory: 44.58GiB(31.99%) tps: 83,256 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.7818 +[titan] 2025-07-10 04:41:50,781 - root - INFO - lr: 9.1347e-05 gnorm: 0.95 [15:06:49< 6:53:26] +[titan] 2025-07-10 04:41:54,725 - root - INFO - step: 68690 loss: 16.3341 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8298 +[titan] 2025-07-10 04:41:54,725 - root - INFO - lr: 9.1329e-05 gnorm: 0.97 [15:06:53< 6:53:22] +[titan] 2025-07-10 04:41:58,644 - root - INFO - step: 68695 loss: 16.4607 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5331 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-10 04:41:58,645 - root - INFO - lr: 9.1311e-05 gnorm: 0.95 [15:06:56< 6:53:18] +[titan] 2025-07-10 04:42:01,769 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:42:02,556 - root - INFO - step: 68700 loss: 16.5554 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5403 global_avg_mtp_loss: 14.0151 +[titan] 2025-07-10 04:42:02,556 - root - INFO - lr: 9.1293e-05 gnorm: 0.91 [15:07:00< 6:53:14] +[titan] 2025-07-10 04:42:06,478 - root - INFO - step: 68705 loss: 16.6907 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5809 global_avg_mtp_loss: 14.1097 +[titan] 2025-07-10 04:42:06,478 - root - INFO - lr: 9.1275e-05 gnorm: 0.89 [15:07:04< 6:53:10] +[titan] 2025-07-10 04:42:10,380 - root - INFO - step: 68710 loss: 16.4557 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 13.9253 +[titan] 2025-07-10 04:42:10,380 - root - INFO - lr: 9.1257e-05 gnorm: 0.90 [15:07:08< 6:53:06] +[titan] 2025-07-10 04:42:14,308 - root - INFO - step: 68715 loss: 16.4972 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9657 +[titan] 2025-07-10 04:42:14,308 - root - INFO - lr: 9.1239e-05 gnorm: 0.88 [15:07:12< 6:53:02] +[titan] 2025-07-10 04:42:18,220 - root - INFO - step: 68720 loss: 16.5067 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9753 +[titan] 2025-07-10 04:42:18,221 - root - INFO - lr: 9.1221e-05 gnorm: 0.90 [15:07:16< 6:52:58] +[titan] 2025-07-10 04:42:22,165 - root - INFO - step: 68725 loss: 16.4845 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9587 +[titan] 2025-07-10 04:42:22,165 - root - INFO - lr: 9.1203e-05 gnorm: 0.88 [15:07:20< 6:52:54] +[titan] 2025-07-10 04:42:26,124 - root - INFO - step: 68730 loss: 16.3845 memory: 44.58GiB(31.99%) tps: 82,784 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8717 +[titan] 2025-07-10 04:42:26,124 - root - INFO - lr: 9.1185e-05 gnorm: 0.87 [15:07:24< 6:52:50] +[titan] 2025-07-10 04:42:30,058 - root - INFO - step: 68735 loss: 16.3820 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8716 +[titan] 2025-07-10 04:42:30,059 - root - INFO - lr: 9.1167e-05 gnorm: 0.93 [15:07:28< 6:52:46] +[titan] 2025-07-10 04:42:33,992 - root - INFO - step: 68740 loss: 16.2354 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7666 +[titan] 2025-07-10 04:42:33,992 - root - INFO - lr: 9.1150e-05 gnorm: 0.95 [15:07:32< 6:52:42] +[titan] 2025-07-10 04:42:37,943 - root - INFO - step: 68745 loss: 16.5747 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 14.0332 +[titan] 2025-07-10 04:42:37,943 - root - INFO - lr: 9.1132e-05 gnorm: 0.97 [15:07:36< 6:52:38] +[titan] 2025-07-10 04:42:41,055 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:42:41,850 - root - INFO - step: 68750 loss: 16.4016 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.8854 +[titan] 2025-07-10 04:42:41,851 - root - INFO - lr: 9.1114e-05 gnorm: 0.94 [15:07:40< 6:52:34] +[titan] 2025-07-10 04:42:45,753 - root - INFO - step: 68755 loss: 16.5132 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.9863 +[titan] 2025-07-10 04:42:45,754 - root - INFO - lr: 9.1096e-05 gnorm: 0.89 [15:07:44< 6:52:30] +[titan] 2025-07-10 04:42:49,652 - root - INFO - step: 68760 loss: 16.1030 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.6362 +[titan] 2025-07-10 04:42:49,652 - root - INFO - lr: 9.1078e-05 gnorm: 0.89 [15:07:47< 6:52:26] +[titan] 2025-07-10 04:42:53,594 - root - INFO - step: 68765 loss: 16.3084 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.7959 +[titan] 2025-07-10 04:42:53,594 - root - INFO - lr: 9.1060e-05 gnorm: 0.95 [15:07:51< 6:52:22] +[titan] 2025-07-10 04:42:57,515 - root - INFO - step: 68770 loss: 16.4654 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9319 +[titan] 2025-07-10 04:42:57,515 - root - INFO - lr: 9.1042e-05 gnorm: 0.92 [15:07:55< 6:52:18] +[titan] 2025-07-10 04:43:01,429 - root - INFO - step: 68775 loss: 16.4575 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.9351 +[titan] 2025-07-10 04:43:01,429 - root - INFO - lr: 9.1024e-05 gnorm: 0.88 [15:07:59< 6:52:14] +[titan] 2025-07-10 04:43:05,326 - root - INFO - step: 68780 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.6938 +[titan] 2025-07-10 04:43:05,326 - root - INFO - lr: 9.1006e-05 gnorm: 0.92 [15:08:03< 6:52:10] +[titan] 2025-07-10 04:43:09,234 - root - INFO - step: 68785 loss: 16.3845 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.8732 +[titan] 2025-07-10 04:43:09,235 - root - INFO - lr: 9.0988e-05 gnorm: 0.92 [15:08:07< 6:52:06] +[titan] 2025-07-10 04:43:13,172 - root - INFO - step: 68790 loss: 16.5116 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9808 +[titan] 2025-07-10 04:43:13,172 - root - INFO - lr: 9.0970e-05 gnorm: 0.96 [15:08:11< 6:52:02] +[titan] 2025-07-10 04:43:17,118 - root - INFO - step: 68795 loss: 16.3908 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.8771 +[titan] 2025-07-10 04:43:17,118 - root - INFO - lr: 9.0952e-05 gnorm: 0.91 [15:08:15< 6:51:58] +[titan] 2025-07-10 04:43:20,243 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:43:21,030 - root - INFO - step: 68800 loss: 16.5156 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 13.9697 +[titan] 2025-07-10 04:43:21,031 - root - INFO - lr: 9.0935e-05 gnorm: 0.96 [15:08:19< 6:51:54] +[titan] 2025-07-10 04:43:24,939 - root - INFO - step: 68805 loss: 16.3743 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8593 +[titan] 2025-07-10 04:43:24,939 - root - INFO - lr: 9.0917e-05 gnorm: 0.93 [15:08:23< 6:51:50] +[titan] 2025-07-10 04:43:28,843 - root - INFO - step: 68810 loss: 16.0116 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.5648 +[titan] 2025-07-10 04:43:28,843 - root - INFO - lr: 9.0899e-05 gnorm: 0.91 [15:08:27< 6:51:46] +[titan] 2025-07-10 04:43:32,744 - root - INFO - step: 68815 loss: 16.0185 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4458 global_avg_mtp_loss: 13.5727 +[titan] 2025-07-10 04:43:32,744 - root - INFO - lr: 9.0881e-05 gnorm: 0.90 [15:08:31< 6:51:42] +[titan] 2025-07-10 04:43:36,669 - root - INFO - step: 68820 loss: 16.2376 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7513 +[titan] 2025-07-10 04:43:36,670 - root - INFO - lr: 9.0863e-05 gnorm: 0.90 [15:08:34< 6:51:38] +[titan] 2025-07-10 04:43:40,575 - root - INFO - step: 68825 loss: 16.2276 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7616 +[titan] 2025-07-10 04:43:40,575 - root - INFO - lr: 9.0845e-05 gnorm: 0.87 [15:08:38< 6:51:34] +[titan] 2025-07-10 04:43:44,513 - root - INFO - step: 68830 loss: 16.3521 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8449 +[titan] 2025-07-10 04:43:44,514 - root - INFO - lr: 9.0827e-05 gnorm: 3.91 [15:08:42< 6:51:30] +[titan] 2025-07-10 04:43:48,425 - root - INFO - step: 68835 loss: 16.1425 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.6687 +[titan] 2025-07-10 04:43:48,425 - root - INFO - lr: 9.0809e-05 gnorm: 0.94 [15:08:46< 6:51:26] +[titan] 2025-07-10 04:43:52,327 - root - INFO - step: 68840 loss: 16.4407 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.9288 +[titan] 2025-07-10 04:43:52,327 - root - INFO - lr: 9.0791e-05 gnorm: 0.91 [15:08:50< 6:51:22] +[titan] 2025-07-10 04:43:56,244 - root - INFO - step: 68845 loss: 16.2874 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 04:43:56,244 - root - INFO - lr: 9.0773e-05 gnorm: 0.94 [15:08:54< 6:51:18] +[titan] 2025-07-10 04:43:59,360 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:44:00,146 - root - INFO - step: 68850 loss: 16.5169 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9888 +[titan] 2025-07-10 04:44:00,147 - root - INFO - lr: 9.0756e-05 gnorm: 0.86 [15:08:58< 6:51:14] +[titan] 2025-07-10 04:44:04,069 - root - INFO - step: 68855 loss: 16.3179 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8208 +[titan] 2025-07-10 04:44:04,069 - root - INFO - lr: 9.0738e-05 gnorm: 0.88 [15:09:02< 6:51:11] +[titan] 2025-07-10 04:44:08,059 - root - INFO - step: 68860 loss: 16.5409 memory: 44.58GiB(31.99%) tps: 82,144 tflops: 283.49 mfu: 28.66% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 14.0118 +[titan] 2025-07-10 04:44:08,059 - root - INFO - lr: 9.0720e-05 gnorm: 0.91 [15:09:06< 6:51:07] +[titan] 2025-07-10 04:44:11,980 - root - INFO - step: 68865 loss: 16.8172 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.6001 global_avg_mtp_loss: 14.2171 +[titan] 2025-07-10 04:44:11,981 - root - INFO - lr: 9.0702e-05 gnorm: 0.92 [15:09:10< 6:51:03] +[titan] 2025-07-10 04:44:15,903 - root - INFO - step: 68870 loss: 16.4175 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5243 global_avg_mtp_loss: 13.8931 +[titan] 2025-07-10 04:44:15,903 - root - INFO - lr: 9.0684e-05 gnorm: 0.91 [15:09:14< 6:50:59] +[titan] 2025-07-10 04:44:19,797 - root - INFO - step: 68875 loss: 16.2496 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7636 +[titan] 2025-07-10 04:44:19,797 - root - INFO - lr: 9.0666e-05 gnorm: 0.88 [15:09:18< 6:50:55] +[titan] 2025-07-10 04:44:23,758 - root - INFO - step: 68880 loss: 16.2373 memory: 44.58GiB(31.99%) tps: 82,731 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7535 +[titan] 2025-07-10 04:44:23,758 - root - INFO - lr: 9.0648e-05 gnorm: 0.93 [15:09:22< 6:50:51] +[titan] 2025-07-10 04:44:27,686 - root - INFO - step: 68885 loss: 16.6440 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5569 global_avg_mtp_loss: 14.0872 +[titan] 2025-07-10 04:44:27,686 - root - INFO - lr: 9.0630e-05 gnorm: 0.90 [15:09:25< 6:50:47] +[titan] 2025-07-10 04:44:31,593 - root - INFO - step: 68890 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 13.9376 +[titan] 2025-07-10 04:44:31,593 - root - INFO - lr: 9.0612e-05 gnorm: 0.93 [15:09:29< 6:50:43] +[titan] 2025-07-10 04:44:35,493 - root - INFO - step: 68895 loss: 16.3088 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.8122 +[titan] 2025-07-10 04:44:35,493 - root - INFO - lr: 9.0595e-05 gnorm: 0.88 [15:09:33< 6:50:39] +[titan] 2025-07-10 04:44:38,594 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:44:39,391 - root - INFO - step: 68900 loss: 16.1918 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7235 +[titan] 2025-07-10 04:44:39,392 - root - INFO - lr: 9.0577e-05 gnorm: 0.91 [15:09:37< 6:50:35] +[titan] 2025-07-10 04:44:43,354 - root - INFO - step: 68905 loss: 16.1537 memory: 44.58GiB(31.99%) tps: 82,704 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.6801 +[titan] 2025-07-10 04:44:43,354 - root - INFO - lr: 9.0559e-05 gnorm: 0.89 [15:09:41< 6:50:31] +[titan] 2025-07-10 04:44:47,266 - root - INFO - step: 68910 loss: 16.1821 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7134 +[titan] 2025-07-10 04:44:47,267 - root - INFO - lr: 9.0541e-05 gnorm: 0.87 [15:09:45< 6:50:27] +[titan] 2025-07-10 04:44:51,184 - root - INFO - step: 68915 loss: 16.2059 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7280 +[titan] 2025-07-10 04:44:51,184 - root - INFO - lr: 9.0523e-05 gnorm: 0.90 [15:09:49< 6:50:23] +[titan] 2025-07-10 04:44:55,134 - root - INFO - step: 68920 loss: 16.1450 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.6708 +[titan] 2025-07-10 04:44:55,135 - root - INFO - lr: 9.0505e-05 gnorm: 0.86 [15:09:53< 6:50:19] +[titan] 2025-07-10 04:44:59,105 - root - INFO - step: 68925 loss: 16.4038 memory: 44.58GiB(31.99%) tps: 82,541 tflops: 284.86 mfu: 28.80% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 04:44:59,105 - root - INFO - lr: 9.0487e-05 gnorm: 0.95 [15:09:57< 6:50:15] +[titan] 2025-07-10 04:45:03,038 - root - INFO - step: 68930 loss: 16.1433 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.6626 +[titan] 2025-07-10 04:45:03,038 - root - INFO - lr: 9.0470e-05 gnorm: 0.92 [15:10:01< 6:50:11] +[titan] 2025-07-10 04:45:06,964 - root - INFO - step: 68935 loss: 16.6185 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 14.0657 +[titan] 2025-07-10 04:45:06,964 - root - INFO - lr: 9.0452e-05 gnorm: 0.90 [15:10:05< 6:50:07] +[titan] 2025-07-10 04:45:10,919 - root - INFO - step: 68940 loss: 16.3265 memory: 44.58GiB(31.99%) tps: 82,845 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8179 +[titan] 2025-07-10 04:45:10,919 - root - INFO - lr: 9.0434e-05 gnorm: 0.90 [15:10:09< 6:50:03] +[titan] 2025-07-10 04:45:14,837 - root - INFO - step: 68945 loss: 16.5939 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5571 global_avg_mtp_loss: 14.0368 +[titan] 2025-07-10 04:45:14,837 - root - INFO - lr: 9.0416e-05 gnorm: 0.86 [15:10:13< 6:49:59] +[titan] 2025-07-10 04:45:17,969 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:45:18,760 - root - INFO - step: 68950 loss: 16.3146 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8169 +[titan] 2025-07-10 04:45:18,761 - root - INFO - lr: 9.0398e-05 gnorm: 0.93 [15:10:17< 6:49:55] +[titan] 2025-07-10 04:45:22,660 - root - INFO - step: 68955 loss: 16.4570 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5339 global_avg_mtp_loss: 13.9231 +[titan] 2025-07-10 04:45:22,660 - root - INFO - lr: 9.0380e-05 gnorm: 0.94 [15:10:20< 6:49:51] +[titan] 2025-07-10 04:45:26,554 - root - INFO - step: 68960 loss: 16.0440 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.5852 +[titan] 2025-07-10 04:45:26,555 - root - INFO - lr: 9.0362e-05 gnorm: 0.90 [15:10:24< 6:49:47] +[titan] 2025-07-10 04:45:30,498 - root - INFO - step: 68965 loss: 16.3284 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8252 +[titan] 2025-07-10 04:45:30,499 - root - INFO - lr: 9.0345e-05 gnorm: 0.92 [15:10:28< 6:49:43] +[titan] 2025-07-10 04:45:34,402 - root - INFO - step: 68970 loss: 16.4718 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9550 +[titan] 2025-07-10 04:45:34,403 - root - INFO - lr: 9.0327e-05 gnorm: 0.92 [15:10:32< 6:49:39] +[titan] 2025-07-10 04:45:38,328 - root - INFO - step: 68975 loss: 16.1513 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.6779 +[titan] 2025-07-10 04:45:38,328 - root - INFO - lr: 9.0309e-05 gnorm: 0.94 [15:10:36< 6:49:35] +[titan] 2025-07-10 04:45:42,233 - root - INFO - step: 68980 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7980 +[titan] 2025-07-10 04:45:42,233 - root - INFO - lr: 9.0291e-05 gnorm: 0.88 [15:10:40< 6:49:31] +[titan] 2025-07-10 04:45:46,159 - root - INFO - step: 68985 loss: 16.3437 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8446 +[titan] 2025-07-10 04:45:46,159 - root - INFO - lr: 9.0273e-05 gnorm: 0.89 [15:10:44< 6:49:27] +[titan] 2025-07-10 04:45:50,136 - root - INFO - step: 68990 loss: 16.6533 memory: 44.58GiB(31.99%) tps: 82,390 tflops: 284.34 mfu: 28.75% global_avg_ntp_loss: 2.5665 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-10 04:45:50,137 - root - INFO - lr: 9.0255e-05 gnorm: 0.89 [15:10:48< 6:49:23] +[titan] 2025-07-10 04:45:54,095 - root - INFO - step: 68995 loss: 16.5346 memory: 44.58GiB(31.99%) tps: 82,779 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.5422 global_avg_mtp_loss: 13.9925 +[titan] 2025-07-10 04:45:54,095 - root - INFO - lr: 9.0238e-05 gnorm: 0.92 [15:10:52< 6:49:19] +[titan] 2025-07-10 04:45:57,235 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:45:58,021 - root - INFO - step: 69000 loss: 16.4255 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.9246 +[titan] 2025-07-10 04:45:58,021 - root - INFO - lr: 9.0220e-05 gnorm: 0.90 [15:10:56< 6:49:15] +[titan] 2025-07-10 04:46:01,993 - root - INFO - step: 69005 loss: 16.2271 memory: 44.58GiB(31.99%) tps: 82,506 tflops: 284.74 mfu: 28.79% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7451 +[titan] 2025-07-10 04:46:01,993 - root - INFO - lr: 9.0202e-05 gnorm: 0.98 [15:11:00< 6:49:11] +[titan] 2025-07-10 04:46:05,937 - root - INFO - step: 69010 loss: 16.3707 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8569 +[titan] 2025-07-10 04:46:05,937 - root - INFO - lr: 9.0184e-05 gnorm: 0.91 [15:11:04< 6:49:07] +[titan] 2025-07-10 04:46:09,883 - root - INFO - step: 69015 loss: 16.3552 memory: 44.58GiB(31.99%) tps: 83,049 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5057 global_avg_mtp_loss: 13.8495 +[titan] 2025-07-10 04:46:09,883 - root - INFO - lr: 9.0166e-05 gnorm: 0.97 [15:11:08< 6:49:03] +[titan] 2025-07-10 04:46:13,825 - root - INFO - step: 69020 loss: 16.2666 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.7683 +[titan] 2025-07-10 04:46:13,825 - root - INFO - lr: 9.0148e-05 gnorm: 0.86 [15:11:12< 6:48:59] +[titan] 2025-07-10 04:46:17,768 - root - INFO - step: 69025 loss: 16.2276 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7434 +[titan] 2025-07-10 04:46:17,768 - root - INFO - lr: 9.0131e-05 gnorm: 0.92 [15:11:16< 6:48:55] +[titan] 2025-07-10 04:46:21,687 - root - INFO - step: 69030 loss: 16.2727 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 04:46:21,687 - root - INFO - lr: 9.0113e-05 gnorm: 0.91 [15:11:19< 6:48:51] +[titan] 2025-07-10 04:46:25,616 - root - INFO - step: 69035 loss: 16.1825 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7085 +[titan] 2025-07-10 04:46:25,616 - root - INFO - lr: 9.0095e-05 gnorm: 0.91 [15:11:23< 6:48:47] +[titan] 2025-07-10 04:46:29,536 - root - INFO - step: 69040 loss: 16.5988 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5544 global_avg_mtp_loss: 14.0444 +[titan] 2025-07-10 04:46:29,536 - root - INFO - lr: 9.0077e-05 gnorm: 0.93 [15:11:27< 6:48:43] +[titan] 2025-07-10 04:46:33,478 - root - INFO - step: 69045 loss: 16.5659 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5503 global_avg_mtp_loss: 14.0157 +[titan] 2025-07-10 04:46:33,479 - root - INFO - lr: 9.0059e-05 gnorm: 0.89 [15:11:31< 6:48:40] +[titan] 2025-07-10 04:46:36,598 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:46:37,389 - root - INFO - step: 69050 loss: 16.1833 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7098 +[titan] 2025-07-10 04:46:37,389 - root - INFO - lr: 9.0041e-05 gnorm: 0.87 [15:11:35< 6:48:36] +[titan] 2025-07-10 04:46:41,336 - root - INFO - step: 69055 loss: 16.4716 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9428 +[titan] 2025-07-10 04:46:41,336 - root - INFO - lr: 9.0024e-05 gnorm: 0.87 [15:11:39< 6:48:32] +[titan] 2025-07-10 04:46:45,234 - root - INFO - step: 69060 loss: 16.2149 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7275 +[titan] 2025-07-10 04:46:45,235 - root - INFO - lr: 9.0006e-05 gnorm: 0.93 [15:11:43< 6:48:28] +[titan] 2025-07-10 04:46:49,139 - root - INFO - step: 69065 loss: 16.0419 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5927 +[titan] 2025-07-10 04:46:49,139 - root - INFO - lr: 8.9988e-05 gnorm: 0.87 [15:11:47< 6:48:24] +[titan] 2025-07-10 04:46:53,073 - root - INFO - step: 69070 loss: 16.4918 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5326 global_avg_mtp_loss: 13.9591 +[titan] 2025-07-10 04:46:53,073 - root - INFO - lr: 8.9970e-05 gnorm: 0.88 [15:11:51< 6:48:20] +[titan] 2025-07-10 04:46:56,996 - root - INFO - step: 69075 loss: 16.1306 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6709 +[titan] 2025-07-10 04:46:56,996 - root - INFO - lr: 8.9952e-05 gnorm: 0.95 [15:11:55< 6:48:16] +[titan] 2025-07-10 04:47:00,899 - root - INFO - step: 69080 loss: 16.5741 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5437 global_avg_mtp_loss: 14.0304 +[titan] 2025-07-10 04:47:00,899 - root - INFO - lr: 8.9935e-05 gnorm: 0.92 [15:11:59< 6:48:12] +[titan] 2025-07-10 04:47:04,822 - root - INFO - step: 69085 loss: 16.3490 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8573 +[titan] 2025-07-10 04:47:04,822 - root - INFO - lr: 8.9917e-05 gnorm: 0.88 [15:12:03< 6:48:08] +[titan] 2025-07-10 04:47:08,720 - root - INFO - step: 69090 loss: 16.6527 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.5658 global_avg_mtp_loss: 14.0868 +[titan] 2025-07-10 04:47:08,720 - root - INFO - lr: 8.9899e-05 gnorm: 0.85 [15:12:07< 6:48:04] +[titan] 2025-07-10 04:47:12,669 - root - INFO - step: 69095 loss: 16.3756 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.8577 +[titan] 2025-07-10 04:47:12,670 - root - INFO - lr: 8.9881e-05 gnorm: 0.94 [15:12:10< 6:48:00] +[titan] 2025-07-10 04:47:15,786 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:47:16,577 - root - INFO - step: 69100 loss: 16.4529 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9287 +[titan] 2025-07-10 04:47:16,578 - root - INFO - lr: 8.9863e-05 gnorm: 0.83 [15:12:14< 6:47:56] +[titan] 2025-07-10 04:47:20,488 - root - INFO - step: 69105 loss: 16.5355 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9940 +[titan] 2025-07-10 04:47:20,489 - root - INFO - lr: 8.9846e-05 gnorm: 0.91 [15:12:18< 6:47:52] +[titan] 2025-07-10 04:47:24,426 - root - INFO - step: 69110 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.9158 +[titan] 2025-07-10 04:47:24,426 - root - INFO - lr: 8.9828e-05 gnorm: 0.89 [15:12:22< 6:47:48] +[titan] 2025-07-10 04:47:28,399 - root - INFO - step: 69115 loss: 16.3736 memory: 44.58GiB(31.99%) tps: 82,475 tflops: 284.64 mfu: 28.78% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8719 +[titan] 2025-07-10 04:47:28,400 - root - INFO - lr: 8.9810e-05 gnorm: 0.84 [15:12:26< 6:47:44] +[titan] 2025-07-10 04:47:32,451 - root - INFO - step: 69120 loss: 16.3597 memory: 44.58GiB(31.99%) tps: 80,879 tflops: 279.13 mfu: 28.22% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.8404 +[titan] 2025-07-10 04:47:32,451 - root - INFO - lr: 8.9792e-05 gnorm: 1.00 [15:12:30< 6:47:40] +[titan] 2025-07-10 04:47:32,605 - root - INFO - Dumping profiler traces at step 69120 +[titan] 2025-07-10 04:47:32,637 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:47:36,580 - root - INFO - step: 69125 loss: 16.0131 memory: 44.58GiB(31.99%) tps: 79,376 tflops: 273.94 mfu: 27.70% global_avg_ntp_loss: 2.4486 global_avg_mtp_loss: 13.5645 +[titan] 2025-07-10 04:47:36,580 - root - INFO - lr: 8.9774e-05 gnorm: 0.97 [15:12:34< 6:47:36] +[titan] 2025-07-10 04:47:40,499 - root - INFO - step: 69130 loss: 16.5793 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 14.0390 +[titan] 2025-07-10 04:47:40,499 - root - INFO - lr: 8.9757e-05 gnorm: 0.92 [15:12:38< 6:47:32] +[titan] 2025-07-10 04:47:44,431 - root - INFO - step: 69135 loss: 16.2315 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.7332 +[titan] 2025-07-10 04:47:44,432 - root - INFO - lr: 8.9739e-05 gnorm: 0.90 [15:12:42< 6:47:28] +[titan] 2025-07-10 04:47:48,359 - root - INFO - step: 69140 loss: 16.3635 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8671 +[titan] 2025-07-10 04:47:48,359 - root - INFO - lr: 8.9721e-05 gnorm: 0.89 [15:12:46< 6:47:24] +[titan] 2025-07-10 04:47:52,278 - root - INFO - step: 69145 loss: 16.4447 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.9228 +[titan] 2025-07-10 04:47:52,278 - root - INFO - lr: 8.9703e-05 gnorm: 0.90 [15:12:50< 6:47:20] +[titan] 2025-07-10 04:47:55,446 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:47:56,250 - root - INFO - step: 69150 loss: 16.3848 memory: 44.58GiB(31.99%) tps: 82,506 tflops: 284.74 mfu: 28.79% global_avg_ntp_loss: 2.5137 global_avg_mtp_loss: 13.8711 +[titan] 2025-07-10 04:47:56,250 - root - INFO - lr: 8.9686e-05 gnorm: 0.89 [15:12:54< 6:47:16] +[titan] 2025-07-10 04:48:00,166 - root - INFO - step: 69155 loss: 16.3819 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8678 +[titan] 2025-07-10 04:48:00,167 - root - INFO - lr: 8.9668e-05 gnorm: 0.88 [15:12:58< 6:47:12] +[titan] 2025-07-10 04:48:04,101 - root - INFO - step: 69160 loss: 16.5459 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9978 +[titan] 2025-07-10 04:48:04,102 - root - INFO - lr: 8.9650e-05 gnorm: 0.88 [15:13:02< 6:47:08] +[titan] 2025-07-10 04:48:08,033 - root - INFO - step: 69165 loss: 16.5179 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5384 global_avg_mtp_loss: 13.9794 +[titan] 2025-07-10 04:48:08,034 - root - INFO - lr: 8.9632e-05 gnorm: 0.86 [15:13:06< 6:47:04] +[titan] 2025-07-10 04:48:11,948 - root - INFO - step: 69170 loss: 16.4766 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9428 +[titan] 2025-07-10 04:48:11,948 - root - INFO - lr: 8.9614e-05 gnorm: 0.90 [15:13:10< 6:47:00] +[titan] 2025-07-10 04:48:15,852 - root - INFO - step: 69175 loss: 16.6704 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5716 global_avg_mtp_loss: 14.0989 +[titan] 2025-07-10 04:48:15,852 - root - INFO - lr: 8.9597e-05 gnorm: 0.90 [15:13:14< 6:46:56] +[titan] 2025-07-10 04:48:19,761 - root - INFO - step: 69180 loss: 16.4319 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.9152 +[titan] 2025-07-10 04:48:19,761 - root - INFO - lr: 8.9579e-05 gnorm: 0.89 [15:13:18< 6:46:52] +[titan] 2025-07-10 04:48:23,740 - root - INFO - step: 69185 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 82,365 tflops: 284.26 mfu: 28.74% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.8873 +[titan] 2025-07-10 04:48:23,740 - root - INFO - lr: 8.9561e-05 gnorm: 0.94 [15:13:22< 6:46:48] +[titan] 2025-07-10 04:48:27,654 - root - INFO - step: 69190 loss: 16.2121 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7334 +[titan] 2025-07-10 04:48:27,654 - root - INFO - lr: 8.9543e-05 gnorm: 0.87 [15:13:25< 6:46:44] +[titan] 2025-07-10 04:48:31,556 - root - INFO - step: 69195 loss: 16.2457 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7540 +[titan] 2025-07-10 04:48:31,557 - root - INFO - lr: 8.9526e-05 gnorm: 0.89 [15:13:29< 6:46:40] +[titan] 2025-07-10 04:48:34,673 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:48:35,465 - root - INFO - step: 69200 loss: 16.4676 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9413 +[titan] 2025-07-10 04:48:35,465 - root - INFO - lr: 8.9508e-05 gnorm: 0.88 [15:13:33< 6:46:36] +[titan] 2025-07-10 04:48:39,379 - root - INFO - step: 69205 loss: 16.1419 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6727 +[titan] 2025-07-10 04:48:39,379 - root - INFO - lr: 8.9490e-05 gnorm: 0.89 [15:13:37< 6:46:32] +[titan] 2025-07-10 04:48:43,303 - root - INFO - step: 69210 loss: 16.2681 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.7731 +[titan] 2025-07-10 04:48:43,303 - root - INFO - lr: 8.9472e-05 gnorm: 0.90 [15:13:41< 6:46:28] +[titan] 2025-07-10 04:48:47,220 - root - INFO - step: 69215 loss: 16.2763 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.7703 +[titan] 2025-07-10 04:48:47,220 - root - INFO - lr: 8.9455e-05 gnorm: 0.94 [15:13:45< 6:46:24] +[titan] 2025-07-10 04:48:51,126 - root - INFO - step: 69220 loss: 16.3756 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8809 +[titan] 2025-07-10 04:48:51,127 - root - INFO - lr: 8.9437e-05 gnorm: 0.90 [15:13:49< 6:46:20] +[titan] 2025-07-10 04:48:55,029 - root - INFO - step: 69225 loss: 16.3743 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8702 +[titan] 2025-07-10 04:48:55,029 - root - INFO - lr: 8.9419e-05 gnorm: 0.90 [15:13:53< 6:46:16] +[titan] 2025-07-10 04:48:58,947 - root - INFO - step: 69230 loss: 16.5112 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9701 +[titan] 2025-07-10 04:48:58,947 - root - INFO - lr: 8.9401e-05 gnorm: 0.88 [15:13:57< 6:46:12] +[titan] 2025-07-10 04:49:02,902 - root - INFO - step: 69235 loss: 16.4417 memory: 44.58GiB(31.99%) tps: 82,855 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.9313 +[titan] 2025-07-10 04:49:02,902 - root - INFO - lr: 8.9384e-05 gnorm: 0.90 [15:14:01< 6:46:09] +[titan] 2025-07-10 04:49:06,867 - root - INFO - step: 69240 loss: 16.1486 memory: 44.58GiB(31.99%) tps: 82,658 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 04:49:06,867 - root - INFO - lr: 8.9366e-05 gnorm: 0.88 [15:14:05< 6:46:05] +[titan] 2025-07-10 04:49:10,786 - root - INFO - step: 69245 loss: 16.4561 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.9357 +[titan] 2025-07-10 04:49:10,786 - root - INFO - lr: 8.9348e-05 gnorm: 0.88 [15:14:09< 6:46:01] +[titan] 2025-07-10 04:49:13,945 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:49:14,728 - root - INFO - step: 69250 loss: 16.4607 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9400 +[titan] 2025-07-10 04:49:14,728 - root - INFO - lr: 8.9330e-05 gnorm: 0.91 [15:14:12< 6:45:57] +[titan] 2025-07-10 04:49:18,676 - root - INFO - step: 69255 loss: 16.4200 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8979 +[titan] 2025-07-10 04:49:18,676 - root - INFO - lr: 8.9313e-05 gnorm: 0.90 [15:14:16< 6:45:53] +[titan] 2025-07-10 04:49:22,607 - root - INFO - step: 69260 loss: 16.2907 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7945 +[titan] 2025-07-10 04:49:22,607 - root - INFO - lr: 8.9295e-05 gnorm: 0.88 [15:14:20< 6:45:49] +[titan] 2025-07-10 04:49:26,525 - root - INFO - step: 69265 loss: 16.7464 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5707 global_avg_mtp_loss: 14.1757 +[titan] 2025-07-10 04:49:26,525 - root - INFO - lr: 8.9277e-05 gnorm: 0.90 [15:14:24< 6:45:45] +[titan] 2025-07-10 04:49:30,435 - root - INFO - step: 69270 loss: 16.2866 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7966 +[titan] 2025-07-10 04:49:30,435 - root - INFO - lr: 8.9259e-05 gnorm: 0.91 [15:14:28< 6:45:41] +[titan] 2025-07-10 04:49:34,367 - root - INFO - step: 69275 loss: 16.5118 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9815 +[titan] 2025-07-10 04:49:34,367 - root - INFO - lr: 8.9242e-05 gnorm: 0.90 [15:14:32< 6:45:37] +[titan] 2025-07-10 04:49:38,276 - root - INFO - step: 69280 loss: 16.4739 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5451 global_avg_mtp_loss: 13.9288 +[titan] 2025-07-10 04:49:38,276 - root - INFO - lr: 8.9224e-05 gnorm: 0.92 [15:14:36< 6:45:33] +[titan] 2025-07-10 04:49:42,192 - root - INFO - step: 69285 loss: 16.4529 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 04:49:42,192 - root - INFO - lr: 8.9206e-05 gnorm: 0.88 [15:14:40< 6:45:29] +[titan] 2025-07-10 04:49:46,115 - root - INFO - step: 69290 loss: 15.9899 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4271 global_avg_mtp_loss: 13.5628 +[titan] 2025-07-10 04:49:46,115 - root - INFO - lr: 8.9189e-05 gnorm: 0.96 [15:14:44< 6:45:25] +[titan] 2025-07-10 04:49:50,063 - root - INFO - step: 69295 loss: 16.2370 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7494 +[titan] 2025-07-10 04:49:50,063 - root - INFO - lr: 8.9171e-05 gnorm: 0.90 [15:14:48< 6:45:21] +[titan] 2025-07-10 04:49:53,204 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:49:54,013 - root - INFO - step: 69300 loss: 16.3181 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.8290 +[titan] 2025-07-10 04:49:54,014 - root - INFO - lr: 8.9153e-05 gnorm: 0.90 [15:14:52< 6:45:17] +[titan] 2025-07-10 04:49:57,934 - root - INFO - step: 69305 loss: 16.3664 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8461 +[titan] 2025-07-10 04:49:57,934 - root - INFO - lr: 8.9135e-05 gnorm: 0.94 [15:14:56< 6:45:13] +[titan] 2025-07-10 04:50:01,854 - root - INFO - step: 69310 loss: 16.1630 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.6863 +[titan] 2025-07-10 04:50:01,854 - root - INFO - lr: 8.9118e-05 gnorm: 1.00 [15:15:00< 6:45:09] +[titan] 2025-07-10 04:50:05,795 - root - INFO - step: 69315 loss: 16.1420 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4563 global_avg_mtp_loss: 13.6857 +[titan] 2025-07-10 04:50:05,796 - root - INFO - lr: 8.9100e-05 gnorm: 0.87 [15:15:04< 6:45:05] +[titan] 2025-07-10 04:50:09,718 - root - INFO - step: 69320 loss: 16.6558 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5637 global_avg_mtp_loss: 14.0922 +[titan] 2025-07-10 04:50:09,718 - root - INFO - lr: 8.9082e-05 gnorm: 0.88 [15:15:07< 6:45:01] +[titan] 2025-07-10 04:50:13,633 - root - INFO - step: 69325 loss: 16.4447 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.9250 +[titan] 2025-07-10 04:50:13,634 - root - INFO - lr: 8.9065e-05 gnorm: 0.92 [15:15:11< 6:44:57] +[titan] 2025-07-10 04:50:17,548 - root - INFO - step: 69330 loss: 16.5910 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5456 global_avg_mtp_loss: 14.0454 +[titan] 2025-07-10 04:50:17,548 - root - INFO - lr: 8.9047e-05 gnorm: 0.89 [15:15:15< 6:44:53] +[titan] 2025-07-10 04:50:21,506 - root - INFO - step: 69335 loss: 16.7007 memory: 44.58GiB(31.99%) tps: 82,792 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5758 global_avg_mtp_loss: 14.1249 +[titan] 2025-07-10 04:50:21,507 - root - INFO - lr: 8.9029e-05 gnorm: 0.92 [15:15:19< 6:44:49] +[titan] 2025-07-10 04:50:25,412 - root - INFO - step: 69340 loss: 15.9787 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4291 global_avg_mtp_loss: 13.5496 +[titan] 2025-07-10 04:50:25,412 - root - INFO - lr: 8.9011e-05 gnorm: 0.93 [15:15:23< 6:44:45] +[titan] 2025-07-10 04:50:29,355 - root - INFO - step: 69345 loss: 16.2323 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7517 +[titan] 2025-07-10 04:50:29,355 - root - INFO - lr: 8.8994e-05 gnorm: 0.90 [15:15:27< 6:44:41] +[titan] 2025-07-10 04:50:32,471 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:50:33,259 - root - INFO - step: 69350 loss: 16.2191 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7215 +[titan] 2025-07-10 04:50:33,259 - root - INFO - lr: 8.8976e-05 gnorm: 0.87 [15:15:31< 6:44:37] +[titan] 2025-07-10 04:50:37,160 - root - INFO - step: 69355 loss: 16.1734 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7003 +[titan] 2025-07-10 04:50:37,160 - root - INFO - lr: 8.8958e-05 gnorm: 0.91 [15:15:35< 6:44:33] +[titan] 2025-07-10 04:50:41,063 - root - INFO - step: 69360 loss: 16.4040 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8977 +[titan] 2025-07-10 04:50:41,064 - root - INFO - lr: 8.8941e-05 gnorm: 0.89 [15:15:39< 6:44:29] +[titan] 2025-07-10 04:50:44,985 - root - INFO - step: 69365 loss: 16.4420 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.9268 +[titan] 2025-07-10 04:50:44,985 - root - INFO - lr: 8.8923e-05 gnorm: 0.90 [15:15:43< 6:44:25] +[titan] 2025-07-10 04:50:48,920 - root - INFO - step: 69370 loss: 16.3633 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8574 +[titan] 2025-07-10 04:50:48,920 - root - INFO - lr: 8.8905e-05 gnorm: 0.89 [15:15:47< 6:44:21] +[titan] 2025-07-10 04:50:52,855 - root - INFO - step: 69375 loss: 16.1746 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.6955 +[titan] 2025-07-10 04:50:52,855 - root - INFO - lr: 8.8888e-05 gnorm: 0.94 [15:15:51< 6:44:17] +[titan] 2025-07-10 04:50:56,799 - root - INFO - step: 69380 loss: 16.3418 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8373 +[titan] 2025-07-10 04:50:56,799 - root - INFO - lr: 8.8870e-05 gnorm: 0.92 [15:15:55< 6:44:13] +[titan] 2025-07-10 04:51:00,728 - root - INFO - step: 69385 loss: 16.2402 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7617 +[titan] 2025-07-10 04:51:00,728 - root - INFO - lr: 8.8852e-05 gnorm: 0.99 [15:15:58< 6:44:09] +[titan] 2025-07-10 04:51:04,622 - root - INFO - step: 69390 loss: 16.3262 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 04:51:04,622 - root - INFO - lr: 8.8834e-05 gnorm: 0.89 [15:16:02< 6:44:05] +[titan] 2025-07-10 04:51:08,524 - root - INFO - step: 69395 loss: 16.2292 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7508 +[titan] 2025-07-10 04:51:08,524 - root - INFO - lr: 8.8817e-05 gnorm: 0.87 [15:16:06< 6:44:01] +[titan] 2025-07-10 04:51:11,647 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:51:12,443 - root - INFO - step: 69400 loss: 16.3690 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.8634 +[titan] 2025-07-10 04:51:12,443 - root - INFO - lr: 8.8799e-05 gnorm: 0.88 [15:16:10< 6:43:57] +[titan] 2025-07-10 04:51:16,349 - root - INFO - step: 69405 loss: 16.1134 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6568 +[titan] 2025-07-10 04:51:16,350 - root - INFO - lr: 8.8781e-05 gnorm: 0.86 [15:16:14< 6:43:53] +[titan] 2025-07-10 04:51:20,249 - root - INFO - step: 69410 loss: 16.2266 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7513 +[titan] 2025-07-10 04:51:20,249 - root - INFO - lr: 8.8764e-05 gnorm: 0.92 [15:16:18< 6:43:49] +[titan] 2025-07-10 04:51:24,152 - root - INFO - step: 69415 loss: 16.1455 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.6686 +[titan] 2025-07-10 04:51:24,152 - root - INFO - lr: 8.8746e-05 gnorm: 0.93 [15:16:22< 6:43:45] +[titan] 2025-07-10 04:51:28,047 - root - INFO - step: 69420 loss: 16.5900 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5757 global_avg_mtp_loss: 14.0143 +[titan] 2025-07-10 04:51:28,048 - root - INFO - lr: 8.8728e-05 gnorm: 0.94 [15:16:26< 6:43:41] +[titan] 2025-07-10 04:51:31,986 - root - INFO - step: 69425 loss: 16.2167 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.7514 +[titan] 2025-07-10 04:51:31,986 - root - INFO - lr: 8.8711e-05 gnorm: 0.90 [15:16:30< 6:43:37] +[titan] 2025-07-10 04:51:35,903 - root - INFO - step: 69430 loss: 16.3101 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8169 +[titan] 2025-07-10 04:51:35,903 - root - INFO - lr: 8.8693e-05 gnorm: 0.89 [15:16:34< 6:43:33] +[titan] 2025-07-10 04:51:39,822 - root - INFO - step: 69435 loss: 16.6424 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5534 global_avg_mtp_loss: 14.0890 +[titan] 2025-07-10 04:51:39,822 - root - INFO - lr: 8.8675e-05 gnorm: 0.89 [15:16:38< 6:43:29] +[titan] 2025-07-10 04:51:43,744 - root - INFO - step: 69440 loss: 16.3267 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8323 +[titan] 2025-07-10 04:51:43,745 - root - INFO - lr: 8.8658e-05 gnorm: 0.95 [15:16:41< 6:43:25] +[titan] 2025-07-10 04:51:47,656 - root - INFO - step: 69445 loss: 16.2461 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7705 +[titan] 2025-07-10 04:51:47,656 - root - INFO - lr: 8.8640e-05 gnorm: 0.92 [15:16:45< 6:43:21] +[titan] 2025-07-10 04:51:50,805 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:51:51,605 - root - INFO - step: 69450 loss: 16.4005 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8858 +[titan] 2025-07-10 04:51:51,605 - root - INFO - lr: 8.8622e-05 gnorm: 0.88 [15:16:49< 6:43:17] +[titan] 2025-07-10 04:51:55,538 - root - INFO - step: 69455 loss: 16.3167 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.8260 +[titan] 2025-07-10 04:51:55,538 - root - INFO - lr: 8.8605e-05 gnorm: 1.02 [15:16:53< 6:43:14] +[titan] 2025-07-10 04:51:59,456 - root - INFO - step: 69460 loss: 16.0144 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.5676 +[titan] 2025-07-10 04:51:59,456 - root - INFO - lr: 8.8587e-05 gnorm: 0.94 [15:16:57< 6:43:10] +[titan] 2025-07-10 04:52:03,359 - root - INFO - step: 69465 loss: 16.3882 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8713 +[titan] 2025-07-10 04:52:03,360 - root - INFO - lr: 8.8569e-05 gnorm: 0.97 [15:17:01< 6:43:06] +[titan] 2025-07-10 04:52:07,263 - root - INFO - step: 69470 loss: 16.4431 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.9173 +[titan] 2025-07-10 04:52:07,263 - root - INFO - lr: 8.8552e-05 gnorm: 0.90 [15:17:05< 6:43:02] +[titan] 2025-07-10 04:52:11,235 - root - INFO - step: 69475 loss: 16.2028 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7216 +[titan] 2025-07-10 04:52:11,235 - root - INFO - lr: 8.8534e-05 gnorm: 0.94 [15:17:09< 6:42:58] +[titan] 2025-07-10 04:52:15,180 - root - INFO - step: 69480 loss: 16.5114 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9812 +[titan] 2025-07-10 04:52:15,180 - root - INFO - lr: 8.8516e-05 gnorm: 0.89 [15:17:13< 6:42:54] +[titan] 2025-07-10 04:52:19,119 - root - INFO - step: 69485 loss: 16.3374 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8298 +[titan] 2025-07-10 04:52:19,120 - root - INFO - lr: 8.8499e-05 gnorm: 0.92 [15:17:17< 6:42:50] +[titan] 2025-07-10 04:52:23,013 - root - INFO - step: 69490 loss: 16.4224 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.9124 +[titan] 2025-07-10 04:52:23,013 - root - INFO - lr: 8.8481e-05 gnorm: 0.98 [15:17:21< 6:42:46] +[titan] 2025-07-10 04:52:26,912 - root - INFO - step: 69495 loss: 16.2484 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7730 +[titan] 2025-07-10 04:52:26,912 - root - INFO - lr: 8.8463e-05 gnorm: 0.85 [15:17:25< 6:42:42] +[titan] 2025-07-10 04:52:30,024 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:52:30,811 - root - INFO - step: 69500 loss: 16.2691 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7741 +[titan] 2025-07-10 04:52:30,811 - root - INFO - lr: 8.8446e-05 gnorm: 0.90 [15:17:29< 6:42:38] +[titan] 2025-07-10 04:52:34,730 - root - INFO - step: 69505 loss: 16.4452 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.9330 +[titan] 2025-07-10 04:52:34,730 - root - INFO - lr: 8.8428e-05 gnorm: 0.93 [15:17:32< 6:42:34] +[titan] 2025-07-10 04:52:38,641 - root - INFO - step: 69510 loss: 16.1598 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 04:52:38,641 - root - INFO - lr: 8.8411e-05 gnorm: 0.89 [15:17:36< 6:42:30] +[titan] 2025-07-10 04:52:42,610 - root - INFO - step: 69515 loss: 16.2184 memory: 44.58GiB(31.99%) tps: 82,555 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 04:52:42,610 - root - INFO - lr: 8.8393e-05 gnorm: 0.90 [15:17:40< 6:42:26] +[titan] 2025-07-10 04:52:46,502 - root - INFO - step: 69520 loss: 16.2974 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8022 +[titan] 2025-07-10 04:52:46,503 - root - INFO - lr: 8.8375e-05 gnorm: 0.88 [15:17:44< 6:42:22] +[titan] 2025-07-10 04:52:50,400 - root - INFO - step: 69525 loss: 16.2827 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.8028 +[titan] 2025-07-10 04:52:50,401 - root - INFO - lr: 8.8358e-05 gnorm: 0.88 [15:17:48< 6:42:18] +[titan] 2025-07-10 04:52:54,339 - root - INFO - step: 69530 loss: 16.3149 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8020 +[titan] 2025-07-10 04:52:54,339 - root - INFO - lr: 8.8340e-05 gnorm: 0.87 [15:17:52< 6:42:14] +[titan] 2025-07-10 04:52:58,309 - root - INFO - step: 69535 loss: 16.2662 memory: 44.58GiB(31.99%) tps: 82,538 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 04:52:58,310 - root - INFO - lr: 8.8322e-05 gnorm: 0.86 [15:17:56< 6:42:10] +[titan] 2025-07-10 04:53:02,235 - root - INFO - step: 69540 loss: 16.4895 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9558 +[titan] 2025-07-10 04:53:02,235 - root - INFO - lr: 8.8305e-05 gnorm: 0.86 [15:18:00< 6:42:06] +[titan] 2025-07-10 04:53:06,251 - root - INFO - step: 69545 loss: 16.5664 memory: 44.58GiB(31.99%) tps: 81,600 tflops: 281.62 mfu: 28.47% global_avg_ntp_loss: 2.5523 global_avg_mtp_loss: 14.0141 +[titan] 2025-07-10 04:53:06,251 - root - INFO - lr: 8.8287e-05 gnorm: 0.88 [15:18:04< 6:42:02] +[titan] 2025-07-10 04:53:09,402 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:53:10,185 - root - INFO - step: 69550 loss: 16.3035 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.7966 +[titan] 2025-07-10 04:53:10,185 - root - INFO - lr: 8.8269e-05 gnorm: 0.89 [15:18:08< 6:41:58] +[titan] 2025-07-10 04:53:14,115 - root - INFO - step: 69555 loss: 16.2941 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.7852 +[titan] 2025-07-10 04:53:14,115 - root - INFO - lr: 8.8252e-05 gnorm: 0.94 [15:18:12< 6:41:54] +[titan] 2025-07-10 04:53:18,009 - root - INFO - step: 69560 loss: 16.4420 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.9198 +[titan] 2025-07-10 04:53:18,009 - root - INFO - lr: 8.8234e-05 gnorm: 0.92 [15:18:16< 6:41:50] +[titan] 2025-07-10 04:53:21,941 - root - INFO - step: 69565 loss: 16.4399 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9025 +[titan] 2025-07-10 04:53:21,941 - root - INFO - lr: 8.8217e-05 gnorm: 0.90 [15:18:20< 6:41:46] +[titan] 2025-07-10 04:53:25,835 - root - INFO - step: 69570 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7247 +[titan] 2025-07-10 04:53:25,835 - root - INFO - lr: 8.8199e-05 gnorm: 0.88 [15:18:24< 6:41:42] +[titan] 2025-07-10 04:53:29,737 - root - INFO - step: 69575 loss: 16.4519 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9242 +[titan] 2025-07-10 04:53:29,737 - root - INFO - lr: 8.8181e-05 gnorm: 0.90 [15:18:27< 6:41:38] +[titan] 2025-07-10 04:53:33,639 - root - INFO - step: 69580 loss: 16.3223 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8218 +[titan] 2025-07-10 04:53:33,640 - root - INFO - lr: 8.8164e-05 gnorm: 0.88 [15:18:31< 6:41:34] +[titan] 2025-07-10 04:53:37,531 - root - INFO - step: 69585 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 84,214 tflops: 290.64 mfu: 29.39% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.7929 +[titan] 2025-07-10 04:53:37,531 - root - INFO - lr: 8.8146e-05 gnorm: 0.95 [15:18:35< 6:41:30] +[titan] 2025-07-10 04:53:41,518 - root - INFO - step: 69590 loss: 16.3964 memory: 44.58GiB(31.99%) tps: 82,200 tflops: 283.69 mfu: 28.68% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8796 +[titan] 2025-07-10 04:53:41,518 - root - INFO - lr: 8.8129e-05 gnorm: 0.93 [15:18:39< 6:41:26] +[titan] 2025-07-10 04:53:45,427 - root - INFO - step: 69595 loss: 16.0243 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.5473 +[titan] 2025-07-10 04:53:45,427 - root - INFO - lr: 8.8111e-05 gnorm: 0.93 [15:18:43< 6:41:22] +[titan] 2025-07-10 04:53:48,533 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:53:49,323 - root - INFO - step: 69600 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.9473 +[titan] 2025-07-10 04:53:49,323 - root - INFO - lr: 8.8093e-05 gnorm: 0.89 [15:18:47< 6:41:18] +[titan] 2025-07-10 04:53:53,274 - root - INFO - step: 69605 loss: 16.3217 memory: 44.58GiB(31.99%) tps: 82,954 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8248 +[titan] 2025-07-10 04:53:53,274 - root - INFO - lr: 8.8076e-05 gnorm: 0.90 [15:18:51< 6:41:14] +[titan] 2025-07-10 04:53:57,188 - root - INFO - step: 69610 loss: 15.9754 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.5246 +[titan] 2025-07-10 04:53:57,188 - root - INFO - lr: 8.8058e-05 gnorm: 1.03 [15:18:55< 6:41:10] +[titan] 2025-07-10 04:54:01,102 - root - INFO - step: 69615 loss: 16.0446 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.5842 +[titan] 2025-07-10 04:54:01,102 - root - INFO - lr: 8.8041e-05 gnorm: 0.91 [15:18:59< 6:41:06] +[titan] 2025-07-10 04:54:05,003 - root - INFO - step: 69620 loss: 16.3051 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.8172 +[titan] 2025-07-10 04:54:05,003 - root - INFO - lr: 8.8023e-05 gnorm: 0.89 [15:19:03< 6:41:02] +[titan] 2025-07-10 04:54:08,927 - root - INFO - step: 69625 loss: 16.2653 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.7706 +[titan] 2025-07-10 04:54:08,927 - root - INFO - lr: 8.8005e-05 gnorm: 0.91 [15:19:07< 6:40:58] +[titan] 2025-07-10 04:54:12,920 - root - INFO - step: 69630 loss: 16.3051 memory: 44.58GiB(31.99%) tps: 82,061 tflops: 283.21 mfu: 28.64% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.8181 +[titan] 2025-07-10 04:54:12,920 - root - INFO - lr: 8.7988e-05 gnorm: 0.89 [15:19:11< 6:40:54] +[titan] 2025-07-10 04:54:14,630 - root - INFO - Dumping profiler traces at step 69632 +[titan] 2025-07-10 04:54:14,662 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 04:54:17,021 - root - INFO - step: 69635 loss: 16.6470 memory: 44.58GiB(31.99%) tps: 79,906 tflops: 275.77 mfu: 27.88% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 14.0965 +[titan] 2025-07-10 04:54:17,022 - root - INFO - lr: 8.7970e-05 gnorm: 0.90 [15:19:15< 6:40:50] +[titan] 2025-07-10 04:54:20,949 - root - INFO - step: 69640 loss: 16.0741 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6147 +[titan] 2025-07-10 04:54:20,949 - root - INFO - lr: 8.7953e-05 gnorm: 1.20 [15:19:19< 6:40:46] +[titan] 2025-07-10 04:54:24,870 - root - INFO - step: 69645 loss: 16.5084 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9744 +[titan] 2025-07-10 04:54:24,870 - root - INFO - lr: 8.7935e-05 gnorm: 0.86 [15:19:23< 6:40:43] +[titan] 2025-07-10 04:54:28,000 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:54:28,795 - root - INFO - step: 69650 loss: 16.4084 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5290 global_avg_mtp_loss: 13.8794 +[titan] 2025-07-10 04:54:28,795 - root - INFO - lr: 8.7917e-05 gnorm: 0.93 [15:19:27< 6:40:39] +[titan] 2025-07-10 04:54:32,707 - root - INFO - step: 69655 loss: 16.6225 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5528 global_avg_mtp_loss: 14.0697 +[titan] 2025-07-10 04:54:32,707 - root - INFO - lr: 8.7900e-05 gnorm: 0.86 [15:19:30< 6:40:35] +[titan] 2025-07-10 04:54:36,602 - root - INFO - step: 69660 loss: 16.0437 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.5903 +[titan] 2025-07-10 04:54:36,602 - root - INFO - lr: 8.7882e-05 gnorm: 0.89 [15:19:34< 6:40:31] +[titan] 2025-07-10 04:54:40,515 - root - INFO - step: 69665 loss: 16.4643 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5371 global_avg_mtp_loss: 13.9272 +[titan] 2025-07-10 04:54:40,515 - root - INFO - lr: 8.7865e-05 gnorm: 0.94 [15:19:38< 6:40:27] +[titan] 2025-07-10 04:54:44,411 - root - INFO - step: 69670 loss: 16.5587 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.5419 global_avg_mtp_loss: 14.0169 +[titan] 2025-07-10 04:54:44,411 - root - INFO - lr: 8.7847e-05 gnorm: 0.88 [15:19:42< 6:40:23] +[titan] 2025-07-10 04:54:48,328 - root - INFO - step: 69675 loss: 16.5443 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 14.0080 +[titan] 2025-07-10 04:54:48,329 - root - INFO - lr: 8.7830e-05 gnorm: 0.90 [15:19:46< 6:40:19] +[titan] 2025-07-10 04:54:52,244 - root - INFO - step: 69680 loss: 16.3895 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8808 +[titan] 2025-07-10 04:54:52,244 - root - INFO - lr: 8.7812e-05 gnorm: 0.90 [15:19:50< 6:40:15] +[titan] 2025-07-10 04:54:56,140 - root - INFO - step: 69685 loss: 16.1172 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6555 +[titan] 2025-07-10 04:54:56,141 - root - INFO - lr: 8.7794e-05 gnorm: 0.91 [15:19:54< 6:40:11] +[titan] 2025-07-10 04:55:00,040 - root - INFO - step: 69690 loss: 16.4765 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9369 +[titan] 2025-07-10 04:55:00,040 - root - INFO - lr: 8.7777e-05 gnorm: 0.93 [15:19:58< 6:40:07] +[titan] 2025-07-10 04:55:03,955 - root - INFO - step: 69695 loss: 16.4504 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9221 +[titan] 2025-07-10 04:55:03,955 - root - INFO - lr: 8.7759e-05 gnorm: 0.92 [15:20:02< 6:40:03] +[titan] 2025-07-10 04:55:07,117 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:55:07,921 - root - INFO - step: 69700 loss: 16.1625 memory: 44.58GiB(31.99%) tps: 82,622 tflops: 285.14 mfu: 28.83% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.6764 +[titan] 2025-07-10 04:55:07,921 - root - INFO - lr: 8.7742e-05 gnorm: 0.89 [15:20:06< 6:39:59] +[titan] 2025-07-10 04:55:11,821 - root - INFO - step: 69705 loss: 16.1929 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4730 global_avg_mtp_loss: 13.7200 +[titan] 2025-07-10 04:55:11,821 - root - INFO - lr: 8.7724e-05 gnorm: 0.88 [15:20:10< 6:39:55] +[titan] 2025-07-10 04:55:15,724 - root - INFO - step: 69710 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.8841 +[titan] 2025-07-10 04:55:15,724 - root - INFO - lr: 8.7707e-05 gnorm: 0.93 [15:20:13< 6:39:51] +[titan] 2025-07-10 04:55:19,625 - root - INFO - step: 69715 loss: 16.5280 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5287 global_avg_mtp_loss: 13.9994 +[titan] 2025-07-10 04:55:19,626 - root - INFO - lr: 8.7689e-05 gnorm: 0.90 [15:20:17< 6:39:47] +[titan] 2025-07-10 04:55:23,550 - root - INFO - step: 69720 loss: 16.3935 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8826 +[titan] 2025-07-10 04:55:23,550 - root - INFO - lr: 8.7671e-05 gnorm: 0.92 [15:20:21< 6:39:43] +[titan] 2025-07-10 04:55:27,477 - root - INFO - step: 69725 loss: 16.5626 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 14.0240 +[titan] 2025-07-10 04:55:27,478 - root - INFO - lr: 8.7654e-05 gnorm: 0.91 [15:20:25< 6:39:39] +[titan] 2025-07-10 04:55:31,393 - root - INFO - step: 69730 loss: 16.0604 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.6131 +[titan] 2025-07-10 04:55:31,393 - root - INFO - lr: 8.7636e-05 gnorm: 0.94 [15:20:29< 6:39:35] +[titan] 2025-07-10 04:55:35,314 - root - INFO - step: 69735 loss: 16.3705 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 04:55:35,314 - root - INFO - lr: 8.7619e-05 gnorm: 0.94 [15:20:33< 6:39:31] +[titan] 2025-07-10 04:55:39,244 - root - INFO - step: 69740 loss: 16.2631 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7784 +[titan] 2025-07-10 04:55:39,244 - root - INFO - lr: 8.7601e-05 gnorm: 0.96 [15:20:37< 6:39:27] +[titan] 2025-07-10 04:55:43,190 - root - INFO - step: 69745 loss: 16.2324 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.7332 +[titan] 2025-07-10 04:55:43,190 - root - INFO - lr: 8.7584e-05 gnorm: 0.97 [15:20:41< 6:39:23] +[titan] 2025-07-10 04:55:46,312 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:55:47,110 - root - INFO - step: 69750 loss: 16.4524 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.9303 +[titan] 2025-07-10 04:55:47,110 - root - INFO - lr: 8.7566e-05 gnorm: 0.88 [15:20:45< 6:39:19] +[titan] 2025-07-10 04:55:51,028 - root - INFO - step: 69755 loss: 16.4739 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-10 04:55:51,028 - root - INFO - lr: 8.7549e-05 gnorm: 0.92 [15:20:49< 6:39:15] +[titan] 2025-07-10 04:55:54,956 - root - INFO - step: 69760 loss: 16.4194 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.9077 +[titan] 2025-07-10 04:55:54,956 - root - INFO - lr: 8.7531e-05 gnorm: 0.89 [15:20:53< 6:39:11] +[titan] 2025-07-10 04:55:58,884 - root - INFO - step: 69765 loss: 16.4263 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.9161 +[titan] 2025-07-10 04:55:58,884 - root - INFO - lr: 8.7513e-05 gnorm: 0.90 [15:20:57< 6:39:07] +[titan] 2025-07-10 04:56:02,858 - root - INFO - step: 69770 loss: 16.2009 memory: 44.58GiB(31.99%) tps: 82,458 tflops: 284.58 mfu: 28.77% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7236 +[titan] 2025-07-10 04:56:02,858 - root - INFO - lr: 8.7496e-05 gnorm: 0.93 [15:21:01< 6:39:03] +[titan] 2025-07-10 04:56:06,753 - root - INFO - step: 69775 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.9082 +[titan] 2025-07-10 04:56:06,754 - root - INFO - lr: 8.7478e-05 gnorm: 0.95 [15:21:04< 6:38:59] +[titan] 2025-07-10 04:56:10,659 - root - INFO - step: 69780 loss: 16.4962 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9670 +[titan] 2025-07-10 04:56:10,659 - root - INFO - lr: 8.7461e-05 gnorm: 0.91 [15:21:08< 6:38:55] +[titan] 2025-07-10 04:56:14,565 - root - INFO - step: 69785 loss: 16.3737 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8616 +[titan] 2025-07-10 04:56:14,566 - root - INFO - lr: 8.7443e-05 gnorm: 0.92 [15:21:12< 6:38:51] +[titan] 2025-07-10 04:56:18,463 - root - INFO - step: 69790 loss: 16.4004 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.8829 +[titan] 2025-07-10 04:56:18,463 - root - INFO - lr: 8.7426e-05 gnorm: 0.92 [15:21:16< 6:38:47] +[titan] 2025-07-10 04:56:22,366 - root - INFO - step: 69795 loss: 16.0894 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4622 global_avg_mtp_loss: 13.6273 +[titan] 2025-07-10 04:56:22,366 - root - INFO - lr: 8.7408e-05 gnorm: 0.90 [15:21:20< 6:38:43] +[titan] 2025-07-10 04:56:25,506 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:56:26,300 - root - INFO - step: 69800 loss: 16.4625 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9386 +[titan] 2025-07-10 04:56:26,300 - root - INFO - lr: 8.7391e-05 gnorm: 0.89 [15:21:24< 6:38:39] +[titan] 2025-07-10 04:56:30,219 - root - INFO - step: 69805 loss: 16.2111 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7195 +[titan] 2025-07-10 04:56:30,219 - root - INFO - lr: 8.7373e-05 gnorm: 0.88 [15:21:28< 6:38:35] +[titan] 2025-07-10 04:56:34,137 - root - INFO - step: 69810 loss: 16.1896 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7167 +[titan] 2025-07-10 04:56:34,137 - root - INFO - lr: 8.7356e-05 gnorm: 0.89 [15:21:32< 6:38:31] +[titan] 2025-07-10 04:56:38,060 - root - INFO - step: 69815 loss: 16.1440 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.6644 +[titan] 2025-07-10 04:56:38,060 - root - INFO - lr: 8.7338e-05 gnorm: 0.98 [15:21:36< 6:38:27] +[titan] 2025-07-10 04:56:41,974 - root - INFO - step: 69820 loss: 16.4206 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.8973 +[titan] 2025-07-10 04:56:41,974 - root - INFO - lr: 8.7321e-05 gnorm: 0.91 [15:21:40< 6:38:23] +[titan] 2025-07-10 04:56:45,871 - root - INFO - step: 69825 loss: 16.3842 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8792 +[titan] 2025-07-10 04:56:45,871 - root - INFO - lr: 8.7303e-05 gnorm: 0.91 [15:21:44< 6:38:19] +[titan] 2025-07-10 04:56:49,773 - root - INFO - step: 69830 loss: 16.2382 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 04:56:49,773 - root - INFO - lr: 8.7286e-05 gnorm: 0.90 [15:21:47< 6:38:15] +[titan] 2025-07-10 04:56:53,685 - root - INFO - step: 69835 loss: 16.5653 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0182 +[titan] 2025-07-10 04:56:53,686 - root - INFO - lr: 8.7268e-05 gnorm: 0.93 [15:21:51< 6:38:11] +[titan] 2025-07-10 04:56:57,610 - root - INFO - step: 69840 loss: 16.2438 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 04:56:57,610 - root - INFO - lr: 8.7251e-05 gnorm: 0.93 [15:21:55< 6:38:07] +[titan] 2025-07-10 04:57:01,548 - root - INFO - step: 69845 loss: 16.3477 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.8430 +[titan] 2025-07-10 04:57:01,548 - root - INFO - lr: 8.7233e-05 gnorm: 0.90 [15:21:59< 6:38:03] +[titan] 2025-07-10 04:57:04,654 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:57:05,455 - root - INFO - step: 69850 loss: 16.1534 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 04:57:05,455 - root - INFO - lr: 8.7216e-05 gnorm: 0.91 [15:22:03< 6:37:59] +[titan] 2025-07-10 04:57:09,409 - root - INFO - step: 69855 loss: 16.5190 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 13.9766 +[titan] 2025-07-10 04:57:09,409 - root - INFO - lr: 8.7198e-05 gnorm: 0.88 [15:22:07< 6:37:55] +[titan] 2025-07-10 04:57:13,318 - root - INFO - step: 69860 loss: 16.0203 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.5671 +[titan] 2025-07-10 04:57:13,318 - root - INFO - lr: 8.7181e-05 gnorm: 0.91 [15:22:11< 6:37:51] +[titan] 2025-07-10 04:57:17,274 - root - INFO - step: 69865 loss: 16.4055 memory: 44.58GiB(31.99%) tps: 82,847 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.8851 +[titan] 2025-07-10 04:57:17,274 - root - INFO - lr: 8.7163e-05 gnorm: 0.97 [15:22:15< 6:37:47] +[titan] 2025-07-10 04:57:21,204 - root - INFO - step: 69870 loss: 16.3348 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8366 +[titan] 2025-07-10 04:57:21,204 - root - INFO - lr: 8.7146e-05 gnorm: 0.94 [15:22:19< 6:37:43] +[titan] 2025-07-10 04:57:25,175 - root - INFO - step: 69875 loss: 16.2469 memory: 44.58GiB(31.99%) tps: 82,512 tflops: 284.76 mfu: 28.79% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7576 +[titan] 2025-07-10 04:57:25,176 - root - INFO - lr: 8.7128e-05 gnorm: 0.97 [15:22:23< 6:37:40] +[titan] 2025-07-10 04:57:29,144 - root - INFO - step: 69880 loss: 16.0893 memory: 44.58GiB(31.99%) tps: 82,571 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6228 +[titan] 2025-07-10 04:57:29,145 - root - INFO - lr: 8.7111e-05 gnorm: 1.01 [15:22:27< 6:37:36] +[titan] 2025-07-10 04:57:33,101 - root - INFO - step: 69885 loss: 16.2951 memory: 44.58GiB(31.99%) tps: 82,823 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8043 +[titan] 2025-07-10 04:57:33,101 - root - INFO - lr: 8.7093e-05 gnorm: 0.90 [15:22:31< 6:37:32] +[titan] 2025-07-10 04:57:37,017 - root - INFO - step: 69890 loss: 16.3830 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8763 +[titan] 2025-07-10 04:57:37,017 - root - INFO - lr: 8.7076e-05 gnorm: 0.90 [15:22:35< 6:37:28] +[titan] 2025-07-10 04:57:40,923 - root - INFO - step: 69895 loss: 16.0289 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4499 global_avg_mtp_loss: 13.5790 +[titan] 2025-07-10 04:57:40,923 - root - INFO - lr: 8.7058e-05 gnorm: 0.99 [15:22:39< 6:37:24] +[titan] 2025-07-10 04:57:44,031 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:57:44,835 - root - INFO - step: 69900 loss: 16.3988 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 04:57:44,835 - root - INFO - lr: 8.7041e-05 gnorm: 0.91 [15:22:43< 6:37:20] +[titan] 2025-07-10 04:57:48,740 - root - INFO - step: 69905 loss: 16.4921 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5287 global_avg_mtp_loss: 13.9634 +[titan] 2025-07-10 04:57:48,740 - root - INFO - lr: 8.7023e-05 gnorm: 0.89 [15:22:46< 6:37:16] +[titan] 2025-07-10 04:57:52,638 - root - INFO - step: 69910 loss: 16.4721 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9338 +[titan] 2025-07-10 04:57:52,639 - root - INFO - lr: 8.7006e-05 gnorm: 0.95 [15:22:50< 6:37:12] +[titan] 2025-07-10 04:57:56,562 - root - INFO - step: 69915 loss: 16.1196 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.6525 +[titan] 2025-07-10 04:57:56,562 - root - INFO - lr: 8.6988e-05 gnorm: 0.96 [15:22:54< 6:37:08] +[titan] 2025-07-10 04:58:00,477 - root - INFO - step: 69920 loss: 16.2845 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7977 +[titan] 2025-07-10 04:58:00,477 - root - INFO - lr: 8.6971e-05 gnorm: 0.92 [15:22:58< 6:37:04] +[titan] 2025-07-10 04:58:04,376 - root - INFO - step: 69925 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8217 +[titan] 2025-07-10 04:58:04,376 - root - INFO - lr: 8.6953e-05 gnorm: 0.88 [15:23:02< 6:37:00] +[titan] 2025-07-10 04:58:08,317 - root - INFO - step: 69930 loss: 16.3442 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 04:58:08,318 - root - INFO - lr: 8.6936e-05 gnorm: 0.88 [15:23:06< 6:36:56] +[titan] 2025-07-10 04:58:12,217 - root - INFO - step: 69935 loss: 16.5009 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5382 global_avg_mtp_loss: 13.9627 +[titan] 2025-07-10 04:58:12,217 - root - INFO - lr: 8.6918e-05 gnorm: 0.94 [15:23:10< 6:36:52] +[titan] 2025-07-10 04:58:16,124 - root - INFO - step: 69940 loss: 16.3098 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8181 +[titan] 2025-07-10 04:58:16,124 - root - INFO - lr: 8.6901e-05 gnorm: 0.92 [15:23:14< 6:36:48] +[titan] 2025-07-10 04:58:20,063 - root - INFO - step: 69945 loss: 16.1282 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.6624 +[titan] 2025-07-10 04:58:20,063 - root - INFO - lr: 8.6883e-05 gnorm: 0.89 [15:23:18< 6:36:44] +[titan] 2025-07-10 04:58:23,200 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:58:23,992 - root - INFO - step: 69950 loss: 16.2947 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.8067 +[titan] 2025-07-10 04:58:23,992 - root - INFO - lr: 8.6866e-05 gnorm: 0.89 [15:23:22< 6:36:40] +[titan] 2025-07-10 04:58:27,929 - root - INFO - step: 69955 loss: 16.1419 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6769 +[titan] 2025-07-10 04:58:27,929 - root - INFO - lr: 8.6848e-05 gnorm: 0.89 [15:23:26< 6:36:36] +[titan] 2025-07-10 04:58:31,844 - root - INFO - step: 69960 loss: 16.5475 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 14.0102 +[titan] 2025-07-10 04:58:31,844 - root - INFO - lr: 8.6831e-05 gnorm: 0.94 [15:23:30< 6:36:32] +[titan] 2025-07-10 04:58:35,776 - root - INFO - step: 69965 loss: 16.7207 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5742 global_avg_mtp_loss: 14.1465 +[titan] 2025-07-10 04:58:35,776 - root - INFO - lr: 8.6813e-05 gnorm: 0.90 [15:23:33< 6:36:28] +[titan] 2025-07-10 04:58:39,716 - root - INFO - step: 69970 loss: 16.4712 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9471 +[titan] 2025-07-10 04:58:39,716 - root - INFO - lr: 8.6796e-05 gnorm: 0.93 [15:23:37< 6:36:24] +[titan] 2025-07-10 04:58:43,645 - root - INFO - step: 69975 loss: 16.7496 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5859 global_avg_mtp_loss: 14.1637 +[titan] 2025-07-10 04:58:43,645 - root - INFO - lr: 8.6779e-05 gnorm: 0.91 [15:23:41< 6:36:20] +[titan] 2025-07-10 04:58:47,584 - root - INFO - step: 69980 loss: 16.2002 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4771 global_avg_mtp_loss: 13.7231 +[titan] 2025-07-10 04:58:47,585 - root - INFO - lr: 8.6761e-05 gnorm: 0.93 [15:23:45< 6:36:16] +[titan] 2025-07-10 04:58:51,508 - root - INFO - step: 69985 loss: 16.5558 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 14.0188 +[titan] 2025-07-10 04:58:51,509 - root - INFO - lr: 8.6744e-05 gnorm: 0.86 [15:23:49< 6:36:12] +[titan] 2025-07-10 04:58:55,462 - root - INFO - step: 69990 loss: 16.4527 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.9494 +[titan] 2025-07-10 04:58:55,462 - root - INFO - lr: 8.6726e-05 gnorm: 0.89 [15:23:53< 6:36:08] +[titan] 2025-07-10 04:58:59,411 - root - INFO - step: 69995 loss: 16.5681 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 14.0228 +[titan] 2025-07-10 04:58:59,412 - root - INFO - lr: 8.6709e-05 gnorm: 0.95 [15:23:57< 6:36:04] +[titan] 2025-07-10 04:59:02,538 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 04:59:03,327 - root - INFO - step: 70000 loss: 16.1990 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.7225 +[titan] 2025-07-10 04:59:03,327 - root - INFO - lr: 8.6691e-05 gnorm: 0.91 [15:24:01< 6:36:00] +[titan] 2025-07-10 04:59:03,327 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 04:59:05,873 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 04:59:05,874 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.55 seconds. +[titan] 2025-07-10 04:59:44,786 - root - INFO - step: 70005 loss: 16.1663 memory: 44.58GiB(31.99%) tps: 7,904 tflops: 27.28 mfu: 2.76% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.6947 +[titan] 2025-07-10 04:59:44,786 - root - INFO - lr: 8.6674e-05 gnorm: 0.94 [15:24:42< 6:36:12] +[titan] 2025-07-10 04:59:48,747 - root - INFO - step: 70010 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 82,734 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8622 +[titan] 2025-07-10 04:59:48,748 - root - INFO - lr: 8.6656e-05 gnorm: 0.94 [15:24:46< 6:36:08] +[titan] 2025-07-10 04:59:52,694 - root - INFO - step: 70015 loss: 16.2197 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7446 +[titan] 2025-07-10 04:59:52,694 - root - INFO - lr: 8.6639e-05 gnorm: 0.99 [15:24:50< 6:36:04] +[titan] 2025-07-10 04:59:56,647 - root - INFO - step: 70020 loss: 16.2392 memory: 44.58GiB(31.99%) tps: 82,898 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7644 +[titan] 2025-07-10 04:59:56,647 - root - INFO - lr: 8.6621e-05 gnorm: 0.95 [15:24:54< 6:36:00] +[titan] 2025-07-10 05:00:00,640 - root - INFO - step: 70025 loss: 16.1325 memory: 44.58GiB(31.99%) tps: 82,070 tflops: 283.24 mfu: 28.64% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.6605 +[titan] 2025-07-10 05:00:00,640 - root - INFO - lr: 8.6604e-05 gnorm: 0.95 [15:24:58< 6:35:56] +[titan] 2025-07-10 05:00:04,556 - root - INFO - step: 70030 loss: 16.2350 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7589 +[titan] 2025-07-10 05:00:04,556 - root - INFO - lr: 8.6587e-05 gnorm: 0.95 [15:25:02< 6:35:52] +[titan] 2025-07-10 05:00:08,493 - root - INFO - step: 70035 loss: 16.5115 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5608 global_avg_mtp_loss: 13.9507 +[titan] 2025-07-10 05:00:08,493 - root - INFO - lr: 8.6569e-05 gnorm: 0.96 [15:25:06< 6:35:48] +[titan] 2025-07-10 05:00:12,451 - root - INFO - step: 70040 loss: 15.9444 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.4347 global_avg_mtp_loss: 13.5097 +[titan] 2025-07-10 05:00:12,451 - root - INFO - lr: 8.6552e-05 gnorm: 0.91 [15:25:10< 6:35:44] +[titan] 2025-07-10 05:00:16,385 - root - INFO - step: 70045 loss: 16.4822 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.9679 +[titan] 2025-07-10 05:00:16,385 - root - INFO - lr: 8.6534e-05 gnorm: 0.90 [15:25:14< 6:35:41] +[titan] 2025-07-10 05:00:19,534 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:00:20,336 - root - INFO - step: 70050 loss: 16.5030 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5299 global_avg_mtp_loss: 13.9731 +[titan] 2025-07-10 05:00:20,336 - root - INFO - lr: 8.6517e-05 gnorm: 0.90 [15:25:18< 6:35:37] +[titan] 2025-07-10 05:00:24,331 - root - INFO - step: 70055 loss: 16.5951 memory: 44.58GiB(31.99%) tps: 82,031 tflops: 283.10 mfu: 28.62% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 14.0456 +[titan] 2025-07-10 05:00:24,331 - root - INFO - lr: 8.6499e-05 gnorm: 0.91 [15:25:22< 6:35:33] +[titan] 2025-07-10 05:00:28,289 - root - INFO - step: 70060 loss: 16.2798 memory: 44.58GiB(31.99%) tps: 82,802 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7844 +[titan] 2025-07-10 05:00:28,289 - root - INFO - lr: 8.6482e-05 gnorm: 0.97 [15:25:26< 6:35:29] +[titan] 2025-07-10 05:00:32,230 - root - INFO - step: 70065 loss: 16.2195 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7382 +[titan] 2025-07-10 05:00:32,230 - root - INFO - lr: 8.6465e-05 gnorm: 0.90 [15:25:30< 6:35:25] +[titan] 2025-07-10 05:00:36,255 - root - INFO - step: 70070 loss: 16.3521 memory: 44.58GiB(31.99%) tps: 81,423 tflops: 281.01 mfu: 28.41% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8379 +[titan] 2025-07-10 05:00:36,255 - root - INFO - lr: 8.6447e-05 gnorm: 0.88 [15:25:34< 6:35:21] +[titan] 2025-07-10 05:00:40,177 - root - INFO - step: 70075 loss: 16.2211 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7388 +[titan] 2025-07-10 05:00:40,177 - root - INFO - lr: 8.6430e-05 gnorm: 0.91 [15:25:38< 6:35:17] +[titan] 2025-07-10 05:00:44,110 - root - INFO - step: 70080 loss: 16.3576 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-10 05:00:44,110 - root - INFO - lr: 8.6412e-05 gnorm: 0.92 [15:25:42< 6:35:13] +[titan] 2025-07-10 05:00:48,057 - root - INFO - step: 70085 loss: 16.3479 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8385 +[titan] 2025-07-10 05:00:48,058 - root - INFO - lr: 8.6395e-05 gnorm: 0.93 [15:25:46< 6:35:09] +[titan] 2025-07-10 05:00:52,093 - root - INFO - step: 70090 loss: 16.4475 memory: 44.58GiB(31.99%) tps: 81,202 tflops: 280.24 mfu: 28.34% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9235 +[titan] 2025-07-10 05:00:52,093 - root - INFO - lr: 8.6378e-05 gnorm: 1.01 [15:25:50< 6:35:05] +[titan] 2025-07-10 05:00:56,030 - root - INFO - step: 70095 loss: 16.1199 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4599 global_avg_mtp_loss: 13.6600 +[titan] 2025-07-10 05:00:56,030 - root - INFO - lr: 8.6360e-05 gnorm: 0.95 [15:25:54< 6:35:01] +[titan] 2025-07-10 05:00:59,151 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:00:59,960 - root - INFO - step: 70100 loss: 16.5181 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5414 global_avg_mtp_loss: 13.9767 +[titan] 2025-07-10 05:00:59,960 - root - INFO - lr: 8.6343e-05 gnorm: 0.94 [15:25:58< 6:34:57] +[titan] 2025-07-10 05:01:03,893 - root - INFO - step: 70105 loss: 16.3609 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8461 +[titan] 2025-07-10 05:01:03,893 - root - INFO - lr: 8.6325e-05 gnorm: 0.99 [15:26:02< 6:34:53] +[titan] 2025-07-10 05:01:07,832 - root - INFO - step: 70110 loss: 16.4341 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 05:01:07,832 - root - INFO - lr: 8.6308e-05 gnorm: 0.94 [15:26:06< 6:34:49] +[titan] 2025-07-10 05:01:11,772 - root - INFO - step: 70115 loss: 16.4212 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.8977 +[titan] 2025-07-10 05:01:11,772 - root - INFO - lr: 8.6290e-05 gnorm: 0.94 [15:26:09< 6:34:45] +[titan] 2025-07-10 05:01:15,676 - root - INFO - step: 70120 loss: 16.5378 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5444 global_avg_mtp_loss: 13.9935 +[titan] 2025-07-10 05:01:15,677 - root - INFO - lr: 8.6273e-05 gnorm: 0.93 [15:26:13< 6:34:41] +[titan] 2025-07-10 05:01:19,583 - root - INFO - step: 70125 loss: 16.2962 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.7971 +[titan] 2025-07-10 05:01:19,583 - root - INFO - lr: 8.6256e-05 gnorm: 0.92 [15:26:17< 6:34:37] +[titan] 2025-07-10 05:01:23,487 - root - INFO - step: 70130 loss: 16.3682 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-10 05:01:23,487 - root - INFO - lr: 8.6238e-05 gnorm: 0.91 [15:26:21< 6:34:33] +[titan] 2025-07-10 05:01:27,415 - root - INFO - step: 70135 loss: 16.5511 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 14.0109 +[titan] 2025-07-10 05:01:27,415 - root - INFO - lr: 8.6221e-05 gnorm: 0.89 [15:26:25< 6:34:29] +[titan] 2025-07-10 05:01:31,335 - root - INFO - step: 70140 loss: 16.2431 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.7401 +[titan] 2025-07-10 05:01:31,336 - root - INFO - lr: 8.6203e-05 gnorm: 0.96 [15:26:29< 6:34:25] +[titan] 2025-07-10 05:01:34,676 - root - INFO - Dumping profiler traces at step 70144 +[titan] 2025-07-10 05:01:34,708 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:01:35,495 - root - INFO - step: 70145 loss: 16.2740 memory: 44.58GiB(31.99%) tps: 78,784 tflops: 271.90 mfu: 27.49% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 05:01:35,495 - root - INFO - lr: 8.6186e-05 gnorm: 0.96 [15:26:33< 6:34:21] +[titan] 2025-07-10 05:01:38,592 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:01:39,389 - root - INFO - step: 70150 loss: 16.4559 memory: 44.58GiB(31.99%) tps: 84,152 tflops: 290.42 mfu: 29.37% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.9307 +[titan] 2025-07-10 05:01:39,390 - root - INFO - lr: 8.6169e-05 gnorm: 0.92 [15:26:37< 6:34:17] +[titan] 2025-07-10 05:01:43,294 - root - INFO - step: 70155 loss: 16.5618 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 14.0231 +[titan] 2025-07-10 05:01:43,294 - root - INFO - lr: 8.6151e-05 gnorm: 0.94 [15:26:41< 6:34:13] +[titan] 2025-07-10 05:01:47,244 - root - INFO - step: 70160 loss: 16.1686 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.6967 +[titan] 2025-07-10 05:01:47,245 - root - INFO - lr: 8.6134e-05 gnorm: 0.87 [15:26:45< 6:34:09] +[titan] 2025-07-10 05:01:51,190 - root - INFO - step: 70165 loss: 16.3208 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8032 +[titan] 2025-07-10 05:01:51,191 - root - INFO - lr: 8.6117e-05 gnorm: 0.99 [15:26:49< 6:34:05] +[titan] 2025-07-10 05:01:55,095 - root - INFO - step: 70170 loss: 16.4343 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9134 +[titan] 2025-07-10 05:01:55,096 - root - INFO - lr: 8.6099e-05 gnorm: 0.97 [15:26:53< 6:34:01] +[titan] 2025-07-10 05:01:58,993 - root - INFO - step: 70175 loss: 16.4308 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.9048 +[titan] 2025-07-10 05:01:58,993 - root - INFO - lr: 8.6082e-05 gnorm: 0.94 [15:26:57< 6:33:57] +[titan] 2025-07-10 05:02:02,891 - root - INFO - step: 70180 loss: 16.0697 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6048 +[titan] 2025-07-10 05:02:02,891 - root - INFO - lr: 8.6064e-05 gnorm: 0.89 [15:27:01< 6:33:53] +[titan] 2025-07-10 05:02:06,799 - root - INFO - step: 70185 loss: 15.9855 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4472 global_avg_mtp_loss: 13.5383 +[titan] 2025-07-10 05:02:06,799 - root - INFO - lr: 8.6047e-05 gnorm: 0.96 [15:27:04< 6:33:49] +[titan] 2025-07-10 05:02:10,729 - root - INFO - step: 70190 loss: 16.5244 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9815 +[titan] 2025-07-10 05:02:10,729 - root - INFO - lr: 8.6030e-05 gnorm: 0.87 [15:27:08< 6:33:45] +[titan] 2025-07-10 05:02:14,650 - root - INFO - step: 70195 loss: 16.2602 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7838 +[titan] 2025-07-10 05:02:14,651 - root - INFO - lr: 8.6012e-05 gnorm: 0.86 [15:27:12< 6:33:41] +[titan] 2025-07-10 05:02:17,770 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:02:18,556 - root - INFO - step: 70200 loss: 16.4141 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8968 +[titan] 2025-07-10 05:02:18,557 - root - INFO - lr: 8.5995e-05 gnorm: 0.85 [15:27:16< 6:33:37] +[titan] 2025-07-10 05:02:22,458 - root - INFO - step: 70205 loss: 16.2039 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-10 05:02:22,458 - root - INFO - lr: 8.5978e-05 gnorm: 0.89 [15:27:20< 6:33:33] +[titan] 2025-07-10 05:02:26,377 - root - INFO - step: 70210 loss: 16.4210 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 05:02:26,377 - root - INFO - lr: 8.5960e-05 gnorm: 0.86 [15:27:24< 6:33:29] +[titan] 2025-07-10 05:02:30,296 - root - INFO - step: 70215 loss: 16.0916 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4656 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 05:02:30,296 - root - INFO - lr: 8.5943e-05 gnorm: 0.92 [15:27:28< 6:33:25] +[titan] 2025-07-10 05:02:34,191 - root - INFO - step: 70220 loss: 16.1591 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.7038 +[titan] 2025-07-10 05:02:34,191 - root - INFO - lr: 8.5925e-05 gnorm: 0.87 [15:27:32< 6:33:21] +[titan] 2025-07-10 05:02:38,101 - root - INFO - step: 70225 loss: 16.4395 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.9244 +[titan] 2025-07-10 05:02:38,101 - root - INFO - lr: 8.5908e-05 gnorm: 0.87 [15:27:36< 6:33:17] +[titan] 2025-07-10 05:02:42,030 - root - INFO - step: 70230 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8597 +[titan] 2025-07-10 05:02:42,030 - root - INFO - lr: 8.5891e-05 gnorm: 0.95 [15:27:40< 6:33:13] +[titan] 2025-07-10 05:02:45,925 - root - INFO - step: 70235 loss: 16.4212 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.9111 +[titan] 2025-07-10 05:02:45,925 - root - INFO - lr: 8.5873e-05 gnorm: 0.92 [15:27:44< 6:33:09] +[titan] 2025-07-10 05:02:49,833 - root - INFO - step: 70240 loss: 16.3455 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8386 +[titan] 2025-07-10 05:02:49,833 - root - INFO - lr: 8.5856e-05 gnorm: 0.92 [15:27:48< 6:33:05] +[titan] 2025-07-10 05:02:53,748 - root - INFO - step: 70245 loss: 16.5455 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5481 global_avg_mtp_loss: 13.9974 +[titan] 2025-07-10 05:02:53,748 - root - INFO - lr: 8.5839e-05 gnorm: 0.95 [15:27:51< 6:33:02] +[titan] 2025-07-10 05:02:56,859 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:02:57,655 - root - INFO - step: 70250 loss: 16.1088 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.6323 +[titan] 2025-07-10 05:02:57,655 - root - INFO - lr: 8.5821e-05 gnorm: 0.93 [15:27:55< 6:32:58] +[titan] 2025-07-10 05:03:01,569 - root - INFO - step: 70255 loss: 16.4063 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8954 +[titan] 2025-07-10 05:03:01,569 - root - INFO - lr: 8.5804e-05 gnorm: 0.96 [15:27:59< 6:32:54] +[titan] 2025-07-10 05:03:05,499 - root - INFO - step: 70260 loss: 16.6026 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5543 global_avg_mtp_loss: 14.0483 +[titan] 2025-07-10 05:03:05,499 - root - INFO - lr: 8.5787e-05 gnorm: 0.97 [15:28:03< 6:32:50] +[titan] 2025-07-10 05:03:09,411 - root - INFO - step: 70265 loss: 16.4942 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5478 global_avg_mtp_loss: 13.9463 +[titan] 2025-07-10 05:03:09,412 - root - INFO - lr: 8.5769e-05 gnorm: 0.90 [15:28:07< 6:32:46] +[titan] 2025-07-10 05:03:13,312 - root - INFO - step: 70270 loss: 16.2159 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.7427 +[titan] 2025-07-10 05:03:13,312 - root - INFO - lr: 8.5752e-05 gnorm: 0.91 [15:28:11< 6:32:42] +[titan] 2025-07-10 05:03:17,225 - root - INFO - step: 70275 loss: 16.4031 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8955 +[titan] 2025-07-10 05:03:17,225 - root - INFO - lr: 8.5735e-05 gnorm: 1.12 [15:28:15< 6:32:38] +[titan] 2025-07-10 05:03:21,155 - root - INFO - step: 70280 loss: 16.5616 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 14.0162 +[titan] 2025-07-10 05:03:21,155 - root - INFO - lr: 8.5717e-05 gnorm: 0.96 [15:28:19< 6:32:34] +[titan] 2025-07-10 05:03:25,059 - root - INFO - step: 70285 loss: 16.3264 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8352 +[titan] 2025-07-10 05:03:25,059 - root - INFO - lr: 8.5700e-05 gnorm: 0.91 [15:28:23< 6:32:30] +[titan] 2025-07-10 05:03:29,008 - root - INFO - step: 70290 loss: 16.3172 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 05:03:29,009 - root - INFO - lr: 8.5683e-05 gnorm: 0.90 [15:28:27< 6:32:26] +[titan] 2025-07-10 05:03:33,015 - root - INFO - step: 70295 loss: 16.6603 memory: 44.58GiB(31.99%) tps: 81,795 tflops: 282.29 mfu: 28.54% global_avg_ntp_loss: 2.5740 global_avg_mtp_loss: 14.0863 +[titan] 2025-07-10 05:03:33,015 - root - INFO - lr: 8.5665e-05 gnorm: 0.95 [15:28:31< 6:32:22] +[titan] 2025-07-10 05:03:36,147 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:03:36,937 - root - INFO - step: 70300 loss: 16.3368 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8326 +[titan] 2025-07-10 05:03:36,937 - root - INFO - lr: 8.5648e-05 gnorm: 0.88 [15:28:35< 6:32:18] +[titan] 2025-07-10 05:03:40,884 - root - INFO - step: 70305 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.4878 global_avg_mtp_loss: 13.7873 +[titan] 2025-07-10 05:03:40,884 - root - INFO - lr: 8.5631e-05 gnorm: 0.94 [15:28:39< 6:32:14] +[titan] 2025-07-10 05:03:44,824 - root - INFO - step: 70310 loss: 16.1907 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7122 +[titan] 2025-07-10 05:03:44,824 - root - INFO - lr: 8.5613e-05 gnorm: 0.91 [15:28:42< 6:32:10] +[titan] 2025-07-10 05:03:48,770 - root - INFO - step: 70315 loss: 16.1297 memory: 44.58GiB(31.99%) tps: 83,058 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.6508 +[titan] 2025-07-10 05:03:48,770 - root - INFO - lr: 8.5596e-05 gnorm: 0.97 [15:28:46< 6:32:06] +[titan] 2025-07-10 05:03:52,680 - root - INFO - step: 70320 loss: 16.2926 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.7993 +[titan] 2025-07-10 05:03:52,680 - root - INFO - lr: 8.5579e-05 gnorm: 0.96 [15:28:50< 6:32:02] +[titan] 2025-07-10 05:03:56,600 - root - INFO - step: 70325 loss: 16.4153 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.9017 +[titan] 2025-07-10 05:03:56,600 - root - INFO - lr: 8.5561e-05 gnorm: 0.89 [15:28:54< 6:31:58] +[titan] 2025-07-10 05:04:00,581 - root - INFO - step: 70330 loss: 16.2522 memory: 44.58GiB(31.99%) tps: 82,325 tflops: 284.12 mfu: 28.73% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7588 +[titan] 2025-07-10 05:04:00,581 - root - INFO - lr: 8.5544e-05 gnorm: 0.89 [15:28:58< 6:31:54] +[titan] 2025-07-10 05:04:04,502 - root - INFO - step: 70335 loss: 16.5553 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 14.0130 +[titan] 2025-07-10 05:04:04,503 - root - INFO - lr: 8.5527e-05 gnorm: 0.97 [15:29:02< 6:31:50] +[titan] 2025-07-10 05:04:08,437 - root - INFO - step: 70340 loss: 16.4060 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8853 +[titan] 2025-07-10 05:04:08,437 - root - INFO - lr: 8.5509e-05 gnorm: 0.93 [15:29:06< 6:31:46] +[titan] 2025-07-10 05:04:12,384 - root - INFO - step: 70345 loss: 16.3652 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8631 +[titan] 2025-07-10 05:04:12,384 - root - INFO - lr: 8.5492e-05 gnorm: 0.88 [15:29:10< 6:31:42] +[titan] 2025-07-10 05:04:15,511 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:04:16,298 - root - INFO - step: 70350 loss: 16.4287 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9048 +[titan] 2025-07-10 05:04:16,298 - root - INFO - lr: 8.5475e-05 gnorm: 0.88 [15:29:14< 6:31:38] +[titan] 2025-07-10 05:04:20,241 - root - INFO - step: 70355 loss: 16.4713 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5378 global_avg_mtp_loss: 13.9335 +[titan] 2025-07-10 05:04:20,241 - root - INFO - lr: 8.5457e-05 gnorm: 0.87 [15:29:18< 6:31:34] +[titan] 2025-07-10 05:04:24,160 - root - INFO - step: 70360 loss: 16.3909 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8820 +[titan] 2025-07-10 05:04:24,160 - root - INFO - lr: 8.5440e-05 gnorm: 0.89 [15:29:22< 6:31:30] +[titan] 2025-07-10 05:04:28,097 - root - INFO - step: 70365 loss: 16.3481 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 05:04:28,097 - root - INFO - lr: 8.5423e-05 gnorm: 0.89 [15:29:26< 6:31:26] +[titan] 2025-07-10 05:04:32,018 - root - INFO - step: 70370 loss: 16.1368 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6737 +[titan] 2025-07-10 05:04:32,018 - root - INFO - lr: 8.5406e-05 gnorm: 1.01 [15:29:30< 6:31:22] +[titan] 2025-07-10 05:04:35,929 - root - INFO - step: 70375 loss: 16.2963 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.7981 +[titan] 2025-07-10 05:04:35,929 - root - INFO - lr: 8.5388e-05 gnorm: 0.99 [15:29:34< 6:31:18] +[titan] 2025-07-10 05:04:39,855 - root - INFO - step: 70380 loss: 16.2422 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 05:04:39,855 - root - INFO - lr: 8.5371e-05 gnorm: 0.93 [15:29:38< 6:31:14] +[titan] 2025-07-10 05:04:43,801 - root - INFO - step: 70385 loss: 15.9826 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4404 global_avg_mtp_loss: 13.5422 +[titan] 2025-07-10 05:04:43,802 - root - INFO - lr: 8.5354e-05 gnorm: 0.90 [15:29:41< 6:31:10] +[titan] 2025-07-10 05:04:47,745 - root - INFO - step: 70390 loss: 16.5943 memory: 44.58GiB(31.99%) tps: 83,093 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0472 +[titan] 2025-07-10 05:04:47,746 - root - INFO - lr: 8.5336e-05 gnorm: 0.92 [15:29:45< 6:31:06] +[titan] 2025-07-10 05:04:51,669 - root - INFO - step: 70395 loss: 16.1522 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.6814 +[titan] 2025-07-10 05:04:51,669 - root - INFO - lr: 8.5319e-05 gnorm: 0.91 [15:29:49< 6:31:02] +[titan] 2025-07-10 05:04:54,796 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:04:55,581 - root - INFO - step: 70400 loss: 16.4973 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5296 global_avg_mtp_loss: 13.9677 +[titan] 2025-07-10 05:04:55,581 - root - INFO - lr: 8.5302e-05 gnorm: 0.93 [15:29:53< 6:30:58] +[titan] 2025-07-10 05:04:59,513 - root - INFO - step: 70405 loss: 16.1998 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7191 +[titan] 2025-07-10 05:04:59,514 - root - INFO - lr: 8.5284e-05 gnorm: 0.89 [15:29:57< 6:30:54] +[titan] 2025-07-10 05:05:03,457 - root - INFO - step: 70410 loss: 16.2078 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7334 +[titan] 2025-07-10 05:05:03,458 - root - INFO - lr: 8.5267e-05 gnorm: 0.91 [15:30:01< 6:30:50] +[titan] 2025-07-10 05:05:07,353 - root - INFO - step: 70415 loss: 16.6107 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5587 global_avg_mtp_loss: 14.0520 +[titan] 2025-07-10 05:05:07,353 - root - INFO - lr: 8.5250e-05 gnorm: 0.94 [15:30:05< 6:30:46] +[titan] 2025-07-10 05:05:11,249 - root - INFO - step: 70420 loss: 16.3608 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8550 +[titan] 2025-07-10 05:05:11,250 - root - INFO - lr: 8.5233e-05 gnorm: 0.94 [15:30:09< 6:30:42] +[titan] 2025-07-10 05:05:15,163 - root - INFO - step: 70425 loss: 16.1857 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7139 +[titan] 2025-07-10 05:05:15,163 - root - INFO - lr: 8.5215e-05 gnorm: 0.93 [15:30:13< 6:30:38] +[titan] 2025-07-10 05:05:19,056 - root - INFO - step: 70430 loss: 16.4007 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8937 +[titan] 2025-07-10 05:05:19,057 - root - INFO - lr: 8.5198e-05 gnorm: 0.92 [15:30:17< 6:30:34] +[titan] 2025-07-10 05:05:22,970 - root - INFO - step: 70435 loss: 16.2808 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7992 +[titan] 2025-07-10 05:05:22,970 - root - INFO - lr: 8.5181e-05 gnorm: 0.94 [15:30:21< 6:30:30] +[titan] 2025-07-10 05:05:26,865 - root - INFO - step: 70440 loss: 16.5173 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 13.9771 +[titan] 2025-07-10 05:05:26,865 - root - INFO - lr: 8.5164e-05 gnorm: 0.93 [15:30:25< 6:30:26] +[titan] 2025-07-10 05:05:30,769 - root - INFO - step: 70445 loss: 16.3953 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8899 +[titan] 2025-07-10 05:05:30,769 - root - INFO - lr: 8.5146e-05 gnorm: 0.90 [15:30:28< 6:30:22] +[titan] 2025-07-10 05:05:33,874 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:05:34,664 - root - INFO - step: 70450 loss: 16.3343 memory: 44.58GiB(31.99%) tps: 84,135 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-10 05:05:34,664 - root - INFO - lr: 8.5129e-05 gnorm: 0.91 [15:30:32< 6:30:18] +[titan] 2025-07-10 05:05:38,594 - root - INFO - step: 70455 loss: 16.4660 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9396 +[titan] 2025-07-10 05:05:38,595 - root - INFO - lr: 8.5112e-05 gnorm: 0.90 [15:30:36< 6:30:14] +[titan] 2025-07-10 05:05:42,547 - root - INFO - step: 70460 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7313 +[titan] 2025-07-10 05:05:42,547 - root - INFO - lr: 8.5094e-05 gnorm: 0.95 [15:30:40< 6:30:10] +[titan] 2025-07-10 05:05:46,460 - root - INFO - step: 70465 loss: 16.2235 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.7534 +[titan] 2025-07-10 05:05:46,461 - root - INFO - lr: 8.5077e-05 gnorm: 0.91 [15:30:44< 6:30:06] +[titan] 2025-07-10 05:05:50,395 - root - INFO - step: 70470 loss: 16.3582 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8554 +[titan] 2025-07-10 05:05:50,396 - root - INFO - lr: 8.5060e-05 gnorm: 0.93 [15:30:48< 6:30:02] +[titan] 2025-07-10 05:05:54,315 - root - INFO - step: 70475 loss: 15.8896 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4141 global_avg_mtp_loss: 13.4756 +[titan] 2025-07-10 05:05:54,315 - root - INFO - lr: 8.5043e-05 gnorm: 0.97 [15:30:52< 6:29:58] +[titan] 2025-07-10 05:05:58,234 - root - INFO - step: 70480 loss: 16.4688 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9432 +[titan] 2025-07-10 05:05:58,235 - root - INFO - lr: 8.5025e-05 gnorm: 0.90 [15:30:56< 6:29:55] +[titan] 2025-07-10 05:06:02,142 - root - INFO - step: 70485 loss: 16.1119 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.6374 +[titan] 2025-07-10 05:06:02,142 - root - INFO - lr: 8.5008e-05 gnorm: 0.90 [15:31:00< 6:29:51] +[titan] 2025-07-10 05:06:06,238 - root - INFO - step: 70490 loss: 16.2051 memory: 44.58GiB(31.99%) tps: 80,013 tflops: 276.14 mfu: 27.92% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.7165 +[titan] 2025-07-10 05:06:06,238 - root - INFO - lr: 8.4991e-05 gnorm: 0.87 [15:31:04< 6:29:47] +[titan] 2025-07-10 05:06:10,179 - root - INFO - step: 70495 loss: 16.3023 memory: 44.58GiB(31.99%) tps: 83,142 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8073 +[titan] 2025-07-10 05:06:10,180 - root - INFO - lr: 8.4974e-05 gnorm: 0.90 [15:31:08< 6:29:43] +[titan] 2025-07-10 05:06:13,362 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:06:14,166 - root - INFO - step: 70500 loss: 16.5297 memory: 44.58GiB(31.99%) tps: 82,206 tflops: 283.71 mfu: 28.69% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9982 +[titan] 2025-07-10 05:06:14,166 - root - INFO - lr: 8.4956e-05 gnorm: 0.92 [15:31:12< 6:29:39] +[titan] 2025-07-10 05:06:18,063 - root - INFO - step: 70505 loss: 16.3429 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.8466 +[titan] 2025-07-10 05:06:18,063 - root - INFO - lr: 8.4939e-05 gnorm: 0.95 [15:31:16< 6:29:35] +[titan] 2025-07-10 05:06:21,981 - root - INFO - step: 70510 loss: 16.1652 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4683 global_avg_mtp_loss: 13.6969 +[titan] 2025-07-10 05:06:21,981 - root - INFO - lr: 8.4922e-05 gnorm: 0.92 [15:31:20< 6:29:31] +[titan] 2025-07-10 05:06:25,880 - root - INFO - step: 70515 loss: 16.5812 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5402 global_avg_mtp_loss: 14.0410 +[titan] 2025-07-10 05:06:25,880 - root - INFO - lr: 8.4905e-05 gnorm: 0.86 [15:31:24< 6:29:27] +[titan] 2025-07-10 05:06:29,777 - root - INFO - step: 70520 loss: 16.2224 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7457 +[titan] 2025-07-10 05:06:29,777 - root - INFO - lr: 8.4887e-05 gnorm: 0.92 [15:31:27< 6:29:23] +[titan] 2025-07-10 05:06:33,709 - root - INFO - step: 70525 loss: 16.4699 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9411 +[titan] 2025-07-10 05:06:33,710 - root - INFO - lr: 8.4870e-05 gnorm: 0.92 [15:31:31< 6:29:19] +[titan] 2025-07-10 05:06:37,668 - root - INFO - step: 70530 loss: 16.3288 memory: 44.58GiB(31.99%) tps: 82,774 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8309 +[titan] 2025-07-10 05:06:37,669 - root - INFO - lr: 8.4853e-05 gnorm: 0.90 [15:31:35< 6:29:15] +[titan] 2025-07-10 05:06:41,583 - root - INFO - step: 70535 loss: 16.4364 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9101 +[titan] 2025-07-10 05:06:41,583 - root - INFO - lr: 8.4836e-05 gnorm: 0.91 [15:31:39< 6:29:11] +[titan] 2025-07-10 05:06:45,483 - root - INFO - step: 70540 loss: 16.5411 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 14.0206 +[titan] 2025-07-10 05:06:45,484 - root - INFO - lr: 8.4818e-05 gnorm: 0.96 [15:31:43< 6:29:07] +[titan] 2025-07-10 05:06:49,396 - root - INFO - step: 70545 loss: 16.3638 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8569 +[titan] 2025-07-10 05:06:49,396 - root - INFO - lr: 8.4801e-05 gnorm: 0.94 [15:31:47< 6:29:03] +[titan] 2025-07-10 05:06:52,521 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:06:53,310 - root - INFO - step: 70550 loss: 16.3139 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8140 +[titan] 2025-07-10 05:06:53,310 - root - INFO - lr: 8.4784e-05 gnorm: 0.95 [15:31:51< 6:28:59] +[titan] 2025-07-10 05:06:57,211 - root - INFO - step: 70555 loss: 16.3213 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8128 +[titan] 2025-07-10 05:06:57,211 - root - INFO - lr: 8.4767e-05 gnorm: 0.89 [15:31:55< 6:28:55] +[titan] 2025-07-10 05:07:01,155 - root - INFO - step: 70560 loss: 16.0904 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6226 +[titan] 2025-07-10 05:07:01,155 - root - INFO - lr: 8.4750e-05 gnorm: 0.91 [15:31:59< 6:28:51] +[titan] 2025-07-10 05:07:05,098 - root - INFO - step: 70565 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.9454 +[titan] 2025-07-10 05:07:05,099 - root - INFO - lr: 8.4732e-05 gnorm: 0.92 [15:32:03< 6:28:47] +[titan] 2025-07-10 05:07:09,004 - root - INFO - step: 70570 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 05:07:09,004 - root - INFO - lr: 8.4715e-05 gnorm: 0.91 [15:32:07< 6:28:43] +[titan] 2025-07-10 05:07:12,928 - root - INFO - step: 70575 loss: 16.3241 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.8383 +[titan] 2025-07-10 05:07:12,929 - root - INFO - lr: 8.4698e-05 gnorm: 0.85 [15:32:11< 6:28:39] +[titan] 2025-07-10 05:07:16,847 - root - INFO - step: 70580 loss: 16.3581 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-10 05:07:16,848 - root - INFO - lr: 8.4681e-05 gnorm: 0.93 [15:32:14< 6:28:35] +[titan] 2025-07-10 05:07:20,749 - root - INFO - step: 70585 loss: 16.5894 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5602 global_avg_mtp_loss: 14.0292 +[titan] 2025-07-10 05:07:20,749 - root - INFO - lr: 8.4663e-05 gnorm: 0.93 [15:32:18< 6:28:31] +[titan] 2025-07-10 05:07:24,674 - root - INFO - step: 70590 loss: 16.3649 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5094 global_avg_mtp_loss: 13.8555 +[titan] 2025-07-10 05:07:24,674 - root - INFO - lr: 8.4646e-05 gnorm: 0.97 [15:32:22< 6:28:27] +[titan] 2025-07-10 05:07:28,605 - root - INFO - step: 70595 loss: 16.3088 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.8280 +[titan] 2025-07-10 05:07:28,605 - root - INFO - lr: 8.4629e-05 gnorm: 0.94 [15:32:26< 6:28:23] +[titan] 2025-07-10 05:07:31,724 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:07:32,528 - root - INFO - step: 70600 loss: 16.3681 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 05:07:32,528 - root - INFO - lr: 8.4612e-05 gnorm: 0.94 [15:32:30< 6:28:19] +[titan] 2025-07-10 05:07:36,466 - root - INFO - step: 70605 loss: 16.3352 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8311 +[titan] 2025-07-10 05:07:36,466 - root - INFO - lr: 8.4595e-05 gnorm: 0.92 [15:32:34< 6:28:15] +[titan] 2025-07-10 05:07:40,430 - root - INFO - step: 70610 loss: 16.2057 memory: 44.58GiB(31.99%) tps: 82,673 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7314 +[titan] 2025-07-10 05:07:40,430 - root - INFO - lr: 8.4577e-05 gnorm: 0.96 [15:32:38< 6:28:11] +[titan] 2025-07-10 05:07:44,331 - root - INFO - step: 70615 loss: 16.4878 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5401 global_avg_mtp_loss: 13.9477 +[titan] 2025-07-10 05:07:44,331 - root - INFO - lr: 8.4560e-05 gnorm: 0.88 [15:32:42< 6:28:07] +[titan] 2025-07-10 05:07:48,266 - root - INFO - step: 70620 loss: 16.2647 memory: 44.58GiB(31.99%) tps: 83,275 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.7679 +[titan] 2025-07-10 05:07:48,266 - root - INFO - lr: 8.4543e-05 gnorm: 0.94 [15:32:46< 6:28:03] +[titan] 2025-07-10 05:07:52,165 - root - INFO - step: 70625 loss: 16.3229 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 05:07:52,165 - root - INFO - lr: 8.4526e-05 gnorm: 0.98 [15:32:50< 6:27:59] +[titan] 2025-07-10 05:07:56,075 - root - INFO - step: 70630 loss: 16.2828 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.7800 +[titan] 2025-07-10 05:07:56,076 - root - INFO - lr: 8.4509e-05 gnorm: 0.98 [15:32:54< 6:27:55] +[titan] 2025-07-10 05:08:00,007 - root - INFO - step: 70635 loss: 16.2925 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.8013 +[titan] 2025-07-10 05:08:00,008 - root - INFO - lr: 8.4491e-05 gnorm: 0.92 [15:32:58< 6:27:51] +[titan] 2025-07-10 05:08:03,915 - root - INFO - step: 70640 loss: 16.4066 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8961 +[titan] 2025-07-10 05:08:03,915 - root - INFO - lr: 8.4474e-05 gnorm: 0.91 [15:33:02< 6:27:47] +[titan] 2025-07-10 05:08:07,816 - root - INFO - step: 70645 loss: 16.5949 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5489 global_avg_mtp_loss: 14.0460 +[titan] 2025-07-10 05:08:07,817 - root - INFO - lr: 8.4457e-05 gnorm: 0.90 [15:33:05< 6:27:43] +[titan] 2025-07-10 05:08:10,916 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:08:11,720 - root - INFO - step: 70650 loss: 16.3206 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8169 +[titan] 2025-07-10 05:08:11,720 - root - INFO - lr: 8.4440e-05 gnorm: 0.91 [15:33:09< 6:27:39] +[titan] 2025-07-10 05:08:15,743 - root - INFO - step: 70655 loss: 16.5552 memory: 44.58GiB(31.99%) tps: 81,440 tflops: 281.06 mfu: 28.42% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 14.0157 +[titan] 2025-07-10 05:08:15,744 - root - INFO - lr: 8.4423e-05 gnorm: 0.88 [15:33:13< 6:27:35] +[titan] 2025-07-10 05:08:16,679 - root - INFO - Dumping profiler traces at step 70656 +[titan] 2025-07-10 05:08:16,711 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:08:19,913 - root - INFO - step: 70660 loss: 16.4700 memory: 44.58GiB(31.99%) tps: 78,598 tflops: 271.25 mfu: 27.43% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 13.9300 +[titan] 2025-07-10 05:08:19,913 - root - INFO - lr: 8.4405e-05 gnorm: 0.93 [15:33:18< 6:27:31] +[titan] 2025-07-10 05:08:23,834 - root - INFO - step: 70665 loss: 16.2042 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7218 +[titan] 2025-07-10 05:08:23,835 - root - INFO - lr: 8.4388e-05 gnorm: 0.89 [15:33:21< 6:27:27] +[titan] 2025-07-10 05:08:27,775 - root - INFO - step: 70670 loss: 16.4917 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9667 +[titan] 2025-07-10 05:08:27,775 - root - INFO - lr: 8.4371e-05 gnorm: 0.90 [15:33:25< 6:27:23] +[titan] 2025-07-10 05:08:31,700 - root - INFO - step: 70675 loss: 16.1418 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6776 +[titan] 2025-07-10 05:08:31,700 - root - INFO - lr: 8.4354e-05 gnorm: 0.95 [15:33:29< 6:27:20] +[titan] 2025-07-10 05:08:35,609 - root - INFO - step: 70680 loss: 16.0969 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6392 +[titan] 2025-07-10 05:08:35,609 - root - INFO - lr: 8.4337e-05 gnorm: 0.96 [15:33:33< 6:27:16] +[titan] 2025-07-10 05:08:39,544 - root - INFO - step: 70685 loss: 16.2580 memory: 44.58GiB(31.99%) tps: 83,278 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7704 +[titan] 2025-07-10 05:08:39,545 - root - INFO - lr: 8.4320e-05 gnorm: 0.89 [15:33:37< 6:27:12] +[titan] 2025-07-10 05:08:43,469 - root - INFO - step: 70690 loss: 16.6121 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0569 +[titan] 2025-07-10 05:08:43,469 - root - INFO - lr: 8.4302e-05 gnorm: 0.96 [15:33:41< 6:27:08] +[titan] 2025-07-10 05:08:47,384 - root - INFO - step: 70695 loss: 16.4564 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5195 global_avg_mtp_loss: 13.9370 +[titan] 2025-07-10 05:08:47,384 - root - INFO - lr: 8.4285e-05 gnorm: 0.90 [15:33:45< 6:27:04] +[titan] 2025-07-10 05:08:50,507 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:08:51,325 - root - INFO - step: 70700 loss: 16.1873 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.7273 +[titan] 2025-07-10 05:08:51,325 - root - INFO - lr: 8.4268e-05 gnorm: 0.90 [15:33:49< 6:27:00] +[titan] 2025-07-10 05:08:55,221 - root - INFO - step: 70705 loss: 16.3761 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8552 +[titan] 2025-07-10 05:08:55,222 - root - INFO - lr: 8.4251e-05 gnorm: 0.95 [15:33:53< 6:26:56] +[titan] 2025-07-10 05:08:59,184 - root - INFO - step: 70710 loss: 16.5057 memory: 44.58GiB(31.99%) tps: 82,703 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9746 +[titan] 2025-07-10 05:08:59,184 - root - INFO - lr: 8.4234e-05 gnorm: 1.00 [15:33:57< 6:26:52] +[titan] 2025-07-10 05:09:03,107 - root - INFO - step: 70715 loss: 16.6583 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5617 global_avg_mtp_loss: 14.0967 +[titan] 2025-07-10 05:09:03,108 - root - INFO - lr: 8.4217e-05 gnorm: 0.92 [15:34:01< 6:26:48] +[titan] 2025-07-10 05:09:07,038 - root - INFO - step: 70720 loss: 16.1699 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.6862 +[titan] 2025-07-10 05:09:07,039 - root - INFO - lr: 8.4199e-05 gnorm: 0.97 [15:34:05< 6:26:44] +[titan] 2025-07-10 05:09:10,954 - root - INFO - step: 70725 loss: 16.3187 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8218 +[titan] 2025-07-10 05:09:10,954 - root - INFO - lr: 8.4182e-05 gnorm: 0.93 [15:34:09< 6:26:40] +[titan] 2025-07-10 05:09:14,857 - root - INFO - step: 70730 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5159 global_avg_mtp_loss: 13.8855 +[titan] 2025-07-10 05:09:14,858 - root - INFO - lr: 8.4165e-05 gnorm: 0.88 [15:34:12< 6:26:36] +[titan] 2025-07-10 05:09:18,779 - root - INFO - step: 70735 loss: 16.4574 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9365 +[titan] 2025-07-10 05:09:18,779 - root - INFO - lr: 8.4148e-05 gnorm: 0.94 [15:34:16< 6:26:32] +[titan] 2025-07-10 05:09:22,677 - root - INFO - step: 70740 loss: 16.2357 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7510 +[titan] 2025-07-10 05:09:22,677 - root - INFO - lr: 8.4131e-05 gnorm: 0.92 [15:34:20< 6:26:28] +[titan] 2025-07-10 05:09:26,613 - root - INFO - step: 70745 loss: 16.3345 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8282 +[titan] 2025-07-10 05:09:26,613 - root - INFO - lr: 8.4114e-05 gnorm: 0.90 [15:34:24< 6:26:24] +[titan] 2025-07-10 05:09:29,734 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:09:30,525 - root - INFO - step: 70750 loss: 16.3528 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8381 +[titan] 2025-07-10 05:09:30,525 - root - INFO - lr: 8.4097e-05 gnorm: 0.90 [15:34:28< 6:26:20] +[titan] 2025-07-10 05:09:34,438 - root - INFO - step: 70755 loss: 16.3325 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 05:09:34,438 - root - INFO - lr: 8.4079e-05 gnorm: 0.96 [15:34:32< 6:26:16] +[titan] 2025-07-10 05:09:38,354 - root - INFO - step: 70760 loss: 16.5156 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5318 global_avg_mtp_loss: 13.9838 +[titan] 2025-07-10 05:09:38,355 - root - INFO - lr: 8.4062e-05 gnorm: 0.86 [15:34:36< 6:26:12] +[titan] 2025-07-10 05:09:42,264 - root - INFO - step: 70765 loss: 16.3192 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8142 +[titan] 2025-07-10 05:09:42,264 - root - INFO - lr: 8.4045e-05 gnorm: 1.30 [15:34:40< 6:26:08] +[titan] 2025-07-10 05:09:46,182 - root - INFO - step: 70770 loss: 16.4028 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8980 +[titan] 2025-07-10 05:09:46,182 - root - INFO - lr: 8.4028e-05 gnorm: 0.90 [15:34:44< 6:26:04] +[titan] 2025-07-10 05:09:50,083 - root - INFO - step: 70775 loss: 16.3493 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8516 +[titan] 2025-07-10 05:09:50,084 - root - INFO - lr: 8.4011e-05 gnorm: 0.88 [15:34:48< 6:26:00] +[titan] 2025-07-10 05:09:54,006 - root - INFO - step: 70780 loss: 16.3893 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8874 +[titan] 2025-07-10 05:09:54,007 - root - INFO - lr: 8.3994e-05 gnorm: 0.95 [15:34:52< 6:25:56] +[titan] 2025-07-10 05:09:57,924 - root - INFO - step: 70785 loss: 16.2702 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7796 +[titan] 2025-07-10 05:09:57,924 - root - INFO - lr: 8.3977e-05 gnorm: 0.97 [15:34:56< 6:25:52] +[titan] 2025-07-10 05:10:01,862 - root - INFO - step: 70790 loss: 16.1968 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7101 +[titan] 2025-07-10 05:10:01,862 - root - INFO - lr: 8.3959e-05 gnorm: 0.96 [15:34:59< 6:25:48] +[titan] 2025-07-10 05:10:05,759 - root - INFO - step: 70795 loss: 16.1513 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 05:10:05,759 - root - INFO - lr: 8.3942e-05 gnorm: 0.90 [15:35:03< 6:25:44] +[titan] 2025-07-10 05:10:08,885 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:10:09,675 - root - INFO - step: 70800 loss: 16.3923 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5007 global_avg_mtp_loss: 13.8916 +[titan] 2025-07-10 05:10:09,675 - root - INFO - lr: 8.3925e-05 gnorm: 0.95 [15:35:07< 6:25:40] +[titan] 2025-07-10 05:10:13,572 - root - INFO - step: 70805 loss: 16.4384 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.9217 +[titan] 2025-07-10 05:10:13,572 - root - INFO - lr: 8.3908e-05 gnorm: 1.00 [15:35:11< 6:25:36] +[titan] 2025-07-10 05:10:17,492 - root - INFO - step: 70810 loss: 16.3844 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8678 +[titan] 2025-07-10 05:10:17,492 - root - INFO - lr: 8.3891e-05 gnorm: 0.87 [15:35:15< 6:25:32] +[titan] 2025-07-10 05:10:21,407 - root - INFO - step: 70815 loss: 16.2938 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 05:10:21,407 - root - INFO - lr: 8.3874e-05 gnorm: 0.91 [15:35:19< 6:25:28] +[titan] 2025-07-10 05:10:25,328 - root - INFO - step: 70820 loss: 16.4008 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8901 +[titan] 2025-07-10 05:10:25,329 - root - INFO - lr: 8.3857e-05 gnorm: 0.89 [15:35:23< 6:25:24] +[titan] 2025-07-10 05:10:29,262 - root - INFO - step: 70825 loss: 16.3509 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.5130 global_avg_mtp_loss: 13.8379 +[titan] 2025-07-10 05:10:29,263 - root - INFO - lr: 8.3840e-05 gnorm: 0.94 [15:35:27< 6:25:20] +[titan] 2025-07-10 05:10:33,176 - root - INFO - step: 70830 loss: 16.2862 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 05:10:33,176 - root - INFO - lr: 8.3822e-05 gnorm: 0.89 [15:35:31< 6:25:16] +[titan] 2025-07-10 05:10:37,146 - root - INFO - step: 70835 loss: 16.0776 memory: 44.58GiB(31.99%) tps: 82,536 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.4657 global_avg_mtp_loss: 13.6118 +[titan] 2025-07-10 05:10:37,147 - root - INFO - lr: 8.3805e-05 gnorm: 0.92 [15:35:35< 6:25:12] +[titan] 2025-07-10 05:10:41,069 - root - INFO - step: 70840 loss: 16.3754 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.8720 +[titan] 2025-07-10 05:10:41,069 - root - INFO - lr: 8.3788e-05 gnorm: 0.94 [15:35:39< 6:25:08] +[titan] 2025-07-10 05:10:44,974 - root - INFO - step: 70845 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.9029 +[titan] 2025-07-10 05:10:44,974 - root - INFO - lr: 8.3771e-05 gnorm: 0.93 [15:35:43< 6:25:04] +[titan] 2025-07-10 05:10:48,106 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:10:48,905 - root - INFO - step: 70850 loss: 16.4293 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.9118 +[titan] 2025-07-10 05:10:48,905 - root - INFO - lr: 8.3754e-05 gnorm: 0.91 [15:35:47< 6:25:00] +[titan] 2025-07-10 05:10:52,835 - root - INFO - step: 70855 loss: 16.8251 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.6231 global_avg_mtp_loss: 14.2020 +[titan] 2025-07-10 05:10:52,836 - root - INFO - lr: 8.3737e-05 gnorm: 0.92 [15:35:50< 6:24:56] +[titan] 2025-07-10 05:10:56,749 - root - INFO - step: 70860 loss: 16.1192 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4545 global_avg_mtp_loss: 13.6646 +[titan] 2025-07-10 05:10:56,749 - root - INFO - lr: 8.3720e-05 gnorm: 0.89 [15:35:54< 6:24:52] +[titan] 2025-07-10 05:11:00,652 - root - INFO - step: 70865 loss: 16.2264 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7362 +[titan] 2025-07-10 05:11:00,652 - root - INFO - lr: 8.3703e-05 gnorm: 0.97 [15:35:58< 6:24:48] +[titan] 2025-07-10 05:11:04,578 - root - INFO - step: 70870 loss: 16.1178 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4730 global_avg_mtp_loss: 13.6448 +[titan] 2025-07-10 05:11:04,578 - root - INFO - lr: 8.3686e-05 gnorm: 0.88 [15:36:02< 6:24:44] +[titan] 2025-07-10 05:11:08,479 - root - INFO - step: 70875 loss: 15.9813 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4367 global_avg_mtp_loss: 13.5446 +[titan] 2025-07-10 05:11:08,479 - root - INFO - lr: 8.3668e-05 gnorm: 0.90 [15:36:06< 6:24:40] +[titan] 2025-07-10 05:11:12,425 - root - INFO - step: 70880 loss: 16.5277 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.9953 +[titan] 2025-07-10 05:11:12,425 - root - INFO - lr: 8.3651e-05 gnorm: 0.94 [15:36:10< 6:24:36] +[titan] 2025-07-10 05:11:16,332 - root - INFO - step: 70885 loss: 16.4982 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9718 +[titan] 2025-07-10 05:11:16,332 - root - INFO - lr: 8.3634e-05 gnorm: 0.94 [15:36:14< 6:24:32] +[titan] 2025-07-10 05:11:20,223 - root - INFO - step: 70890 loss: 16.2296 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7405 +[titan] 2025-07-10 05:11:20,223 - root - INFO - lr: 8.3617e-05 gnorm: 0.92 [15:36:18< 6:24:28] +[titan] 2025-07-10 05:11:24,170 - root - INFO - step: 70895 loss: 16.5069 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9734 +[titan] 2025-07-10 05:11:24,170 - root - INFO - lr: 8.3600e-05 gnorm: 0.96 [15:36:22< 6:24:24] +[titan] 2025-07-10 05:11:27,317 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:11:28,119 - root - INFO - step: 70900 loss: 16.8374 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.5995 global_avg_mtp_loss: 14.2379 +[titan] 2025-07-10 05:11:28,119 - root - INFO - lr: 8.3583e-05 gnorm: 1.03 [15:36:26< 6:24:20] +[titan] 2025-07-10 05:11:32,031 - root - INFO - step: 70905 loss: 15.9382 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4388 global_avg_mtp_loss: 13.4995 +[titan] 2025-07-10 05:11:32,031 - root - INFO - lr: 8.3566e-05 gnorm: 0.94 [15:36:30< 6:24:16] +[titan] 2025-07-10 05:11:35,958 - root - INFO - step: 70910 loss: 16.6132 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 14.0741 +[titan] 2025-07-10 05:11:35,958 - root - INFO - lr: 8.3549e-05 gnorm: 0.91 [15:36:34< 6:24:12] +[titan] 2025-07-10 05:11:39,896 - root - INFO - step: 70915 loss: 16.1281 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6717 +[titan] 2025-07-10 05:11:39,896 - root - INFO - lr: 8.3532e-05 gnorm: 0.96 [15:36:38< 6:24:08] +[titan] 2025-07-10 05:11:43,809 - root - INFO - step: 70920 loss: 16.0991 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6395 +[titan] 2025-07-10 05:11:43,810 - root - INFO - lr: 8.3515e-05 gnorm: 0.93 [15:36:41< 6:24:05] +[titan] 2025-07-10 05:11:47,749 - root - INFO - step: 70925 loss: 16.3035 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8016 +[titan] 2025-07-10 05:11:47,749 - root - INFO - lr: 8.3498e-05 gnorm: 0.89 [15:36:45< 6:24:01] +[titan] 2025-07-10 05:11:51,655 - root - INFO - step: 70930 loss: 16.1301 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.6692 +[titan] 2025-07-10 05:11:51,655 - root - INFO - lr: 8.3481e-05 gnorm: 0.90 [15:36:49< 6:23:57] +[titan] 2025-07-10 05:11:55,620 - root - INFO - step: 70935 loss: 16.6106 memory: 44.58GiB(31.99%) tps: 82,649 tflops: 285.23 mfu: 28.84% global_avg_ntp_loss: 2.5550 global_avg_mtp_loss: 14.0556 +[titan] 2025-07-10 05:11:55,620 - root - INFO - lr: 8.3463e-05 gnorm: 0.90 [15:36:53< 6:23:53] +[titan] 2025-07-10 05:11:59,556 - root - INFO - step: 70940 loss: 16.4793 memory: 44.58GiB(31.99%) tps: 83,264 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9593 +[titan] 2025-07-10 05:11:59,556 - root - INFO - lr: 8.3446e-05 gnorm: 0.91 [15:36:57< 6:23:49] +[titan] 2025-07-10 05:12:03,615 - root - INFO - step: 70945 loss: 16.3058 memory: 44.58GiB(31.99%) tps: 80,733 tflops: 278.62 mfu: 28.17% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8071 +[titan] 2025-07-10 05:12:03,616 - root - INFO - lr: 8.3429e-05 gnorm: 0.91 [15:37:01< 6:23:45] +[titan] 2025-07-10 05:12:06,751 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:12:07,542 - root - INFO - step: 70950 loss: 16.2151 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7423 +[titan] 2025-07-10 05:12:07,542 - root - INFO - lr: 8.3412e-05 gnorm: 0.92 [15:37:05< 6:23:41] +[titan] 2025-07-10 05:12:11,515 - root - INFO - step: 70955 loss: 16.4211 memory: 44.58GiB(31.99%) tps: 82,489 tflops: 284.68 mfu: 28.78% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.8951 +[titan] 2025-07-10 05:12:11,515 - root - INFO - lr: 8.3395e-05 gnorm: 0.91 [15:37:09< 6:23:37] +[titan] 2025-07-10 05:12:15,438 - root - INFO - step: 70960 loss: 16.1787 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7056 +[titan] 2025-07-10 05:12:15,438 - root - INFO - lr: 8.3378e-05 gnorm: 0.97 [15:37:13< 6:23:33] +[titan] 2025-07-10 05:12:19,344 - root - INFO - step: 70965 loss: 16.3611 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8469 +[titan] 2025-07-10 05:12:19,345 - root - INFO - lr: 8.3361e-05 gnorm: 0.93 [15:37:17< 6:23:29] +[titan] 2025-07-10 05:12:23,242 - root - INFO - step: 70970 loss: 16.3063 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.7996 +[titan] 2025-07-10 05:12:23,242 - root - INFO - lr: 8.3344e-05 gnorm: 0.92 [15:37:21< 6:23:25] +[titan] 2025-07-10 05:12:27,137 - root - INFO - step: 70975 loss: 16.0403 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.5928 +[titan] 2025-07-10 05:12:27,137 - root - INFO - lr: 8.3327e-05 gnorm: 0.99 [15:37:25< 6:23:21] +[titan] 2025-07-10 05:12:31,053 - root - INFO - step: 70980 loss: 16.1632 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.7046 +[titan] 2025-07-10 05:12:31,053 - root - INFO - lr: 8.3310e-05 gnorm: 0.91 [15:37:29< 6:23:17] +[titan] 2025-07-10 05:12:34,955 - root - INFO - step: 70985 loss: 16.3622 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 05:12:34,955 - root - INFO - lr: 8.3293e-05 gnorm: 0.89 [15:37:33< 6:23:13] +[titan] 2025-07-10 05:12:38,868 - root - INFO - step: 70990 loss: 16.3885 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.22% global_avg_ntp_loss: 2.5258 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 05:12:38,868 - root - INFO - lr: 8.3276e-05 gnorm: 0.87 [15:37:36< 6:23:09] +[titan] 2025-07-10 05:12:42,772 - root - INFO - step: 70995 loss: 16.3716 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8763 +[titan] 2025-07-10 05:12:42,772 - root - INFO - lr: 8.3259e-05 gnorm: 0.88 [15:37:40< 6:23:05] +[titan] 2025-07-10 05:12:45,900 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:12:46,694 - root - INFO - step: 71000 loss: 16.4641 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5210 global_avg_mtp_loss: 13.9431 +[titan] 2025-07-10 05:12:46,694 - root - INFO - lr: 8.3242e-05 gnorm: 0.86 [15:37:44< 6:23:01] +[titan] 2025-07-10 05:12:50,636 - root - INFO - step: 71005 loss: 16.1454 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6792 +[titan] 2025-07-10 05:12:50,637 - root - INFO - lr: 8.3225e-05 gnorm: 0.91 [15:37:48< 6:22:57] +[titan] 2025-07-10 05:12:54,578 - root - INFO - step: 71010 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8906 +[titan] 2025-07-10 05:12:54,578 - root - INFO - lr: 8.3208e-05 gnorm: 0.89 [15:37:52< 6:22:53] +[titan] 2025-07-10 05:12:58,503 - root - INFO - step: 71015 loss: 16.3823 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8683 +[titan] 2025-07-10 05:12:58,504 - root - INFO - lr: 8.3191e-05 gnorm: 0.95 [15:37:56< 6:22:49] +[titan] 2025-07-10 05:13:02,419 - root - INFO - step: 71020 loss: 16.1414 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.6745 +[titan] 2025-07-10 05:13:02,419 - root - INFO - lr: 8.3174e-05 gnorm: 0.90 [15:38:00< 6:22:45] +[titan] 2025-07-10 05:13:06,364 - root - INFO - step: 71025 loss: 16.3648 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8646 +[titan] 2025-07-10 05:13:06,364 - root - INFO - lr: 8.3156e-05 gnorm: 0.93 [15:38:04< 6:22:41] +[titan] 2025-07-10 05:13:10,260 - root - INFO - step: 71030 loss: 16.5142 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5381 global_avg_mtp_loss: 13.9761 +[titan] 2025-07-10 05:13:10,260 - root - INFO - lr: 8.3139e-05 gnorm: 0.97 [15:38:08< 6:22:37] +[titan] 2025-07-10 05:13:14,181 - root - INFO - step: 71035 loss: 16.1817 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.7103 +[titan] 2025-07-10 05:13:14,181 - root - INFO - lr: 8.3122e-05 gnorm: 0.97 [15:38:12< 6:22:33] +[titan] 2025-07-10 05:13:18,120 - root - INFO - step: 71040 loss: 16.0779 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.6354 +[titan] 2025-07-10 05:13:18,120 - root - INFO - lr: 8.3105e-05 gnorm: 0.98 [15:38:16< 6:22:29] +[titan] 2025-07-10 05:13:22,016 - root - INFO - step: 71045 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6819 +[titan] 2025-07-10 05:13:22,017 - root - INFO - lr: 8.3088e-05 gnorm: 1.01 [15:38:20< 6:22:25] +[titan] 2025-07-10 05:13:25,168 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:13:25,955 - root - INFO - step: 71050 loss: 16.0245 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.16 mfu: 29.03% global_avg_ntp_loss: 2.4419 global_avg_mtp_loss: 13.5826 +[titan] 2025-07-10 05:13:25,955 - root - INFO - lr: 8.3071e-05 gnorm: 0.94 [15:38:24< 6:22:21] +[titan] 2025-07-10 05:13:29,877 - root - INFO - step: 71055 loss: 16.1644 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.7013 +[titan] 2025-07-10 05:13:29,877 - root - INFO - lr: 8.3054e-05 gnorm: 0.94 [15:38:27< 6:22:17] +[titan] 2025-07-10 05:13:33,839 - root - INFO - step: 71060 loss: 16.4373 memory: 44.58GiB(31.99%) tps: 82,703 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.9350 +[titan] 2025-07-10 05:13:33,840 - root - INFO - lr: 8.3037e-05 gnorm: 0.95 [15:38:31< 6:22:13] +[titan] 2025-07-10 05:13:37,779 - root - INFO - step: 71065 loss: 16.3372 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8392 +[titan] 2025-07-10 05:13:37,779 - root - INFO - lr: 8.3020e-05 gnorm: 0.93 [15:38:35< 6:22:09] +[titan] 2025-07-10 05:13:41,746 - root - INFO - step: 71070 loss: 16.4940 memory: 44.58GiB(31.99%) tps: 82,618 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 13.9575 +[titan] 2025-07-10 05:13:41,746 - root - INFO - lr: 8.3003e-05 gnorm: 0.89 [15:38:39< 6:22:05] +[titan] 2025-07-10 05:13:45,689 - root - INFO - step: 71075 loss: 16.2644 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 05:13:45,689 - root - INFO - lr: 8.2986e-05 gnorm: 0.94 [15:38:43< 6:22:01] +[titan] 2025-07-10 05:13:49,618 - root - INFO - step: 71080 loss: 16.1031 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6444 +[titan] 2025-07-10 05:13:49,618 - root - INFO - lr: 8.2969e-05 gnorm: 0.89 [15:38:47< 6:21:57] +[titan] 2025-07-10 05:13:53,558 - root - INFO - step: 71085 loss: 16.4678 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5346 global_avg_mtp_loss: 13.9332 +[titan] 2025-07-10 05:13:53,558 - root - INFO - lr: 8.2952e-05 gnorm: 0.89 [15:38:51< 6:21:53] +[titan] 2025-07-10 05:13:57,482 - root - INFO - step: 71090 loss: 16.0759 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6074 +[titan] 2025-07-10 05:13:57,482 - root - INFO - lr: 8.2935e-05 gnorm: 0.89 [15:38:55< 6:21:49] +[titan] 2025-07-10 05:14:01,428 - root - INFO - step: 71095 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 83,049 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8637 +[titan] 2025-07-10 05:14:01,428 - root - INFO - lr: 8.2918e-05 gnorm: 0.93 [15:38:59< 6:21:45] +[titan] 2025-07-10 05:14:04,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:14:05,352 - root - INFO - step: 71100 loss: 16.0807 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.6219 +[titan] 2025-07-10 05:14:05,352 - root - INFO - lr: 8.2901e-05 gnorm: 0.95 [15:39:03< 6:21:41] +[titan] 2025-07-10 05:14:09,264 - root - INFO - step: 71105 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7889 +[titan] 2025-07-10 05:14:09,265 - root - INFO - lr: 8.2884e-05 gnorm: 0.93 [15:39:07< 6:21:37] +[titan] 2025-07-10 05:14:13,177 - root - INFO - step: 71110 loss: 16.0588 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.22% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6035 +[titan] 2025-07-10 05:14:13,178 - root - INFO - lr: 8.2867e-05 gnorm: 0.89 [15:39:11< 6:21:33] +[titan] 2025-07-10 05:14:17,074 - root - INFO - step: 71115 loss: 16.3900 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.5206 global_avg_mtp_loss: 13.8694 +[titan] 2025-07-10 05:14:17,075 - root - INFO - lr: 8.2850e-05 gnorm: 0.90 [15:39:15< 6:21:29] +[titan] 2025-07-10 05:14:21,017 - root - INFO - step: 71120 loss: 16.4093 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.8818 +[titan] 2025-07-10 05:14:21,018 - root - INFO - lr: 8.2833e-05 gnorm: 0.92 [15:39:19< 6:21:25] +[titan] 2025-07-10 05:14:24,938 - root - INFO - step: 71125 loss: 16.4650 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-10 05:14:24,938 - root - INFO - lr: 8.2816e-05 gnorm: 0.98 [15:39:23< 6:21:22] +[titan] 2025-07-10 05:14:28,846 - root - INFO - step: 71130 loss: 16.6263 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5612 global_avg_mtp_loss: 14.0651 +[titan] 2025-07-10 05:14:28,847 - root - INFO - lr: 8.2799e-05 gnorm: 0.92 [15:39:26< 6:21:18] +[titan] 2025-07-10 05:14:32,786 - root - INFO - step: 71135 loss: 16.1525 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.6847 +[titan] 2025-07-10 05:14:32,786 - root - INFO - lr: 8.2782e-05 gnorm: 0.92 [15:39:30< 6:21:14] +[titan] 2025-07-10 05:14:36,703 - root - INFO - step: 71140 loss: 16.1921 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7180 +[titan] 2025-07-10 05:14:36,703 - root - INFO - lr: 8.2765e-05 gnorm: 0.88 [15:39:34< 6:21:10] +[titan] 2025-07-10 05:14:40,610 - root - INFO - step: 71145 loss: 16.1869 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7066 +[titan] 2025-07-10 05:14:40,610 - root - INFO - lr: 8.2748e-05 gnorm: 0.88 [15:39:38< 6:21:06] +[titan] 2025-07-10 05:14:43,724 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:14:44,525 - root - INFO - step: 71150 loss: 16.1449 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.6669 +[titan] 2025-07-10 05:14:44,525 - root - INFO - lr: 8.2731e-05 gnorm: 0.90 [15:39:42< 6:21:02] +[titan] 2025-07-10 05:14:48,434 - root - INFO - step: 71155 loss: 16.1492 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.6604 +[titan] 2025-07-10 05:14:48,434 - root - INFO - lr: 8.2714e-05 gnorm: 0.90 [15:39:46< 6:20:58] +[titan] 2025-07-10 05:14:52,340 - root - INFO - step: 71160 loss: 16.0844 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4592 global_avg_mtp_loss: 13.6252 +[titan] 2025-07-10 05:14:52,340 - root - INFO - lr: 8.2697e-05 gnorm: 0.90 [15:39:50< 6:20:54] +[titan] 2025-07-10 05:14:56,302 - root - INFO - step: 71165 loss: 16.1809 memory: 44.58GiB(31.99%) tps: 82,720 tflops: 285.48 mfu: 28.87% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7075 +[titan] 2025-07-10 05:14:56,302 - root - INFO - lr: 8.2680e-05 gnorm: 0.91 [15:39:54< 6:20:50] +[titan] 2025-07-10 05:14:58,799 - root - INFO - Dumping profiler traces at step 71168 +[titan] 2025-07-10 05:14:58,830 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:15:00,399 - root - INFO - step: 71170 loss: 16.2284 memory: 44.58GiB(31.99%) tps: 79,978 tflops: 276.02 mfu: 27.91% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7503 +[titan] 2025-07-10 05:15:00,399 - root - INFO - lr: 8.2663e-05 gnorm: 0.98 [15:39:58< 6:20:46] +[titan] 2025-07-10 05:15:04,316 - root - INFO - step: 71175 loss: 16.4418 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.9228 +[titan] 2025-07-10 05:15:04,316 - root - INFO - lr: 8.2646e-05 gnorm: 0.89 [15:40:02< 6:20:42] +[titan] 2025-07-10 05:15:08,218 - root - INFO - step: 71180 loss: 16.2816 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.7734 +[titan] 2025-07-10 05:15:08,218 - root - INFO - lr: 8.2629e-05 gnorm: 0.92 [15:40:06< 6:20:38] +[titan] 2025-07-10 05:15:12,109 - root - INFO - step: 71185 loss: 16.3869 memory: 44.58GiB(31.99%) tps: 84,229 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.8709 +[titan] 2025-07-10 05:15:12,109 - root - INFO - lr: 8.2612e-05 gnorm: 0.96 [15:40:10< 6:20:34] +[titan] 2025-07-10 05:15:16,032 - root - INFO - step: 71190 loss: 16.2445 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7661 +[titan] 2025-07-10 05:15:16,032 - root - INFO - lr: 8.2595e-05 gnorm: 0.88 [15:40:14< 6:20:30] +[titan] 2025-07-10 05:15:19,989 - root - INFO - step: 71195 loss: 16.1155 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.6586 +[titan] 2025-07-10 05:15:19,989 - root - INFO - lr: 8.2579e-05 gnorm: 0.98 [15:40:18< 6:20:26] +[titan] 2025-07-10 05:15:23,110 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:15:23,907 - root - INFO - step: 71200 loss: 16.1475 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.6750 +[titan] 2025-07-10 05:15:23,907 - root - INFO - lr: 8.2562e-05 gnorm: 0.91 [15:40:21< 6:20:22] +[titan] 2025-07-10 05:15:27,808 - root - INFO - step: 71205 loss: 16.3424 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8302 +[titan] 2025-07-10 05:15:27,809 - root - INFO - lr: 8.2545e-05 gnorm: 0.90 [15:40:25< 6:20:18] +[titan] 2025-07-10 05:15:31,723 - root - INFO - step: 71210 loss: 15.9768 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4306 global_avg_mtp_loss: 13.5462 +[titan] 2025-07-10 05:15:31,723 - root - INFO - lr: 8.2528e-05 gnorm: 0.91 [15:40:29< 6:20:14] +[titan] 2025-07-10 05:15:35,647 - root - INFO - step: 71215 loss: 16.3243 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 05:15:35,647 - root - INFO - lr: 8.2511e-05 gnorm: 0.91 [15:40:33< 6:20:10] +[titan] 2025-07-10 05:15:39,554 - root - INFO - step: 71220 loss: 16.4046 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8963 +[titan] 2025-07-10 05:15:39,554 - root - INFO - lr: 8.2494e-05 gnorm: 0.93 [15:40:37< 6:20:06] +[titan] 2025-07-10 05:15:43,479 - root - INFO - step: 71225 loss: 16.3559 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 05:15:43,479 - root - INFO - lr: 8.2477e-05 gnorm: 0.91 [15:40:41< 6:20:02] +[titan] 2025-07-10 05:15:47,432 - root - INFO - step: 71230 loss: 16.0308 memory: 44.58GiB(31.99%) tps: 82,895 tflops: 286.09 mfu: 28.93% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.5873 +[titan] 2025-07-10 05:15:47,433 - root - INFO - lr: 8.2460e-05 gnorm: 0.96 [15:40:45< 6:19:58] +[titan] 2025-07-10 05:15:51,344 - root - INFO - step: 71235 loss: 16.3107 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 05:15:51,344 - root - INFO - lr: 8.2443e-05 gnorm: 0.92 [15:40:49< 6:19:54] +[titan] 2025-07-10 05:15:55,298 - root - INFO - step: 71240 loss: 16.4747 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5477 global_avg_mtp_loss: 13.9270 +[titan] 2025-07-10 05:15:55,299 - root - INFO - lr: 8.2426e-05 gnorm: 1.09 [15:40:53< 6:19:50] +[titan] 2025-07-10 05:15:59,227 - root - INFO - step: 71245 loss: 16.2102 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7342 +[titan] 2025-07-10 05:15:59,227 - root - INFO - lr: 8.2409e-05 gnorm: 1.05 [15:40:57< 6:19:46] +[titan] 2025-07-10 05:16:02,381 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:16:03,165 - root - INFO - step: 71250 loss: 16.4248 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.8974 +[titan] 2025-07-10 05:16:03,165 - root - INFO - lr: 8.2392e-05 gnorm: 1.01 [15:41:01< 6:19:42] +[titan] 2025-07-10 05:16:07,070 - root - INFO - step: 71255 loss: 16.4992 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5354 global_avg_mtp_loss: 13.9638 +[titan] 2025-07-10 05:16:07,070 - root - INFO - lr: 8.2375e-05 gnorm: 0.88 [15:41:05< 6:19:38] +[titan] 2025-07-10 05:16:11,042 - root - INFO - step: 71260 loss: 16.3538 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8488 +[titan] 2025-07-10 05:16:11,042 - root - INFO - lr: 8.2358e-05 gnorm: 0.93 [15:41:09< 6:19:34] +[titan] 2025-07-10 05:16:14,983 - root - INFO - step: 71265 loss: 16.2552 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 05:16:14,983 - root - INFO - lr: 8.2341e-05 gnorm: 0.89 [15:41:13< 6:19:30] +[titan] 2025-07-10 05:16:18,902 - root - INFO - step: 71270 loss: 16.4936 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.9717 +[titan] 2025-07-10 05:16:18,902 - root - INFO - lr: 8.2324e-05 gnorm: 0.95 [15:41:16< 6:19:26] +[titan] 2025-07-10 05:16:22,836 - root - INFO - step: 71275 loss: 16.2811 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 05:16:22,836 - root - INFO - lr: 8.2307e-05 gnorm: 0.99 [15:41:20< 6:19:22] +[titan] 2025-07-10 05:16:26,749 - root - INFO - step: 71280 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.8011 +[titan] 2025-07-10 05:16:26,749 - root - INFO - lr: 8.2290e-05 gnorm: 0.94 [15:41:24< 6:19:18] +[titan] 2025-07-10 05:16:30,671 - root - INFO - step: 71285 loss: 16.3809 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8737 +[titan] 2025-07-10 05:16:30,671 - root - INFO - lr: 8.2273e-05 gnorm: 0.93 [15:41:28< 6:19:14] +[titan] 2025-07-10 05:16:34,571 - root - INFO - step: 71290 loss: 16.3193 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.8249 +[titan] 2025-07-10 05:16:34,571 - root - INFO - lr: 8.2257e-05 gnorm: 0.91 [15:41:32< 6:19:10] +[titan] 2025-07-10 05:16:38,491 - root - INFO - step: 71295 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8595 +[titan] 2025-07-10 05:16:38,491 - root - INFO - lr: 8.2240e-05 gnorm: 0.98 [15:41:36< 6:19:06] +[titan] 2025-07-10 05:16:41,604 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:16:42,402 - root - INFO - step: 71300 loss: 16.1613 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.6846 +[titan] 2025-07-10 05:16:42,402 - root - INFO - lr: 8.2223e-05 gnorm: 0.92 [15:41:40< 6:19:02] +[titan] 2025-07-10 05:16:46,305 - root - INFO - step: 71305 loss: 16.0114 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.5657 +[titan] 2025-07-10 05:16:46,305 - root - INFO - lr: 8.2206e-05 gnorm: 0.97 [15:41:44< 6:18:58] +[titan] 2025-07-10 05:16:50,221 - root - INFO - step: 71310 loss: 16.3408 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8439 +[titan] 2025-07-10 05:16:50,221 - root - INFO - lr: 8.2189e-05 gnorm: 0.91 [15:41:48< 6:18:54] +[titan] 2025-07-10 05:16:54,149 - root - INFO - step: 71315 loss: 16.4479 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9203 +[titan] 2025-07-10 05:16:54,149 - root - INFO - lr: 8.2172e-05 gnorm: 0.99 [15:41:52< 6:18:50] +[titan] 2025-07-10 05:16:58,093 - root - INFO - step: 71320 loss: 16.4425 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 05:16:58,093 - root - INFO - lr: 8.2155e-05 gnorm: 0.91 [15:41:56< 6:18:46] +[titan] 2025-07-10 05:17:02,015 - root - INFO - step: 71325 loss: 15.9940 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5514 +[titan] 2025-07-10 05:17:02,015 - root - INFO - lr: 8.2138e-05 gnorm: 0.92 [15:42:00< 6:18:42] +[titan] 2025-07-10 05:17:05,925 - root - INFO - step: 71330 loss: 16.0683 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6006 +[titan] 2025-07-10 05:17:05,925 - root - INFO - lr: 8.2121e-05 gnorm: 0.99 [15:42:04< 6:18:38] +[titan] 2025-07-10 05:17:09,849 - root - INFO - step: 71335 loss: 16.1212 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.6471 +[titan] 2025-07-10 05:17:09,849 - root - INFO - lr: 8.2104e-05 gnorm: 0.89 [15:42:07< 6:18:34] +[titan] 2025-07-10 05:17:13,743 - root - INFO - step: 71340 loss: 16.3835 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8767 +[titan] 2025-07-10 05:17:13,743 - root - INFO - lr: 8.2087e-05 gnorm: 0.90 [15:42:11< 6:18:30] +[titan] 2025-07-10 05:17:17,650 - root - INFO - step: 71345 loss: 16.1046 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6396 +[titan] 2025-07-10 05:17:17,650 - root - INFO - lr: 8.2071e-05 gnorm: 0.93 [15:42:15< 6:18:27] +[titan] 2025-07-10 05:17:20,767 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:17:21,568 - root - INFO - step: 71350 loss: 16.3770 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8839 +[titan] 2025-07-10 05:17:21,568 - root - INFO - lr: 8.2054e-05 gnorm: 0.89 [15:42:19< 6:18:23] +[titan] 2025-07-10 05:17:25,480 - root - INFO - step: 71355 loss: 16.4502 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5334 global_avg_mtp_loss: 13.9169 +[titan] 2025-07-10 05:17:25,480 - root - INFO - lr: 8.2037e-05 gnorm: 0.91 [15:42:23< 6:18:19] +[titan] 2025-07-10 05:17:29,410 - root - INFO - step: 71360 loss: 16.4962 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.9710 +[titan] 2025-07-10 05:17:29,411 - root - INFO - lr: 8.2020e-05 gnorm: 0.90 [15:42:27< 6:18:15] +[titan] 2025-07-10 05:17:33,357 - root - INFO - step: 71365 loss: 16.3497 memory: 44.58GiB(31.99%) tps: 83,034 tflops: 286.56 mfu: 28.98% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8550 +[titan] 2025-07-10 05:17:33,357 - root - INFO - lr: 8.2003e-05 gnorm: 0.90 [15:42:31< 6:18:11] +[titan] 2025-07-10 05:17:37,263 - root - INFO - step: 71370 loss: 16.4588 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9324 +[titan] 2025-07-10 05:17:37,263 - root - INFO - lr: 8.1986e-05 gnorm: 0.95 [15:42:35< 6:18:07] +[titan] 2025-07-10 05:17:41,207 - root - INFO - step: 71375 loss: 16.3592 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8434 +[titan] 2025-07-10 05:17:41,207 - root - INFO - lr: 8.1969e-05 gnorm: 0.92 [15:42:39< 6:18:03] +[titan] 2025-07-10 05:17:45,140 - root - INFO - step: 71380 loss: 15.9925 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4400 global_avg_mtp_loss: 13.5525 +[titan] 2025-07-10 05:17:45,141 - root - INFO - lr: 8.1952e-05 gnorm: 0.97 [15:42:43< 6:17:59] +[titan] 2025-07-10 05:17:49,075 - root - INFO - step: 71385 loss: 16.2797 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 05:17:49,075 - root - INFO - lr: 8.1935e-05 gnorm: 0.93 [15:42:47< 6:17:55] +[titan] 2025-07-10 05:17:53,029 - root - INFO - step: 71390 loss: 16.3449 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.8565 +[titan] 2025-07-10 05:17:53,029 - root - INFO - lr: 8.1918e-05 gnorm: 0.95 [15:42:51< 6:17:51] +[titan] 2025-07-10 05:17:56,962 - root - INFO - step: 71395 loss: 16.3833 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 05:17:56,962 - root - INFO - lr: 8.1902e-05 gnorm: 0.94 [15:42:55< 6:17:47] +[titan] 2025-07-10 05:18:00,101 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:18:00,914 - root - INFO - step: 71400 loss: 16.3319 memory: 44.58GiB(31.99%) tps: 82,911 tflops: 286.14 mfu: 28.93% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 05:18:00,915 - root - INFO - lr: 8.1885e-05 gnorm: 0.96 [15:42:58< 6:17:43] +[titan] 2025-07-10 05:18:04,844 - root - INFO - step: 71405 loss: 16.3961 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8852 +[titan] 2025-07-10 05:18:04,844 - root - INFO - lr: 8.1868e-05 gnorm: 0.95 [15:43:02< 6:17:39] +[titan] 2025-07-10 05:18:08,764 - root - INFO - step: 71410 loss: 16.2366 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 05:18:08,764 - root - INFO - lr: 8.1851e-05 gnorm: 0.92 [15:43:06< 6:17:35] +[titan] 2025-07-10 05:18:12,671 - root - INFO - step: 71415 loss: 16.1428 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.6800 +[titan] 2025-07-10 05:18:12,672 - root - INFO - lr: 8.1834e-05 gnorm: 1.04 [15:43:10< 6:17:31] +[titan] 2025-07-10 05:18:16,584 - root - INFO - step: 71420 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.6921 +[titan] 2025-07-10 05:18:16,584 - root - INFO - lr: 8.1817e-05 gnorm: 0.99 [15:43:14< 6:17:27] +[titan] 2025-07-10 05:18:20,502 - root - INFO - step: 71425 loss: 16.2447 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7575 +[titan] 2025-07-10 05:18:20,502 - root - INFO - lr: 8.1800e-05 gnorm: 0.99 [15:43:18< 6:17:23] +[titan] 2025-07-10 05:18:24,405 - root - INFO - step: 71430 loss: 16.3565 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8625 +[titan] 2025-07-10 05:18:24,406 - root - INFO - lr: 8.1783e-05 gnorm: 0.92 [15:43:22< 6:17:19] +[titan] 2025-07-10 05:18:28,362 - root - INFO - step: 71435 loss: 16.4605 memory: 44.58GiB(31.99%) tps: 82,829 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9308 +[titan] 2025-07-10 05:18:28,362 - root - INFO - lr: 8.1767e-05 gnorm: 0.88 [15:43:26< 6:17:15] +[titan] 2025-07-10 05:18:32,326 - root - INFO - step: 71440 loss: 16.1646 memory: 44.58GiB(31.99%) tps: 82,657 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.6951 +[titan] 2025-07-10 05:18:32,327 - root - INFO - lr: 8.1750e-05 gnorm: 0.94 [15:43:30< 6:17:11] +[titan] 2025-07-10 05:18:36,243 - root - INFO - step: 71445 loss: 16.4282 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.9106 +[titan] 2025-07-10 05:18:36,243 - root - INFO - lr: 8.1733e-05 gnorm: 0.88 [15:43:34< 6:17:07] +[titan] 2025-07-10 05:18:39,361 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:18:40,152 - root - INFO - step: 71450 loss: 16.3629 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8481 +[titan] 2025-07-10 05:18:40,152 - root - INFO - lr: 8.1716e-05 gnorm: 0.93 [15:43:38< 6:17:03] +[titan] 2025-07-10 05:18:44,070 - root - INFO - step: 71455 loss: 16.3405 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8425 +[titan] 2025-07-10 05:18:44,070 - root - INFO - lr: 8.1699e-05 gnorm: 0.93 [15:43:42< 6:16:59] +[titan] 2025-07-10 05:18:47,984 - root - INFO - step: 71460 loss: 16.1094 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6436 +[titan] 2025-07-10 05:18:47,984 - root - INFO - lr: 8.1682e-05 gnorm: 0.93 [15:43:46< 6:16:55] +[titan] 2025-07-10 05:18:51,912 - root - INFO - step: 71465 loss: 16.2987 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.8128 +[titan] 2025-07-10 05:18:51,912 - root - INFO - lr: 8.1665e-05 gnorm: 0.88 [15:43:49< 6:16:51] +[titan] 2025-07-10 05:18:55,828 - root - INFO - step: 71470 loss: 16.3248 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 05:18:55,828 - root - INFO - lr: 8.1649e-05 gnorm: 0.89 [15:43:53< 6:16:47] +[titan] 2025-07-10 05:18:59,762 - root - INFO - step: 71475 loss: 16.3476 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 05:18:59,762 - root - INFO - lr: 8.1632e-05 gnorm: 0.92 [15:43:57< 6:16:43] +[titan] 2025-07-10 05:19:03,696 - root - INFO - step: 71480 loss: 16.3544 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8513 +[titan] 2025-07-10 05:19:03,696 - root - INFO - lr: 8.1615e-05 gnorm: 0.95 [15:44:01< 6:16:39] +[titan] 2025-07-10 05:19:07,615 - root - INFO - step: 71485 loss: 16.1754 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.7108 +[titan] 2025-07-10 05:19:07,616 - root - INFO - lr: 8.1598e-05 gnorm: 0.96 [15:44:05< 6:16:35] +[titan] 2025-07-10 05:19:11,508 - root - INFO - step: 71490 loss: 16.4381 memory: 44.58GiB(31.99%) tps: 84,180 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.9282 +[titan] 2025-07-10 05:19:11,509 - root - INFO - lr: 8.1581e-05 gnorm: 0.93 [15:44:09< 6:16:31] +[titan] 2025-07-10 05:19:15,414 - root - INFO - step: 71495 loss: 16.2082 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7327 +[titan] 2025-07-10 05:19:15,414 - root - INFO - lr: 8.1564e-05 gnorm: 0.93 [15:44:13< 6:16:27] +[titan] 2025-07-10 05:19:18,532 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:19:19,332 - root - INFO - step: 71500 loss: 16.4003 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.8858 +[titan] 2025-07-10 05:19:19,332 - root - INFO - lr: 8.1548e-05 gnorm: 0.91 [15:44:17< 6:16:23] +[titan] 2025-07-10 05:19:23,252 - root - INFO - step: 71505 loss: 16.0855 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6327 +[titan] 2025-07-10 05:19:23,252 - root - INFO - lr: 8.1531e-05 gnorm: 0.92 [15:44:21< 6:16:19] +[titan] 2025-07-10 05:19:27,200 - root - INFO - step: 71510 loss: 16.1850 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7159 +[titan] 2025-07-10 05:19:27,201 - root - INFO - lr: 8.1514e-05 gnorm: 0.94 [15:44:25< 6:16:15] +[titan] 2025-07-10 05:19:31,137 - root - INFO - step: 71515 loss: 16.1951 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7074 +[titan] 2025-07-10 05:19:31,137 - root - INFO - lr: 8.1497e-05 gnorm: 0.91 [15:44:29< 6:16:11] +[titan] 2025-07-10 05:19:35,059 - root - INFO - step: 71520 loss: 16.2686 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7797 +[titan] 2025-07-10 05:19:35,059 - root - INFO - lr: 8.1480e-05 gnorm: 0.93 [15:44:33< 6:16:07] +[titan] 2025-07-10 05:19:38,992 - root - INFO - step: 71525 loss: 16.5530 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 14.0129 +[titan] 2025-07-10 05:19:38,992 - root - INFO - lr: 8.1463e-05 gnorm: 0.89 [15:44:37< 6:16:03] +[titan] 2025-07-10 05:19:43,008 - root - INFO - step: 71530 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 81,593 tflops: 281.59 mfu: 28.47% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.7961 +[titan] 2025-07-10 05:19:43,008 - root - INFO - lr: 8.1447e-05 gnorm: 0.93 [15:44:41< 6:15:59] +[titan] 2025-07-10 05:19:46,924 - root - INFO - step: 71535 loss: 16.4282 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 05:19:46,924 - root - INFO - lr: 8.1430e-05 gnorm: 0.91 [15:44:44< 6:15:55] +[titan] 2025-07-10 05:19:50,840 - root - INFO - step: 71540 loss: 16.3422 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8408 +[titan] 2025-07-10 05:19:50,840 - root - INFO - lr: 8.1413e-05 gnorm: 0.95 [15:44:48< 6:15:51] +[titan] 2025-07-10 05:19:54,789 - root - INFO - step: 71545 loss: 16.5806 memory: 44.58GiB(31.99%) tps: 82,986 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5459 global_avg_mtp_loss: 14.0348 +[titan] 2025-07-10 05:19:54,789 - root - INFO - lr: 8.1396e-05 gnorm: 0.90 [15:44:52< 6:15:47] +[titan] 2025-07-10 05:19:57,890 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:19:58,684 - root - INFO - step: 71550 loss: 16.3122 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8145 +[titan] 2025-07-10 05:19:58,684 - root - INFO - lr: 8.1379e-05 gnorm: 0.96 [15:44:56< 6:15:43] +[titan] 2025-07-10 05:20:02,581 - root - INFO - step: 71555 loss: 16.4268 memory: 44.58GiB(31.99%) tps: 84,094 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 05:20:02,581 - root - INFO - lr: 8.1362e-05 gnorm: 1.01 [15:45:00< 6:15:39] +[titan] 2025-07-10 05:20:06,492 - root - INFO - step: 71560 loss: 16.0792 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.6182 +[titan] 2025-07-10 05:20:06,492 - root - INFO - lr: 8.1346e-05 gnorm: 0.98 [15:45:04< 6:15:36] +[titan] 2025-07-10 05:20:10,418 - root - INFO - step: 71565 loss: 16.4514 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9299 +[titan] 2025-07-10 05:20:10,418 - root - INFO - lr: 8.1329e-05 gnorm: 0.91 [15:45:08< 6:15:32] +[titan] 2025-07-10 05:20:14,403 - root - INFO - step: 71570 loss: 16.1189 memory: 44.58GiB(31.99%) tps: 82,231 tflops: 283.79 mfu: 28.69% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.6421 +[titan] 2025-07-10 05:20:14,404 - root - INFO - lr: 8.1312e-05 gnorm: 0.96 [15:45:12< 6:15:28] +[titan] 2025-07-10 05:20:18,328 - root - INFO - step: 71575 loss: 16.2625 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7758 +[titan] 2025-07-10 05:20:18,329 - root - INFO - lr: 8.1295e-05 gnorm: 0.98 [15:45:16< 6:15:24] +[titan] 2025-07-10 05:20:22,239 - root - INFO - step: 71580 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.9125 +[titan] 2025-07-10 05:20:22,239 - root - INFO - lr: 8.1278e-05 gnorm: 0.91 [15:45:20< 6:15:20] +[titan] 2025-07-10 05:20:26,203 - root - INFO - step: 71585 loss: 16.2551 memory: 44.58GiB(31.99%) tps: 82,666 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 05:20:26,203 - root - INFO - lr: 8.1262e-05 gnorm: 0.93 [15:45:24< 6:15:16] +[titan] 2025-07-10 05:20:30,133 - root - INFO - step: 71590 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 05:20:30,133 - root - INFO - lr: 8.1245e-05 gnorm: 0.91 [15:45:28< 6:15:12] +[titan] 2025-07-10 05:20:34,095 - root - INFO - step: 71595 loss: 16.6056 memory: 44.58GiB(31.99%) tps: 82,711 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.5512 global_avg_mtp_loss: 14.0543 +[titan] 2025-07-10 05:20:34,095 - root - INFO - lr: 8.1228e-05 gnorm: 0.95 [15:45:32< 6:15:08] +[titan] 2025-07-10 05:20:37,223 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:20:38,056 - root - INFO - step: 71600 loss: 16.1102 memory: 44.58GiB(31.99%) tps: 82,735 tflops: 285.53 mfu: 28.87% global_avg_ntp_loss: 2.4683 global_avg_mtp_loss: 13.6419 +[titan] 2025-07-10 05:20:38,056 - root - INFO - lr: 8.1211e-05 gnorm: 0.94 [15:45:36< 6:15:04] +[titan] 2025-07-10 05:20:41,967 - root - INFO - step: 71605 loss: 16.3586 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8609 +[titan] 2025-07-10 05:20:41,968 - root - INFO - lr: 8.1194e-05 gnorm: 0.92 [15:45:40< 6:15:00] +[titan] 2025-07-10 05:20:45,889 - root - INFO - step: 71610 loss: 16.6933 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5989 global_avg_mtp_loss: 14.0943 +[titan] 2025-07-10 05:20:45,889 - root - INFO - lr: 8.1178e-05 gnorm: 1.02 [15:45:43< 6:14:56] +[titan] 2025-07-10 05:20:49,833 - root - INFO - step: 71615 loss: 16.2718 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.7785 +[titan] 2025-07-10 05:20:49,834 - root - INFO - lr: 8.1161e-05 gnorm: 0.87 [15:45:47< 6:14:52] +[titan] 2025-07-10 05:20:53,765 - root - INFO - step: 71620 loss: 16.2638 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7796 +[titan] 2025-07-10 05:20:53,766 - root - INFO - lr: 8.1144e-05 gnorm: 0.92 [15:45:51< 6:14:48] +[titan] 2025-07-10 05:20:57,799 - root - INFO - step: 71625 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 81,245 tflops: 280.39 mfu: 28.35% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 05:20:57,799 - root - INFO - lr: 8.1127e-05 gnorm: 0.93 [15:45:55< 6:14:44] +[titan] 2025-07-10 05:21:01,736 - root - INFO - step: 71630 loss: 16.3582 memory: 44.58GiB(31.99%) tps: 83,247 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8536 +[titan] 2025-07-10 05:21:01,736 - root - INFO - lr: 8.1111e-05 gnorm: 0.98 [15:45:59< 6:14:40] +[titan] 2025-07-10 05:21:05,657 - root - INFO - step: 71635 loss: 16.1975 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.7131 +[titan] 2025-07-10 05:21:05,657 - root - INFO - lr: 8.1094e-05 gnorm: 0.91 [15:46:03< 6:14:36] +[titan] 2025-07-10 05:21:09,580 - root - INFO - step: 71640 loss: 16.2936 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.7870 +[titan] 2025-07-10 05:21:09,581 - root - INFO - lr: 8.1077e-05 gnorm: 0.97 [15:46:07< 6:14:32] +[titan] 2025-07-10 05:21:13,529 - root - INFO - step: 71645 loss: 16.4649 memory: 44.58GiB(31.99%) tps: 83,001 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9331 +[titan] 2025-07-10 05:21:13,529 - root - INFO - lr: 8.1060e-05 gnorm: 1.00 [15:46:11< 6:14:28] +[titan] 2025-07-10 05:21:16,650 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:21:17,452 - root - INFO - step: 71650 loss: 16.5048 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9776 +[titan] 2025-07-10 05:21:17,452 - root - INFO - lr: 8.1043e-05 gnorm: 0.96 [15:46:15< 6:14:24] +[titan] 2025-07-10 05:21:21,369 - root - INFO - step: 71655 loss: 16.2752 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.7872 +[titan] 2025-07-10 05:21:21,369 - root - INFO - lr: 8.1027e-05 gnorm: 0.95 [15:46:19< 6:14:20] +[titan] 2025-07-10 05:21:25,285 - root - INFO - step: 71660 loss: 16.2450 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7498 +[titan] 2025-07-10 05:21:25,285 - root - INFO - lr: 8.1010e-05 gnorm: 0.92 [15:46:23< 6:14:16] +[titan] 2025-07-10 05:21:29,233 - root - INFO - step: 71665 loss: 16.1061 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.6345 +[titan] 2025-07-10 05:21:29,233 - root - INFO - lr: 8.0993e-05 gnorm: 0.91 [15:46:27< 6:14:12] +[titan] 2025-07-10 05:21:33,127 - root - INFO - step: 71670 loss: 16.4361 memory: 44.58GiB(31.99%) tps: 84,164 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9146 +[titan] 2025-07-10 05:21:33,127 - root - INFO - lr: 8.0976e-05 gnorm: 0.88 [15:46:31< 6:14:08] +[titan] 2025-07-10 05:21:37,050 - root - INFO - step: 71675 loss: 16.2564 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7786 +[titan] 2025-07-10 05:21:37,050 - root - INFO - lr: 8.0960e-05 gnorm: 0.91 [15:46:35< 6:14:04] +[titan] 2025-07-10 05:21:41,062 - root - INFO - step: 71680 loss: 16.1273 memory: 44.58GiB(31.99%) tps: 81,685 tflops: 281.91 mfu: 28.50% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.6620 +[titan] 2025-07-10 05:21:41,062 - root - INFO - lr: 8.0943e-05 gnorm: 0.92 [15:46:39< 6:14:00] +[titan] 2025-07-10 05:21:41,215 - root - INFO - Dumping profiler traces at step 71680 +[titan] 2025-07-10 05:21:41,248 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:21:45,184 - root - INFO - step: 71685 loss: 16.5369 memory: 44.58GiB(31.99%) tps: 79,501 tflops: 274.37 mfu: 27.74% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 14.0021 +[titan] 2025-07-10 05:21:45,184 - root - INFO - lr: 8.0926e-05 gnorm: 0.92 [15:46:43< 6:13:56] +[titan] 2025-07-10 05:21:49,112 - root - INFO - step: 71690 loss: 16.4167 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8943 +[titan] 2025-07-10 05:21:49,112 - root - INFO - lr: 8.0909e-05 gnorm: 0.91 [15:46:47< 6:13:52] +[titan] 2025-07-10 05:21:53,049 - root - INFO - step: 71695 loss: 16.4668 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9540 +[titan] 2025-07-10 05:21:53,049 - root - INFO - lr: 8.0893e-05 gnorm: 0.99 [15:46:51< 6:13:48] +[titan] 2025-07-10 05:21:56,208 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:21:57,007 - root - INFO - step: 71700 loss: 16.5971 memory: 44.58GiB(31.99%) tps: 82,802 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.5425 global_avg_mtp_loss: 14.0547 +[titan] 2025-07-10 05:21:57,007 - root - INFO - lr: 8.0876e-05 gnorm: 0.93 [15:46:55< 6:13:44] +[titan] 2025-07-10 05:22:00,925 - root - INFO - step: 71705 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8377 +[titan] 2025-07-10 05:22:00,926 - root - INFO - lr: 8.0859e-05 gnorm: 0.91 [15:46:58< 6:13:40] +[titan] 2025-07-10 05:22:04,833 - root - INFO - step: 71710 loss: 16.1699 memory: 44.58GiB(31.99%) tps: 83,867 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.7061 +[titan] 2025-07-10 05:22:04,833 - root - INFO - lr: 8.0842e-05 gnorm: 0.91 [15:47:02< 6:13:36] +[titan] 2025-07-10 05:22:08,754 - root - INFO - step: 71715 loss: 16.4043 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.8832 +[titan] 2025-07-10 05:22:08,754 - root - INFO - lr: 8.0826e-05 gnorm: 0.95 [15:47:06< 6:13:32] +[titan] 2025-07-10 05:22:12,697 - root - INFO - step: 71720 loss: 16.3965 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 05:22:12,698 - root - INFO - lr: 8.0809e-05 gnorm: 0.98 [15:47:10< 6:13:28] +[titan] 2025-07-10 05:22:16,643 - root - INFO - step: 71725 loss: 16.3110 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 05:22:16,643 - root - INFO - lr: 8.0792e-05 gnorm: 0.89 [15:47:14< 6:13:25] +[titan] 2025-07-10 05:22:20,546 - root - INFO - step: 71730 loss: 16.6807 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5750 global_avg_mtp_loss: 14.1057 +[titan] 2025-07-10 05:22:20,546 - root - INFO - lr: 8.0775e-05 gnorm: 1.02 [15:47:18< 6:13:21] +[titan] 2025-07-10 05:22:24,453 - root - INFO - step: 71735 loss: 16.1629 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6931 +[titan] 2025-07-10 05:22:24,453 - root - INFO - lr: 8.0759e-05 gnorm: 0.98 [15:47:22< 6:13:17] +[titan] 2025-07-10 05:22:28,405 - root - INFO - step: 71740 loss: 16.3603 memory: 44.58GiB(31.99%) tps: 82,909 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8574 +[titan] 2025-07-10 05:22:28,406 - root - INFO - lr: 8.0742e-05 gnorm: 0.90 [15:47:26< 6:13:13] +[titan] 2025-07-10 05:22:32,306 - root - INFO - step: 71745 loss: 16.3833 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8760 +[titan] 2025-07-10 05:22:32,307 - root - INFO - lr: 8.0725e-05 gnorm: 0.93 [15:47:30< 6:13:09] +[titan] 2025-07-10 05:22:35,432 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:22:36,219 - root - INFO - step: 71750 loss: 16.1087 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.6503 +[titan] 2025-07-10 05:22:36,219 - root - INFO - lr: 8.0708e-05 gnorm: 0.91 [15:47:34< 6:13:05] +[titan] 2025-07-10 05:22:40,116 - root - INFO - step: 71755 loss: 16.2502 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7630 +[titan] 2025-07-10 05:22:40,117 - root - INFO - lr: 8.0692e-05 gnorm: 0.89 [15:47:38< 6:13:01] +[titan] 2025-07-10 05:22:44,030 - root - INFO - step: 71760 loss: 16.2359 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7536 +[titan] 2025-07-10 05:22:44,031 - root - INFO - lr: 8.0675e-05 gnorm: 0.90 [15:47:42< 6:12:57] +[titan] 2025-07-10 05:22:47,931 - root - INFO - step: 71765 loss: 16.3171 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8031 +[titan] 2025-07-10 05:22:47,931 - root - INFO - lr: 8.0658e-05 gnorm: 0.89 [15:47:45< 6:12:53] +[titan] 2025-07-10 05:22:51,849 - root - INFO - step: 71770 loss: 16.1755 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.6876 +[titan] 2025-07-10 05:22:51,850 - root - INFO - lr: 8.0641e-05 gnorm: 0.93 [15:47:49< 6:12:49] +[titan] 2025-07-10 05:22:55,763 - root - INFO - step: 71775 loss: 16.1807 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 05:22:55,764 - root - INFO - lr: 8.0625e-05 gnorm: 0.97 [15:47:53< 6:12:45] +[titan] 2025-07-10 05:22:59,663 - root - INFO - step: 71780 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 05:22:59,663 - root - INFO - lr: 8.0608e-05 gnorm: 0.95 [15:47:57< 6:12:41] +[titan] 2025-07-10 05:23:03,587 - root - INFO - step: 71785 loss: 16.2482 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7672 +[titan] 2025-07-10 05:23:03,587 - root - INFO - lr: 8.0591e-05 gnorm: 0.88 [15:48:01< 6:12:37] +[titan] 2025-07-10 05:23:07,485 - root - INFO - step: 71790 loss: 16.2215 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7391 +[titan] 2025-07-10 05:23:07,485 - root - INFO - lr: 8.0575e-05 gnorm: 0.91 [15:48:05< 6:12:33] +[titan] 2025-07-10 05:23:11,411 - root - INFO - step: 71795 loss: 16.1029 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6399 +[titan] 2025-07-10 05:23:11,412 - root - INFO - lr: 8.0558e-05 gnorm: 0.91 [15:48:09< 6:12:29] +[titan] 2025-07-10 05:23:14,535 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:23:15,321 - root - INFO - step: 71800 loss: 16.6027 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5553 global_avg_mtp_loss: 14.0474 +[titan] 2025-07-10 05:23:15,321 - root - INFO - lr: 8.0541e-05 gnorm: 0.97 [15:48:13< 6:12:25] +[titan] 2025-07-10 05:23:19,218 - root - INFO - step: 71805 loss: 16.6022 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5548 global_avg_mtp_loss: 14.0475 +[titan] 2025-07-10 05:23:19,218 - root - INFO - lr: 8.0524e-05 gnorm: 0.94 [15:48:17< 6:12:21] +[titan] 2025-07-10 05:23:23,184 - root - INFO - step: 71810 loss: 16.4771 memory: 44.58GiB(31.99%) tps: 82,628 tflops: 285.16 mfu: 28.83% global_avg_ntp_loss: 2.5305 global_avg_mtp_loss: 13.9466 +[titan] 2025-07-10 05:23:23,185 - root - INFO - lr: 8.0508e-05 gnorm: 0.93 [15:48:21< 6:12:17] +[titan] 2025-07-10 05:23:27,102 - root - INFO - step: 71815 loss: 16.0977 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.6334 +[titan] 2025-07-10 05:23:27,103 - root - INFO - lr: 8.0491e-05 gnorm: 0.90 [15:48:25< 6:12:13] +[titan] 2025-07-10 05:23:31,065 - root - INFO - step: 71820 loss: 16.2092 memory: 44.58GiB(31.99%) tps: 82,705 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7308 +[titan] 2025-07-10 05:23:31,065 - root - INFO - lr: 8.0474e-05 gnorm: 0.91 [15:48:29< 6:12:09] +[titan] 2025-07-10 05:23:34,998 - root - INFO - step: 71825 loss: 16.2729 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.7747 +[titan] 2025-07-10 05:23:34,998 - root - INFO - lr: 8.0458e-05 gnorm: 0.98 [15:48:33< 6:12:05] +[titan] 2025-07-10 05:23:38,931 - root - INFO - step: 71830 loss: 16.4611 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5117 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-10 05:23:38,931 - root - INFO - lr: 8.0441e-05 gnorm: 0.94 [15:48:36< 6:12:01] +[titan] 2025-07-10 05:23:42,856 - root - INFO - step: 71835 loss: 16.1714 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.7004 +[titan] 2025-07-10 05:23:42,856 - root - INFO - lr: 8.0424e-05 gnorm: 0.97 [15:48:40< 6:11:57] +[titan] 2025-07-10 05:23:46,801 - root - INFO - step: 71840 loss: 16.0605 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6072 +[titan] 2025-07-10 05:23:46,802 - root - INFO - lr: 8.0408e-05 gnorm: 0.89 [15:48:44< 6:11:53] +[titan] 2025-07-10 05:23:50,727 - root - INFO - step: 71845 loss: 16.2205 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.7207 +[titan] 2025-07-10 05:23:50,728 - root - INFO - lr: 8.0391e-05 gnorm: 0.94 [15:48:48< 6:11:49] +[titan] 2025-07-10 05:23:53,859 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:23:54,668 - root - INFO - step: 71850 loss: 16.4249 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9072 +[titan] 2025-07-10 05:23:54,669 - root - INFO - lr: 8.0374e-05 gnorm: 0.97 [15:48:52< 6:11:45] +[titan] 2025-07-10 05:23:58,578 - root - INFO - step: 71855 loss: 16.3781 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.8620 +[titan] 2025-07-10 05:23:58,578 - root - INFO - lr: 8.0357e-05 gnorm: 0.93 [15:48:56< 6:11:41] +[titan] 2025-07-10 05:24:02,491 - root - INFO - step: 71860 loss: 16.1930 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.7180 +[titan] 2025-07-10 05:24:02,491 - root - INFO - lr: 8.0341e-05 gnorm: 0.89 [15:49:00< 6:11:37] +[titan] 2025-07-10 05:24:06,407 - root - INFO - step: 71865 loss: 16.1341 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4774 global_avg_mtp_loss: 13.6567 +[titan] 2025-07-10 05:24:06,407 - root - INFO - lr: 8.0324e-05 gnorm: 0.98 [15:49:04< 6:11:33] +[titan] 2025-07-10 05:24:10,332 - root - INFO - step: 71870 loss: 16.2288 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7475 +[titan] 2025-07-10 05:24:10,332 - root - INFO - lr: 8.0307e-05 gnorm: 0.94 [15:49:08< 6:11:29] +[titan] 2025-07-10 05:24:14,298 - root - INFO - step: 71875 loss: 16.5699 memory: 44.58GiB(31.99%) tps: 82,629 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 14.0267 +[titan] 2025-07-10 05:24:14,298 - root - INFO - lr: 8.0291e-05 gnorm: 0.89 [15:49:12< 6:11:25] +[titan] 2025-07-10 05:24:18,212 - root - INFO - step: 71880 loss: 15.9579 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4372 global_avg_mtp_loss: 13.5207 +[titan] 2025-07-10 05:24:18,213 - root - INFO - lr: 8.0274e-05 gnorm: 0.88 [15:49:16< 6:11:21] +[titan] 2025-07-10 05:24:22,158 - root - INFO - step: 71885 loss: 16.5208 memory: 44.58GiB(31.99%) tps: 83,060 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9809 +[titan] 2025-07-10 05:24:22,158 - root - INFO - lr: 8.0257e-05 gnorm: 0.92 [15:49:20< 6:11:17] +[titan] 2025-07-10 05:24:26,097 - root - INFO - step: 71890 loss: 16.7401 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5840 global_avg_mtp_loss: 14.1561 +[titan] 2025-07-10 05:24:26,097 - root - INFO - lr: 8.0241e-05 gnorm: 0.91 [15:49:24< 6:11:13] +[titan] 2025-07-10 05:24:30,027 - root - INFO - step: 71895 loss: 16.3069 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8148 +[titan] 2025-07-10 05:24:30,027 - root - INFO - lr: 8.0224e-05 gnorm: 0.95 [15:49:28< 6:11:09] +[titan] 2025-07-10 05:24:33,163 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:24:33,963 - root - INFO - step: 71900 loss: 16.5054 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5315 global_avg_mtp_loss: 13.9739 +[titan] 2025-07-10 05:24:33,963 - root - INFO - lr: 8.0207e-05 gnorm: 0.88 [15:49:31< 6:11:05] +[titan] 2025-07-10 05:24:37,919 - root - INFO - step: 71905 loss: 16.5076 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9670 +[titan] 2025-07-10 05:24:37,919 - root - INFO - lr: 8.0191e-05 gnorm: 0.92 [15:49:35< 6:11:01] +[titan] 2025-07-10 05:24:41,823 - root - INFO - step: 71910 loss: 16.2308 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7406 +[titan] 2025-07-10 05:24:41,823 - root - INFO - lr: 8.0174e-05 gnorm: 0.91 [15:49:39< 6:10:57] +[titan] 2025-07-10 05:24:45,749 - root - INFO - step: 71915 loss: 16.0744 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.6264 +[titan] 2025-07-10 05:24:45,749 - root - INFO - lr: 8.0157e-05 gnorm: 0.89 [15:49:43< 6:10:53] +[titan] 2025-07-10 05:24:49,660 - root - INFO - step: 71920 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 05:24:49,661 - root - INFO - lr: 8.0141e-05 gnorm: 0.94 [15:49:47< 6:10:49] +[titan] 2025-07-10 05:24:53,587 - root - INFO - step: 71925 loss: 16.1491 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 05:24:53,587 - root - INFO - lr: 8.0124e-05 gnorm: 0.90 [15:49:51< 6:10:45] +[titan] 2025-07-10 05:24:57,527 - root - INFO - step: 71930 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5244 global_avg_mtp_loss: 13.9306 +[titan] 2025-07-10 05:24:57,528 - root - INFO - lr: 8.0107e-05 gnorm: 0.93 [15:49:55< 6:10:41] +[titan] 2025-07-10 05:25:01,497 - root - INFO - step: 71935 loss: 16.5796 memory: 44.58GiB(31.99%) tps: 82,556 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.5473 global_avg_mtp_loss: 14.0322 +[titan] 2025-07-10 05:25:01,497 - root - INFO - lr: 8.0091e-05 gnorm: 0.97 [15:49:59< 6:10:38] +[titan] 2025-07-10 05:25:05,466 - root - INFO - step: 71940 loss: 16.0776 memory: 44.58GiB(31.99%) tps: 82,568 tflops: 284.95 mfu: 28.81% global_avg_ntp_loss: 2.4525 global_avg_mtp_loss: 13.6252 +[titan] 2025-07-10 05:25:05,466 - root - INFO - lr: 8.0074e-05 gnorm: 0.93 [15:50:03< 6:10:34] +[titan] 2025-07-10 05:25:09,399 - root - INFO - step: 71945 loss: 16.2282 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7625 +[titan] 2025-07-10 05:25:09,399 - root - INFO - lr: 8.0057e-05 gnorm: 0.89 [15:50:07< 6:10:30] +[titan] 2025-07-10 05:25:12,522 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:25:13,321 - root - INFO - step: 71950 loss: 16.0391 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4449 global_avg_mtp_loss: 13.5941 +[titan] 2025-07-10 05:25:13,321 - root - INFO - lr: 8.0041e-05 gnorm: 0.91 [15:50:11< 6:10:26] +[titan] 2025-07-10 05:25:17,271 - root - INFO - step: 71955 loss: 16.5409 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.5331 global_avg_mtp_loss: 14.0077 +[titan] 2025-07-10 05:25:17,271 - root - INFO - lr: 8.0024e-05 gnorm: 0.90 [15:50:15< 6:10:22] +[titan] 2025-07-10 05:25:21,210 - root - INFO - step: 71960 loss: 16.3629 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 05:25:21,210 - root - INFO - lr: 8.0008e-05 gnorm: 0.91 [15:50:19< 6:10:18] +[titan] 2025-07-10 05:25:25,155 - root - INFO - step: 71965 loss: 16.3792 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8748 +[titan] 2025-07-10 05:25:25,155 - root - INFO - lr: 7.9991e-05 gnorm: 0.95 [15:50:23< 6:10:14] +[titan] 2025-07-10 05:25:29,094 - root - INFO - step: 71970 loss: 16.3908 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.8733 +[titan] 2025-07-10 05:25:29,095 - root - INFO - lr: 7.9974e-05 gnorm: 0.91 [15:50:27< 6:10:10] +[titan] 2025-07-10 05:25:32,999 - root - INFO - step: 71975 loss: 16.3646 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8657 +[titan] 2025-07-10 05:25:32,999 - root - INFO - lr: 7.9958e-05 gnorm: 0.89 [15:50:31< 6:10:06] +[titan] 2025-07-10 05:25:36,920 - root - INFO - step: 71980 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.9157 +[titan] 2025-07-10 05:25:36,921 - root - INFO - lr: 7.9941e-05 gnorm: 0.93 [15:50:34< 6:10:02] +[titan] 2025-07-10 05:25:40,824 - root - INFO - step: 71985 loss: 16.3765 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8678 +[titan] 2025-07-10 05:25:40,824 - root - INFO - lr: 7.9924e-05 gnorm: 0.92 [15:50:38< 6:09:58] +[titan] 2025-07-10 05:25:44,751 - root - INFO - step: 71990 loss: 16.1996 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4749 global_avg_mtp_loss: 13.7248 +[titan] 2025-07-10 05:25:44,751 - root - INFO - lr: 7.9908e-05 gnorm: 0.93 [15:50:42< 6:09:54] +[titan] 2025-07-10 05:25:48,643 - root - INFO - step: 71995 loss: 16.2148 memory: 44.58GiB(31.99%) tps: 84,194 tflops: 290.57 mfu: 29.38% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7319 +[titan] 2025-07-10 05:25:48,643 - root - INFO - lr: 7.9891e-05 gnorm: 0.93 [15:50:46< 6:09:50] +[titan] 2025-07-10 05:25:51,749 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:25:52,543 - root - INFO - step: 72000 loss: 16.5521 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.33% global_avg_ntp_loss: 2.5395 global_avg_mtp_loss: 14.0126 +[titan] 2025-07-10 05:25:52,543 - root - INFO - lr: 7.9874e-05 gnorm: 0.91 [15:50:50< 6:09:46] +[titan] 2025-07-10 05:25:56,454 - root - INFO - step: 72005 loss: 16.4698 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9472 +[titan] 2025-07-10 05:25:56,454 - root - INFO - lr: 7.9858e-05 gnorm: 0.90 [15:50:54< 6:09:42] +[titan] 2025-07-10 05:26:00,348 - root - INFO - step: 72010 loss: 16.3348 memory: 44.58GiB(31.99%) tps: 84,142 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8359 +[titan] 2025-07-10 05:26:00,349 - root - INFO - lr: 7.9841e-05 gnorm: 0.87 [15:50:58< 6:09:38] +[titan] 2025-07-10 05:26:04,244 - root - INFO - step: 72015 loss: 16.3950 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.8677 +[titan] 2025-07-10 05:26:04,244 - root - INFO - lr: 7.9825e-05 gnorm: 0.94 [15:51:02< 6:09:34] +[titan] 2025-07-10 05:26:08,216 - root - INFO - step: 72020 loss: 16.2470 memory: 44.58GiB(31.99%) tps: 82,497 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7670 +[titan] 2025-07-10 05:26:08,217 - root - INFO - lr: 7.9808e-05 gnorm: 0.93 [15:51:06< 6:09:30] +[titan] 2025-07-10 05:26:12,148 - root - INFO - step: 72025 loss: 16.4950 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.9565 +[titan] 2025-07-10 05:26:12,148 - root - INFO - lr: 7.9791e-05 gnorm: 0.99 [15:51:10< 6:09:26] +[titan] 2025-07-10 05:26:16,066 - root - INFO - step: 72030 loss: 16.3544 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8548 +[titan] 2025-07-10 05:26:16,067 - root - INFO - lr: 7.9775e-05 gnorm: 0.90 [15:51:14< 6:09:22] +[titan] 2025-07-10 05:26:19,978 - root - INFO - step: 72035 loss: 16.5621 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5558 global_avg_mtp_loss: 14.0063 +[titan] 2025-07-10 05:26:19,978 - root - INFO - lr: 7.9758e-05 gnorm: 0.95 [15:51:17< 6:09:18] +[titan] 2025-07-10 05:26:23,948 - root - INFO - step: 72040 loss: 16.2846 memory: 44.58GiB(31.99%) tps: 82,546 tflops: 284.88 mfu: 28.80% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7922 +[titan] 2025-07-10 05:26:23,948 - root - INFO - lr: 7.9742e-05 gnorm: 0.90 [15:51:21< 6:09:14] +[titan] 2025-07-10 05:26:27,866 - root - INFO - step: 72045 loss: 16.6009 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0517 +[titan] 2025-07-10 05:26:27,866 - root - INFO - lr: 7.9725e-05 gnorm: 0.99 [15:51:25< 6:09:10] +[titan] 2025-07-10 05:26:31,013 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:26:31,800 - root - INFO - step: 72050 loss: 16.2010 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7256 +[titan] 2025-07-10 05:26:31,800 - root - INFO - lr: 7.9708e-05 gnorm: 0.94 [15:51:29< 6:09:06] +[titan] 2025-07-10 05:26:35,734 - root - INFO - step: 72055 loss: 16.3698 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8675 +[titan] 2025-07-10 05:26:35,735 - root - INFO - lr: 7.9692e-05 gnorm: 0.93 [15:51:33< 6:09:02] +[titan] 2025-07-10 05:26:39,686 - root - INFO - step: 72060 loss: 16.5375 memory: 44.58GiB(31.99%) tps: 82,924 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9921 +[titan] 2025-07-10 05:26:39,687 - root - INFO - lr: 7.9675e-05 gnorm: 0.90 [15:51:37< 6:08:58] +[titan] 2025-07-10 05:26:43,641 - root - INFO - step: 72065 loss: 16.4747 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.9494 +[titan] 2025-07-10 05:26:43,642 - root - INFO - lr: 7.9659e-05 gnorm: 1.01 [15:51:41< 6:08:54] +[titan] 2025-07-10 05:26:47,585 - root - INFO - step: 72070 loss: 16.0791 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.6239 +[titan] 2025-07-10 05:26:47,585 - root - INFO - lr: 7.9642e-05 gnorm: 0.94 [15:51:45< 6:08:50] +[titan] 2025-07-10 05:26:51,542 - root - INFO - step: 72075 loss: 16.4052 memory: 44.58GiB(31.99%) tps: 82,819 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.9021 +[titan] 2025-07-10 05:26:51,542 - root - INFO - lr: 7.9625e-05 gnorm: 0.95 [15:51:49< 6:08:46] +[titan] 2025-07-10 05:26:55,468 - root - INFO - step: 72080 loss: 16.4534 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 05:26:55,468 - root - INFO - lr: 7.9609e-05 gnorm: 1.02 [15:51:53< 6:08:42] +[titan] 2025-07-10 05:26:59,410 - root - INFO - step: 72085 loss: 16.3761 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-10 05:26:59,411 - root - INFO - lr: 7.9592e-05 gnorm: 1.02 [15:51:57< 6:08:38] +[titan] 2025-07-10 05:27:03,371 - root - INFO - step: 72090 loss: 16.4998 memory: 44.58GiB(31.99%) tps: 82,741 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9670 +[titan] 2025-07-10 05:27:03,371 - root - INFO - lr: 7.9576e-05 gnorm: 0.97 [15:52:01< 6:08:34] +[titan] 2025-07-10 05:27:07,304 - root - INFO - step: 72095 loss: 16.4371 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-10 05:27:07,304 - root - INFO - lr: 7.9559e-05 gnorm: 0.93 [15:52:05< 6:08:30] +[titan] 2025-07-10 05:27:10,419 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:27:11,206 - root - INFO - step: 72100 loss: 16.2655 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.83 mfu: 29.30% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7834 +[titan] 2025-07-10 05:27:11,206 - root - INFO - lr: 7.9542e-05 gnorm: 0.95 [15:52:09< 6:08:26] +[titan] 2025-07-10 05:27:15,108 - root - INFO - step: 72105 loss: 16.3016 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.8162 +[titan] 2025-07-10 05:27:15,108 - root - INFO - lr: 7.9526e-05 gnorm: 0.90 [15:52:13< 6:08:22] +[titan] 2025-07-10 05:27:19,019 - root - INFO - step: 72110 loss: 16.1655 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.6879 +[titan] 2025-07-10 05:27:19,019 - root - INFO - lr: 7.9509e-05 gnorm: 0.91 [15:52:17< 6:08:18] +[titan] 2025-07-10 05:27:22,928 - root - INFO - step: 72115 loss: 16.2972 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 05:27:22,928 - root - INFO - lr: 7.9493e-05 gnorm: 0.92 [15:52:20< 6:08:14] +[titan] 2025-07-10 05:27:26,877 - root - INFO - step: 72120 loss: 16.4167 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.9077 +[titan] 2025-07-10 05:27:26,877 - root - INFO - lr: 7.9476e-05 gnorm: 0.95 [15:52:24< 6:08:10] +[titan] 2025-07-10 05:27:30,781 - root - INFO - step: 72125 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8479 +[titan] 2025-07-10 05:27:30,781 - root - INFO - lr: 7.9460e-05 gnorm: 0.92 [15:52:28< 6:08:06] +[titan] 2025-07-10 05:27:34,695 - root - INFO - step: 72130 loss: 16.1979 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7165 +[titan] 2025-07-10 05:27:34,695 - root - INFO - lr: 7.9443e-05 gnorm: 0.98 [15:52:32< 6:08:02] +[titan] 2025-07-10 05:27:38,595 - root - INFO - step: 72135 loss: 16.3448 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8452 +[titan] 2025-07-10 05:27:38,596 - root - INFO - lr: 7.9426e-05 gnorm: 0.96 [15:52:36< 6:07:58] +[titan] 2025-07-10 05:27:42,550 - root - INFO - step: 72140 loss: 16.2233 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 05:27:42,550 - root - INFO - lr: 7.9410e-05 gnorm: 0.92 [15:52:40< 6:07:55] +[titan] 2025-07-10 05:27:46,489 - root - INFO - step: 72145 loss: 16.1755 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.7178 +[titan] 2025-07-10 05:27:46,489 - root - INFO - lr: 7.9393e-05 gnorm: 0.91 [15:52:44< 6:07:51] +[titan] 2025-07-10 05:27:49,595 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:27:50,389 - root - INFO - step: 72150 loss: 16.2466 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7711 +[titan] 2025-07-10 05:27:50,389 - root - INFO - lr: 7.9377e-05 gnorm: 0.89 [15:52:48< 6:07:47] +[titan] 2025-07-10 05:27:54,320 - root - INFO - step: 72155 loss: 16.0055 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.5594 +[titan] 2025-07-10 05:27:54,320 - root - INFO - lr: 7.9360e-05 gnorm: 0.98 [15:52:52< 6:07:43] +[titan] 2025-07-10 05:27:58,242 - root - INFO - step: 72160 loss: 16.4239 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.9068 +[titan] 2025-07-10 05:27:58,242 - root - INFO - lr: 7.9344e-05 gnorm: 0.93 [15:52:56< 6:07:39] +[titan] 2025-07-10 05:28:02,146 - root - INFO - step: 72165 loss: 15.9774 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5356 +[titan] 2025-07-10 05:28:02,147 - root - INFO - lr: 7.9327e-05 gnorm: 0.93 [15:53:00< 6:07:35] +[titan] 2025-07-10 05:28:06,078 - root - INFO - step: 72170 loss: 16.2856 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 05:28:06,078 - root - INFO - lr: 7.9310e-05 gnorm: 0.90 [15:53:04< 6:07:31] +[titan] 2025-07-10 05:28:09,978 - root - INFO - step: 72175 loss: 16.2691 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7794 +[titan] 2025-07-10 05:28:09,978 - root - INFO - lr: 7.9294e-05 gnorm: 0.94 [15:53:07< 6:07:27] +[titan] 2025-07-10 05:28:13,949 - root - INFO - step: 72180 loss: 16.5291 memory: 44.58GiB(31.99%) tps: 82,520 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.5412 global_avg_mtp_loss: 13.9879 +[titan] 2025-07-10 05:28:13,949 - root - INFO - lr: 7.9277e-05 gnorm: 0.94 [15:53:11< 6:07:23] +[titan] 2025-07-10 05:28:17,931 - root - INFO - step: 72185 loss: 16.3600 memory: 44.58GiB(31.99%) tps: 82,302 tflops: 284.04 mfu: 28.72% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8543 +[titan] 2025-07-10 05:28:17,931 - root - INFO - lr: 7.9261e-05 gnorm: 0.96 [15:53:15< 6:07:19] +[titan] 2025-07-10 05:28:21,971 - root - INFO - step: 72190 loss: 16.0690 memory: 44.58GiB(31.99%) tps: 81,122 tflops: 279.97 mfu: 28.31% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6028 +[titan] 2025-07-10 05:28:21,971 - root - INFO - lr: 7.9244e-05 gnorm: 0.90 [15:53:19< 6:07:15] +[titan] 2025-07-10 05:28:23,717 - root - INFO - Dumping profiler traces at step 72192 +[titan] 2025-07-10 05:28:23,749 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:28:26,108 - root - INFO - step: 72195 loss: 16.3132 memory: 44.58GiB(31.99%) tps: 79,201 tflops: 273.33 mfu: 27.64% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 05:28:26,109 - root - INFO - lr: 7.9228e-05 gnorm: 0.93 [15:53:24< 6:07:11] +[titan] 2025-07-10 05:28:29,230 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:28:30,021 - root - INFO - step: 72200 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8451 +[titan] 2025-07-10 05:28:30,021 - root - INFO - lr: 7.9211e-05 gnorm: 0.94 [15:53:28< 6:07:07] +[titan] 2025-07-10 05:28:33,952 - root - INFO - step: 72205 loss: 16.1518 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.6735 +[titan] 2025-07-10 05:28:33,952 - root - INFO - lr: 7.9195e-05 gnorm: 0.94 [15:53:31< 6:07:03] +[titan] 2025-07-10 05:28:37,876 - root - INFO - step: 72210 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8918 +[titan] 2025-07-10 05:28:37,876 - root - INFO - lr: 7.9178e-05 gnorm: 0.98 [15:53:35< 6:06:59] +[titan] 2025-07-10 05:28:41,787 - root - INFO - step: 72215 loss: 16.0872 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.6266 +[titan] 2025-07-10 05:28:41,788 - root - INFO - lr: 7.9162e-05 gnorm: 0.91 [15:53:39< 6:06:55] +[titan] 2025-07-10 05:28:45,695 - root - INFO - step: 72220 loss: 16.0938 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4387 global_avg_mtp_loss: 13.6551 +[titan] 2025-07-10 05:28:45,695 - root - INFO - lr: 7.9145e-05 gnorm: 1.06 [15:53:43< 6:06:51] +[titan] 2025-07-10 05:28:49,606 - root - INFO - step: 72225 loss: 16.3046 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8080 +[titan] 2025-07-10 05:28:49,607 - root - INFO - lr: 7.9129e-05 gnorm: 0.93 [15:53:47< 6:06:47] +[titan] 2025-07-10 05:28:53,548 - root - INFO - step: 72230 loss: 16.1710 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.7034 +[titan] 2025-07-10 05:28:53,549 - root - INFO - lr: 7.9112e-05 gnorm: 0.95 [15:53:51< 6:06:43] +[titan] 2025-07-10 05:28:57,459 - root - INFO - step: 72235 loss: 16.2510 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7683 +[titan] 2025-07-10 05:28:57,459 - root - INFO - lr: 7.9095e-05 gnorm: 1.02 [15:53:55< 6:06:39] +[titan] 2025-07-10 05:29:01,377 - root - INFO - step: 72240 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.9021 +[titan] 2025-07-10 05:29:01,377 - root - INFO - lr: 7.9079e-05 gnorm: 0.95 [15:53:59< 6:06:35] +[titan] 2025-07-10 05:29:05,330 - root - INFO - step: 72245 loss: 16.1133 memory: 44.58GiB(31.99%) tps: 82,908 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6503 +[titan] 2025-07-10 05:29:05,330 - root - INFO - lr: 7.9062e-05 gnorm: 0.92 [15:54:03< 6:06:31] +[titan] 2025-07-10 05:29:08,446 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:29:09,237 - root - INFO - step: 72250 loss: 16.4330 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.9150 +[titan] 2025-07-10 05:29:09,237 - root - INFO - lr: 7.9046e-05 gnorm: 0.89 [15:54:07< 6:06:27] +[titan] 2025-07-10 05:29:13,191 - root - INFO - step: 72255 loss: 16.3419 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8374 +[titan] 2025-07-10 05:29:13,191 - root - INFO - lr: 7.9029e-05 gnorm: 0.94 [15:54:11< 6:06:23] +[titan] 2025-07-10 05:29:17,124 - root - INFO - step: 72260 loss: 16.1805 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7012 +[titan] 2025-07-10 05:29:17,124 - root - INFO - lr: 7.9013e-05 gnorm: 0.98 [15:54:15< 6:06:19] +[titan] 2025-07-10 05:29:21,036 - root - INFO - step: 72265 loss: 16.3366 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8280 +[titan] 2025-07-10 05:29:21,037 - root - INFO - lr: 7.8996e-05 gnorm: 0.95 [15:54:19< 6:06:15] +[titan] 2025-07-10 05:29:24,946 - root - INFO - step: 72270 loss: 16.2963 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.7900 +[titan] 2025-07-10 05:29:24,946 - root - INFO - lr: 7.8980e-05 gnorm: 0.94 [15:54:22< 6:06:11] +[titan] 2025-07-10 05:29:28,860 - root - INFO - step: 72275 loss: 16.2930 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.7780 +[titan] 2025-07-10 05:29:28,860 - root - INFO - lr: 7.8963e-05 gnorm: 0.96 [15:54:26< 6:06:07] +[titan] 2025-07-10 05:29:32,751 - root - INFO - step: 72280 loss: 16.1611 memory: 44.58GiB(31.99%) tps: 84,221 tflops: 290.66 mfu: 29.39% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.7000 +[titan] 2025-07-10 05:29:32,751 - root - INFO - lr: 7.8947e-05 gnorm: 0.90 [15:54:30< 6:06:03] +[titan] 2025-07-10 05:29:36,656 - root - INFO - step: 72285 loss: 16.4785 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5252 global_avg_mtp_loss: 13.9533 +[titan] 2025-07-10 05:29:36,656 - root - INFO - lr: 7.8930e-05 gnorm: 0.92 [15:54:34< 6:05:59] +[titan] 2025-07-10 05:29:40,558 - root - INFO - step: 72290 loss: 16.1960 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7126 +[titan] 2025-07-10 05:29:40,558 - root - INFO - lr: 7.8914e-05 gnorm: 0.94 [15:54:38< 6:05:55] +[titan] 2025-07-10 05:29:44,458 - root - INFO - step: 72295 loss: 16.2595 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7711 +[titan] 2025-07-10 05:29:44,459 - root - INFO - lr: 7.8897e-05 gnorm: 1.03 [15:54:42< 6:05:51] +[titan] 2025-07-10 05:29:47,572 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:29:48,363 - root - INFO - step: 72300 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.7905 +[titan] 2025-07-10 05:29:48,364 - root - INFO - lr: 7.8881e-05 gnorm: 0.91 [15:54:46< 6:05:47] +[titan] 2025-07-10 05:29:52,272 - root - INFO - step: 72305 loss: 16.2922 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.8069 +[titan] 2025-07-10 05:29:52,272 - root - INFO - lr: 7.8864e-05 gnorm: 0.98 [15:54:50< 6:05:43] +[titan] 2025-07-10 05:29:56,192 - root - INFO - step: 72310 loss: 16.1853 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7050 +[titan] 2025-07-10 05:29:56,192 - root - INFO - lr: 7.8848e-05 gnorm: 0.95 [15:54:54< 6:05:39] +[titan] 2025-07-10 05:30:00,130 - root - INFO - step: 72315 loss: 16.4244 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.16 mfu: 29.03% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.9013 +[titan] 2025-07-10 05:30:00,130 - root - INFO - lr: 7.8831e-05 gnorm: 0.90 [15:54:58< 6:05:35] +[titan] 2025-07-10 05:30:04,067 - root - INFO - step: 72320 loss: 15.9927 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4410 global_avg_mtp_loss: 13.5517 +[titan] 2025-07-10 05:30:04,068 - root - INFO - lr: 7.8815e-05 gnorm: 0.99 [15:55:02< 6:05:31] +[titan] 2025-07-10 05:30:07,975 - root - INFO - step: 72325 loss: 16.3664 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8601 +[titan] 2025-07-10 05:30:07,976 - root - INFO - lr: 7.8798e-05 gnorm: 0.93 [15:55:05< 6:05:27] +[titan] 2025-07-10 05:30:11,922 - root - INFO - step: 72330 loss: 15.9349 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.4296 global_avg_mtp_loss: 13.5052 +[titan] 2025-07-10 05:30:11,923 - root - INFO - lr: 7.8782e-05 gnorm: 0.95 [15:55:09< 6:05:24] +[titan] 2025-07-10 05:30:15,852 - root - INFO - step: 72335 loss: 16.1846 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7121 +[titan] 2025-07-10 05:30:15,853 - root - INFO - lr: 7.8765e-05 gnorm: 0.89 [15:55:13< 6:05:20] +[titan] 2025-07-10 05:30:19,798 - root - INFO - step: 72340 loss: 16.6449 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.5644 global_avg_mtp_loss: 14.0805 +[titan] 2025-07-10 05:30:19,798 - root - INFO - lr: 7.8749e-05 gnorm: 0.95 [15:55:17< 6:05:16] +[titan] 2025-07-10 05:30:23,705 - root - INFO - step: 72345 loss: 16.3872 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8764 +[titan] 2025-07-10 05:30:23,705 - root - INFO - lr: 7.8732e-05 gnorm: 0.93 [15:55:21< 6:05:12] +[titan] 2025-07-10 05:30:26,818 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:30:27,609 - root - INFO - step: 72350 loss: 16.2310 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7481 +[titan] 2025-07-10 05:30:27,609 - root - INFO - lr: 7.8716e-05 gnorm: 0.94 [15:55:25< 6:05:08] +[titan] 2025-07-10 05:30:31,516 - root - INFO - step: 72355 loss: 16.2333 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7454 +[titan] 2025-07-10 05:30:31,516 - root - INFO - lr: 7.8700e-05 gnorm: 0.97 [15:55:29< 6:05:04] +[titan] 2025-07-10 05:30:35,421 - root - INFO - step: 72360 loss: 16.3238 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 05:30:35,422 - root - INFO - lr: 7.8683e-05 gnorm: 0.95 [15:55:33< 6:05:00] +[titan] 2025-07-10 05:30:39,327 - root - INFO - step: 72365 loss: 16.4300 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9023 +[titan] 2025-07-10 05:30:39,327 - root - INFO - lr: 7.8667e-05 gnorm: 0.94 [15:55:37< 6:04:56] +[titan] 2025-07-10 05:30:43,264 - root - INFO - step: 72370 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7211 +[titan] 2025-07-10 05:30:43,265 - root - INFO - lr: 7.8650e-05 gnorm: 0.93 [15:55:41< 6:04:52] +[titan] 2025-07-10 05:30:47,189 - root - INFO - step: 72375 loss: 16.0233 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.5871 +[titan] 2025-07-10 05:30:47,189 - root - INFO - lr: 7.8634e-05 gnorm: 0.90 [15:55:45< 6:04:48] +[titan] 2025-07-10 05:30:51,103 - root - INFO - step: 72380 loss: 16.2457 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 05:30:51,104 - root - INFO - lr: 7.8617e-05 gnorm: 0.93 [15:55:49< 6:04:44] +[titan] 2025-07-10 05:30:55,031 - root - INFO - step: 72385 loss: 16.2426 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7651 +[titan] 2025-07-10 05:30:55,031 - root - INFO - lr: 7.8601e-05 gnorm: 0.94 [15:55:53< 6:04:40] +[titan] 2025-07-10 05:30:58,973 - root - INFO - step: 72390 loss: 16.4140 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.8895 +[titan] 2025-07-10 05:30:58,973 - root - INFO - lr: 7.8584e-05 gnorm: 0.90 [15:55:56< 6:04:36] +[titan] 2025-07-10 05:31:02,903 - root - INFO - step: 72395 loss: 16.3371 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8328 +[titan] 2025-07-10 05:31:02,903 - root - INFO - lr: 7.8568e-05 gnorm: 0.91 [15:56:00< 6:04:32] +[titan] 2025-07-10 05:31:06,014 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:31:06,813 - root - INFO - step: 72400 loss: 16.4000 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8830 +[titan] 2025-07-10 05:31:06,813 - root - INFO - lr: 7.8551e-05 gnorm: 0.95 [15:56:04< 6:04:28] +[titan] 2025-07-10 05:31:10,714 - root - INFO - step: 72405 loss: 16.4631 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9343 +[titan] 2025-07-10 05:31:10,714 - root - INFO - lr: 7.8535e-05 gnorm: 0.94 [15:56:08< 6:04:24] +[titan] 2025-07-10 05:31:14,610 - root - INFO - step: 72410 loss: 16.3545 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8472 +[titan] 2025-07-10 05:31:14,610 - root - INFO - lr: 7.8518e-05 gnorm: 0.92 [15:56:12< 6:04:20] +[titan] 2025-07-10 05:31:18,527 - root - INFO - step: 72415 loss: 16.2950 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8019 +[titan] 2025-07-10 05:31:18,528 - root - INFO - lr: 7.8502e-05 gnorm: 0.90 [15:56:16< 6:04:16] +[titan] 2025-07-10 05:31:22,453 - root - INFO - step: 72420 loss: 16.2025 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7218 +[titan] 2025-07-10 05:31:22,453 - root - INFO - lr: 7.8486e-05 gnorm: 0.91 [15:56:20< 6:04:12] +[titan] 2025-07-10 05:31:26,398 - root - INFO - step: 72425 loss: 16.0976 memory: 44.58GiB(31.99%) tps: 83,078 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6325 +[titan] 2025-07-10 05:31:26,398 - root - INFO - lr: 7.8469e-05 gnorm: 0.93 [15:56:24< 6:04:08] +[titan] 2025-07-10 05:31:30,331 - root - INFO - step: 72430 loss: 16.1199 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.6490 +[titan] 2025-07-10 05:31:30,331 - root - INFO - lr: 7.8453e-05 gnorm: 0.97 [15:56:28< 6:04:04] +[titan] 2025-07-10 05:31:34,265 - root - INFO - step: 72435 loss: 16.3450 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8361 +[titan] 2025-07-10 05:31:34,265 - root - INFO - lr: 7.8436e-05 gnorm: 1.00 [15:56:32< 6:04:00] +[titan] 2025-07-10 05:31:38,159 - root - INFO - step: 72440 loss: 16.5207 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9906 +[titan] 2025-07-10 05:31:38,160 - root - INFO - lr: 7.8420e-05 gnorm: 0.96 [15:56:36< 6:03:56] +[titan] 2025-07-10 05:31:42,104 - root - INFO - step: 72445 loss: 16.6752 memory: 44.58GiB(31.99%) tps: 83,076 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5715 global_avg_mtp_loss: 14.1037 +[titan] 2025-07-10 05:31:42,104 - root - INFO - lr: 7.8403e-05 gnorm: 0.94 [15:56:40< 6:03:52] +[titan] 2025-07-10 05:31:45,247 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:31:46,031 - root - INFO - step: 72450 loss: 15.9453 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4351 global_avg_mtp_loss: 13.5102 +[titan] 2025-07-10 05:31:46,032 - root - INFO - lr: 7.8387e-05 gnorm: 0.92 [15:56:44< 6:03:48] +[titan] 2025-07-10 05:31:49,928 - root - INFO - step: 72455 loss: 16.1889 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7133 +[titan] 2025-07-10 05:31:49,929 - root - INFO - lr: 7.8371e-05 gnorm: 0.91 [15:56:47< 6:03:44] +[titan] 2025-07-10 05:31:53,835 - root - INFO - step: 72460 loss: 16.3989 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8924 +[titan] 2025-07-10 05:31:53,835 - root - INFO - lr: 7.8354e-05 gnorm: 0.95 [15:56:51< 6:03:40] +[titan] 2025-07-10 05:31:57,754 - root - INFO - step: 72465 loss: 16.2619 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7788 +[titan] 2025-07-10 05:31:57,754 - root - INFO - lr: 7.8338e-05 gnorm: 0.95 [15:56:55< 6:03:36] +[titan] 2025-07-10 05:32:01,659 - root - INFO - step: 72470 loss: 16.4466 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5202 global_avg_mtp_loss: 13.9264 +[titan] 2025-07-10 05:32:01,659 - root - INFO - lr: 7.8321e-05 gnorm: 0.94 [15:56:59< 6:03:32] +[titan] 2025-07-10 05:32:05,604 - root - INFO - step: 72475 loss: 16.3213 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8118 +[titan] 2025-07-10 05:32:05,604 - root - INFO - lr: 7.8305e-05 gnorm: 0.89 [15:57:03< 6:03:28] +[titan] 2025-07-10 05:32:09,582 - root - INFO - step: 72480 loss: 16.2604 memory: 44.58GiB(31.99%) tps: 82,384 tflops: 284.32 mfu: 28.75% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7773 +[titan] 2025-07-10 05:32:09,582 - root - INFO - lr: 7.8288e-05 gnorm: 0.94 [15:57:07< 6:03:24] +[titan] 2025-07-10 05:32:13,517 - root - INFO - step: 72485 loss: 16.2520 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7645 +[titan] 2025-07-10 05:32:13,517 - root - INFO - lr: 7.8272e-05 gnorm: 0.96 [15:57:11< 6:03:20] +[titan] 2025-07-10 05:32:17,458 - root - INFO - step: 72490 loss: 16.3176 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 05:32:17,458 - root - INFO - lr: 7.8256e-05 gnorm: 0.91 [15:57:15< 6:03:16] +[titan] 2025-07-10 05:32:21,362 - root - INFO - step: 72495 loss: 16.2723 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7856 +[titan] 2025-07-10 05:32:21,362 - root - INFO - lr: 7.8239e-05 gnorm: 0.94 [15:57:19< 6:03:12] +[titan] 2025-07-10 05:32:24,491 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:32:25,276 - root - INFO - step: 72500 loss: 16.1920 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.7202 +[titan] 2025-07-10 05:32:25,276 - root - INFO - lr: 7.8223e-05 gnorm: 0.94 [15:57:23< 6:03:08] +[titan] 2025-07-10 05:32:29,194 - root - INFO - step: 72505 loss: 16.5874 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5482 global_avg_mtp_loss: 14.0392 +[titan] 2025-07-10 05:32:29,194 - root - INFO - lr: 7.8206e-05 gnorm: 0.94 [15:57:27< 6:03:04] +[titan] 2025-07-10 05:32:33,083 - root - INFO - step: 72510 loss: 16.0899 memory: 44.58GiB(31.99%) tps: 84,254 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.6438 +[titan] 2025-07-10 05:32:33,084 - root - INFO - lr: 7.8190e-05 gnorm: 1.01 [15:57:31< 6:03:00] +[titan] 2025-07-10 05:32:36,986 - root - INFO - step: 72515 loss: 16.3980 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.8828 +[titan] 2025-07-10 05:32:36,986 - root - INFO - lr: 7.8174e-05 gnorm: 0.98 [15:57:34< 6:02:56] +[titan] 2025-07-10 05:32:40,940 - root - INFO - step: 72520 loss: 16.4594 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9227 +[titan] 2025-07-10 05:32:40,940 - root - INFO - lr: 7.8157e-05 gnorm: 0.98 [15:57:38< 6:02:52] +[titan] 2025-07-10 05:32:44,869 - root - INFO - step: 72525 loss: 16.1774 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.7073 +[titan] 2025-07-10 05:32:44,870 - root - INFO - lr: 7.8141e-05 gnorm: 0.91 [15:57:42< 6:02:48] +[titan] 2025-07-10 05:32:48,797 - root - INFO - step: 72530 loss: 16.2182 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7323 +[titan] 2025-07-10 05:32:48,797 - root - INFO - lr: 7.8124e-05 gnorm: 0.95 [15:57:46< 6:02:44] +[titan] 2025-07-10 05:32:52,729 - root - INFO - step: 72535 loss: 16.3022 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.8116 +[titan] 2025-07-10 05:32:52,729 - root - INFO - lr: 7.8108e-05 gnorm: 0.92 [15:57:50< 6:02:40] +[titan] 2025-07-10 05:32:56,728 - root - INFO - step: 72540 loss: 16.2411 memory: 44.58GiB(31.99%) tps: 81,940 tflops: 282.79 mfu: 28.59% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.7401 +[titan] 2025-07-10 05:32:56,728 - root - INFO - lr: 7.8092e-05 gnorm: 0.92 [15:57:54< 6:02:37] +[titan] 2025-07-10 05:33:00,644 - root - INFO - step: 72545 loss: 16.3305 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.8398 +[titan] 2025-07-10 05:33:00,645 - root - INFO - lr: 7.8075e-05 gnorm: 0.92 [15:57:58< 6:02:33] +[titan] 2025-07-10 05:33:03,762 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:33:04,557 - root - INFO - step: 72550 loss: 15.7334 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.3890 global_avg_mtp_loss: 13.3444 +[titan] 2025-07-10 05:33:04,557 - root - INFO - lr: 7.8059e-05 gnorm: 0.99 [15:58:02< 6:02:29] +[titan] 2025-07-10 05:33:08,452 - root - INFO - step: 72555 loss: 16.2007 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7221 +[titan] 2025-07-10 05:33:08,452 - root - INFO - lr: 7.8042e-05 gnorm: 0.93 [15:58:06< 6:02:25] +[titan] 2025-07-10 05:33:12,370 - root - INFO - step: 72560 loss: 16.1771 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7016 +[titan] 2025-07-10 05:33:12,370 - root - INFO - lr: 7.8026e-05 gnorm: 0.94 [15:58:10< 6:02:21] +[titan] 2025-07-10 05:33:16,261 - root - INFO - step: 72565 loss: 16.3226 memory: 44.58GiB(31.99%) tps: 84,213 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 05:33:16,261 - root - INFO - lr: 7.8010e-05 gnorm: 0.92 [15:58:14< 6:02:17] +[titan] 2025-07-10 05:33:20,176 - root - INFO - step: 72570 loss: 16.2405 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7559 +[titan] 2025-07-10 05:33:20,176 - root - INFO - lr: 7.7993e-05 gnorm: 0.90 [15:58:18< 6:02:13] +[titan] 2025-07-10 05:33:24,084 - root - INFO - step: 72575 loss: 16.1870 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7078 +[titan] 2025-07-10 05:33:24,084 - root - INFO - lr: 7.7977e-05 gnorm: 0.91 [15:58:22< 6:02:09] +[titan] 2025-07-10 05:33:27,996 - root - INFO - step: 72580 loss: 16.3159 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8178 +[titan] 2025-07-10 05:33:27,996 - root - INFO - lr: 7.7961e-05 gnorm: 0.96 [15:58:25< 6:02:05] +[titan] 2025-07-10 05:33:31,941 - root - INFO - step: 72585 loss: 16.5326 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5380 global_avg_mtp_loss: 13.9946 +[titan] 2025-07-10 05:33:31,942 - root - INFO - lr: 7.7944e-05 gnorm: 0.93 [15:58:29< 6:02:01] +[titan] 2025-07-10 05:33:35,874 - root - INFO - step: 72590 loss: 16.2915 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.8047 +[titan] 2025-07-10 05:33:35,874 - root - INFO - lr: 7.7928e-05 gnorm: 0.92 [15:58:33< 6:01:57] +[titan] 2025-07-10 05:33:39,786 - root - INFO - step: 72595 loss: 16.1229 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4605 global_avg_mtp_loss: 13.6624 +[titan] 2025-07-10 05:33:39,786 - root - INFO - lr: 7.7911e-05 gnorm: 0.93 [15:58:37< 6:01:53] +[titan] 2025-07-10 05:33:42,892 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:33:43,679 - root - INFO - step: 72600 loss: 16.3451 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8435 +[titan] 2025-07-10 05:33:43,679 - root - INFO - lr: 7.7895e-05 gnorm: 0.93 [15:58:41< 6:01:49] +[titan] 2025-07-10 05:33:47,574 - root - INFO - step: 72605 loss: 16.4379 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 05:33:47,574 - root - INFO - lr: 7.7879e-05 gnorm: 0.95 [15:58:45< 6:01:45] +[titan] 2025-07-10 05:33:51,502 - root - INFO - step: 72610 loss: 16.3371 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8274 +[titan] 2025-07-10 05:33:51,503 - root - INFO - lr: 7.7862e-05 gnorm: 0.96 [15:58:49< 6:01:41] +[titan] 2025-07-10 05:33:55,463 - root - INFO - step: 72615 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 82,738 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.8118 +[titan] 2025-07-10 05:33:55,464 - root - INFO - lr: 7.7846e-05 gnorm: 1.02 [15:58:53< 6:01:37] +[titan] 2025-07-10 05:33:59,423 - root - INFO - step: 72620 loss: 16.3293 memory: 44.58GiB(31.99%) tps: 82,763 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.8452 +[titan] 2025-07-10 05:33:59,423 - root - INFO - lr: 7.7830e-05 gnorm: 0.99 [15:58:57< 6:01:33] +[titan] 2025-07-10 05:34:03,331 - root - INFO - step: 72625 loss: 16.1165 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4484 global_avg_mtp_loss: 13.6681 +[titan] 2025-07-10 05:34:03,331 - root - INFO - lr: 7.7813e-05 gnorm: 0.93 [15:59:01< 6:01:29] +[titan] 2025-07-10 05:34:07,281 - root - INFO - step: 72630 loss: 16.1199 memory: 44.58GiB(31.99%) tps: 82,962 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6502 +[titan] 2025-07-10 05:34:07,281 - root - INFO - lr: 7.7797e-05 gnorm: 0.95 [15:59:05< 6:01:25] +[titan] 2025-07-10 05:34:11,214 - root - INFO - step: 72635 loss: 16.3688 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.8714 +[titan] 2025-07-10 05:34:11,214 - root - INFO - lr: 7.7781e-05 gnorm: 0.90 [15:59:09< 6:01:21] +[titan] 2025-07-10 05:34:15,157 - root - INFO - step: 72640 loss: 16.3452 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.8287 +[titan] 2025-07-10 05:34:15,157 - root - INFO - lr: 7.7764e-05 gnorm: 0.93 [15:59:13< 6:01:17] +[titan] 2025-07-10 05:34:19,092 - root - INFO - step: 72645 loss: 16.6236 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 14.0725 +[titan] 2025-07-10 05:34:19,092 - root - INFO - lr: 7.7748e-05 gnorm: 0.98 [15:59:17< 6:01:13] +[titan] 2025-07-10 05:34:22,227 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:34:23,021 - root - INFO - step: 72650 loss: 16.1710 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.7094 +[titan] 2025-07-10 05:34:23,022 - root - INFO - lr: 7.7731e-05 gnorm: 0.95 [15:59:20< 6:01:09] +[titan] 2025-07-10 05:34:26,948 - root - INFO - step: 72655 loss: 16.3537 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8365 +[titan] 2025-07-10 05:34:26,948 - root - INFO - lr: 7.7715e-05 gnorm: 0.96 [15:59:24< 6:01:05] +[titan] 2025-07-10 05:34:30,845 - root - INFO - step: 72660 loss: 16.5441 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5442 global_avg_mtp_loss: 13.9999 +[titan] 2025-07-10 05:34:30,845 - root - INFO - lr: 7.7699e-05 gnorm: 0.97 [15:59:28< 6:01:01] +[titan] 2025-07-10 05:34:34,743 - root - INFO - step: 72665 loss: 16.4847 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9474 +[titan] 2025-07-10 05:34:34,743 - root - INFO - lr: 7.7682e-05 gnorm: 0.95 [15:59:32< 6:00:57] +[titan] 2025-07-10 05:34:38,630 - root - INFO - step: 72670 loss: 16.1579 memory: 44.58GiB(31.99%) tps: 84,308 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6958 +[titan] 2025-07-10 05:34:38,630 - root - INFO - lr: 7.7666e-05 gnorm: 0.93 [15:59:36< 6:00:53] +[titan] 2025-07-10 05:34:42,540 - root - INFO - step: 72675 loss: 16.4080 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5441 global_avg_mtp_loss: 13.8639 +[titan] 2025-07-10 05:34:42,540 - root - INFO - lr: 7.7650e-05 gnorm: 1.06 [15:59:40< 6:00:49] +[titan] 2025-07-10 05:34:46,451 - root - INFO - step: 72680 loss: 16.3397 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.8482 +[titan] 2025-07-10 05:34:46,451 - root - INFO - lr: 7.7633e-05 gnorm: 0.91 [15:59:44< 6:00:45] +[titan] 2025-07-10 05:34:50,349 - root - INFO - step: 72685 loss: 16.4704 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.9436 +[titan] 2025-07-10 05:34:50,349 - root - INFO - lr: 7.7617e-05 gnorm: 0.92 [15:59:48< 6:00:41] +[titan] 2025-07-10 05:34:54,252 - root - INFO - step: 72690 loss: 16.4798 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5302 global_avg_mtp_loss: 13.9495 +[titan] 2025-07-10 05:34:54,252 - root - INFO - lr: 7.7601e-05 gnorm: 0.91 [15:59:52< 6:00:37] +[titan] 2025-07-10 05:34:58,167 - root - INFO - step: 72695 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 05:34:58,167 - root - INFO - lr: 7.7585e-05 gnorm: 0.90 [15:59:56< 6:00:33] +[titan] 2025-07-10 05:35:01,271 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:35:02,061 - root - INFO - step: 72700 loss: 16.0540 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6010 +[titan] 2025-07-10 05:35:02,062 - root - INFO - lr: 7.7568e-05 gnorm: 0.96 [16:00:00< 6:00:29] +[titan] 2025-07-10 05:35:05,418 - root - INFO - Dumping profiler traces at step 72704 +[titan] 2025-07-10 05:35:05,450 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:35:06,241 - root - INFO - step: 72705 loss: 16.1981 memory: 44.58GiB(31.99%) tps: 78,398 tflops: 270.56 mfu: 27.36% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7155 +[titan] 2025-07-10 05:35:06,242 - root - INFO - lr: 7.7552e-05 gnorm: 0.93 [16:00:04< 6:00:25] +[titan] 2025-07-10 05:35:10,167 - root - INFO - step: 72710 loss: 16.3134 memory: 44.58GiB(31.99%) tps: 83,479 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8106 +[titan] 2025-07-10 05:35:10,167 - root - INFO - lr: 7.7536e-05 gnorm: 0.94 [16:00:08< 6:00:21] +[titan] 2025-07-10 05:35:14,085 - root - INFO - step: 72715 loss: 16.2537 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.7481 +[titan] 2025-07-10 05:35:14,085 - root - INFO - lr: 7.7519e-05 gnorm: 0.95 [16:00:12< 6:00:17] +[titan] 2025-07-10 05:35:17,984 - root - INFO - step: 72720 loss: 16.1068 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6418 +[titan] 2025-07-10 05:35:17,984 - root - INFO - lr: 7.7503e-05 gnorm: 0.92 [16:00:15< 6:00:13] +[titan] 2025-07-10 05:35:21,888 - root - INFO - step: 72725 loss: 16.2753 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7866 +[titan] 2025-07-10 05:35:21,888 - root - INFO - lr: 7.7487e-05 gnorm: 0.95 [16:00:19< 6:00:09] +[titan] 2025-07-10 05:35:25,811 - root - INFO - step: 72730 loss: 16.4532 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9332 +[titan] 2025-07-10 05:35:25,811 - root - INFO - lr: 7.7470e-05 gnorm: 0.92 [16:00:23< 6:00:05] +[titan] 2025-07-10 05:35:29,713 - root - INFO - step: 72735 loss: 16.5070 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9675 +[titan] 2025-07-10 05:35:29,713 - root - INFO - lr: 7.7454e-05 gnorm: 0.89 [16:00:27< 6:00:01] +[titan] 2025-07-10 05:35:33,653 - root - INFO - step: 72740 loss: 16.3014 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.7960 +[titan] 2025-07-10 05:35:33,653 - root - INFO - lr: 7.7438e-05 gnorm: 0.98 [16:00:31< 5:59:57] +[titan] 2025-07-10 05:35:37,541 - root - INFO - step: 72745 loss: 16.2608 memory: 44.58GiB(31.99%) tps: 84,290 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7832 +[titan] 2025-07-10 05:35:37,541 - root - INFO - lr: 7.7421e-05 gnorm: 0.95 [16:00:35< 5:59:53] +[titan] 2025-07-10 05:35:40,679 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:35:41,480 - root - INFO - step: 72750 loss: 16.1384 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.6751 +[titan] 2025-07-10 05:35:41,480 - root - INFO - lr: 7.7405e-05 gnorm: 0.96 [16:00:39< 5:59:50] +[titan] 2025-07-10 05:35:45,373 - root - INFO - step: 72755 loss: 15.8900 memory: 44.58GiB(31.99%) tps: 84,181 tflops: 290.52 mfu: 29.38% global_avg_ntp_loss: 2.4151 global_avg_mtp_loss: 13.4749 +[titan] 2025-07-10 05:35:45,373 - root - INFO - lr: 7.7389e-05 gnorm: 0.90 [16:00:43< 5:59:46] +[titan] 2025-07-10 05:35:49,333 - root - INFO - step: 72760 loss: 16.3206 memory: 44.58GiB(31.99%) tps: 82,740 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8250 +[titan] 2025-07-10 05:35:49,334 - root - INFO - lr: 7.7372e-05 gnorm: 0.89 [16:00:47< 5:59:42] +[titan] 2025-07-10 05:35:53,258 - root - INFO - step: 72765 loss: 16.4077 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.9032 +[titan] 2025-07-10 05:35:53,258 - root - INFO - lr: 7.7356e-05 gnorm: 0.99 [16:00:51< 5:59:38] +[titan] 2025-07-10 05:35:57,199 - root - INFO - step: 72770 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.9113 +[titan] 2025-07-10 05:35:57,199 - root - INFO - lr: 7.7340e-05 gnorm: 0.90 [16:00:55< 5:59:34] +[titan] 2025-07-10 05:36:01,114 - root - INFO - step: 72775 loss: 16.4971 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9730 +[titan] 2025-07-10 05:36:01,114 - root - INFO - lr: 7.7324e-05 gnorm: 0.95 [16:00:59< 5:59:30] +[titan] 2025-07-10 05:36:05,059 - root - INFO - step: 72780 loss: 16.2568 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7703 +[titan] 2025-07-10 05:36:05,059 - root - INFO - lr: 7.7307e-05 gnorm: 0.89 [16:01:03< 5:59:26] +[titan] 2025-07-10 05:36:08,975 - root - INFO - step: 72785 loss: 16.1282 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4657 global_avg_mtp_loss: 13.6625 +[titan] 2025-07-10 05:36:08,975 - root - INFO - lr: 7.7291e-05 gnorm: 0.89 [16:01:06< 5:59:22] +[titan] 2025-07-10 05:36:12,916 - root - INFO - step: 72790 loss: 15.9693 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.4357 global_avg_mtp_loss: 13.5337 +[titan] 2025-07-10 05:36:12,917 - root - INFO - lr: 7.7275e-05 gnorm: 0.95 [16:01:10< 5:59:18] +[titan] 2025-07-10 05:36:16,882 - root - INFO - step: 72795 loss: 16.1579 memory: 44.58GiB(31.99%) tps: 82,644 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6975 +[titan] 2025-07-10 05:36:16,882 - root - INFO - lr: 7.7258e-05 gnorm: 0.94 [16:01:14< 5:59:14] +[titan] 2025-07-10 05:36:20,009 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:36:20,797 - root - INFO - step: 72800 loss: 15.8395 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4212 global_avg_mtp_loss: 13.4183 +[titan] 2025-07-10 05:36:20,797 - root - INFO - lr: 7.7242e-05 gnorm: 0.95 [16:01:18< 5:59:10] +[titan] 2025-07-10 05:36:24,727 - root - INFO - step: 72805 loss: 16.2715 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7853 +[titan] 2025-07-10 05:36:24,727 - root - INFO - lr: 7.7226e-05 gnorm: 0.98 [16:01:22< 5:59:06] +[titan] 2025-07-10 05:36:28,660 - root - INFO - step: 72810 loss: 16.0604 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.6147 +[titan] 2025-07-10 05:36:28,660 - root - INFO - lr: 7.7210e-05 gnorm: 0.91 [16:01:26< 5:59:02] +[titan] 2025-07-10 05:36:32,588 - root - INFO - step: 72815 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6881 +[titan] 2025-07-10 05:36:32,589 - root - INFO - lr: 7.7193e-05 gnorm: 0.89 [16:01:30< 5:58:58] +[titan] 2025-07-10 05:36:36,514 - root - INFO - step: 72820 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.7924 +[titan] 2025-07-10 05:36:36,515 - root - INFO - lr: 7.7177e-05 gnorm: 0.97 [16:01:34< 5:58:54] +[titan] 2025-07-10 05:36:40,462 - root - INFO - step: 72825 loss: 16.3661 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.8546 +[titan] 2025-07-10 05:36:40,462 - root - INFO - lr: 7.7161e-05 gnorm: 0.96 [16:01:38< 5:58:50] +[titan] 2025-07-10 05:36:44,383 - root - INFO - step: 72830 loss: 16.4167 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.9063 +[titan] 2025-07-10 05:36:44,384 - root - INFO - lr: 7.7145e-05 gnorm: 0.93 [16:01:42< 5:58:46] +[titan] 2025-07-10 05:36:48,327 - root - INFO - step: 72835 loss: 16.6268 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.5538 global_avg_mtp_loss: 14.0730 +[titan] 2025-07-10 05:36:48,328 - root - INFO - lr: 7.7128e-05 gnorm: 0.97 [16:01:46< 5:58:42] +[titan] 2025-07-10 05:36:52,229 - root - INFO - step: 72840 loss: 16.2463 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 05:36:52,229 - root - INFO - lr: 7.7112e-05 gnorm: 0.97 [16:01:50< 5:58:38] +[titan] 2025-07-10 05:36:56,160 - root - INFO - step: 72845 loss: 16.3293 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8309 +[titan] 2025-07-10 05:36:56,160 - root - INFO - lr: 7.7096e-05 gnorm: 0.98 [16:01:54< 5:58:34] +[titan] 2025-07-10 05:36:59,294 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:37:00,102 - root - INFO - step: 72850 loss: 16.0339 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.5702 +[titan] 2025-07-10 05:37:00,102 - root - INFO - lr: 7.7080e-05 gnorm: 0.93 [16:01:58< 5:58:30] +[titan] 2025-07-10 05:37:04,050 - root - INFO - step: 72855 loss: 16.2304 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.4749 global_avg_mtp_loss: 13.7555 +[titan] 2025-07-10 05:37:04,050 - root - INFO - lr: 7.7063e-05 gnorm: 0.89 [16:02:01< 5:58:26] +[titan] 2025-07-10 05:37:07,966 - root - INFO - step: 72860 loss: 16.3628 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 05:37:07,967 - root - INFO - lr: 7.7047e-05 gnorm: 0.94 [16:02:05< 5:58:22] +[titan] 2025-07-10 05:37:11,901 - root - INFO - step: 72865 loss: 16.2187 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 05:37:11,901 - root - INFO - lr: 7.7031e-05 gnorm: 0.93 [16:02:09< 5:58:18] +[titan] 2025-07-10 05:37:15,859 - root - INFO - step: 72870 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 82,788 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7353 +[titan] 2025-07-10 05:37:15,859 - root - INFO - lr: 7.7015e-05 gnorm: 0.88 [16:02:13< 5:58:14] +[titan] 2025-07-10 05:37:19,759 - root - INFO - step: 72875 loss: 16.1748 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.7080 +[titan] 2025-07-10 05:37:19,760 - root - INFO - lr: 7.6998e-05 gnorm: 0.90 [16:02:17< 5:58:10] +[titan] 2025-07-10 05:37:23,684 - root - INFO - step: 72880 loss: 16.1884 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7167 +[titan] 2025-07-10 05:37:23,684 - root - INFO - lr: 7.6982e-05 gnorm: 0.97 [16:02:21< 5:58:06] +[titan] 2025-07-10 05:37:27,635 - root - INFO - step: 72885 loss: 16.2942 memory: 44.58GiB(31.99%) tps: 82,949 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.7934 +[titan] 2025-07-10 05:37:27,635 - root - INFO - lr: 7.6966e-05 gnorm: 0.94 [16:02:25< 5:58:02] +[titan] 2025-07-10 05:37:31,560 - root - INFO - step: 72890 loss: 16.2899 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 05:37:31,560 - root - INFO - lr: 7.6950e-05 gnorm: 0.91 [16:02:29< 5:57:58] +[titan] 2025-07-10 05:37:35,454 - root - INFO - step: 72895 loss: 16.2589 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7766 +[titan] 2025-07-10 05:37:35,454 - root - INFO - lr: 7.6933e-05 gnorm: 0.93 [16:02:33< 5:57:54] +[titan] 2025-07-10 05:37:38,571 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:37:39,360 - root - INFO - step: 72900 loss: 16.0076 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4373 global_avg_mtp_loss: 13.5704 +[titan] 2025-07-10 05:37:39,361 - root - INFO - lr: 7.6917e-05 gnorm: 1.42 [16:02:37< 5:57:50] +[titan] 2025-07-10 05:37:43,270 - root - INFO - step: 72905 loss: 16.4054 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8946 +[titan] 2025-07-10 05:37:43,270 - root - INFO - lr: 7.6901e-05 gnorm: 0.92 [16:02:41< 5:57:46] +[titan] 2025-07-10 05:37:47,201 - root - INFO - step: 72910 loss: 16.2651 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 05:37:47,201 - root - INFO - lr: 7.6885e-05 gnorm: 0.93 [16:02:45< 5:57:42] +[titan] 2025-07-10 05:37:51,119 - root - INFO - step: 72915 loss: 16.2993 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.8021 +[titan] 2025-07-10 05:37:51,119 - root - INFO - lr: 7.6868e-05 gnorm: 0.89 [16:02:49< 5:57:38] +[titan] 2025-07-10 05:37:55,031 - root - INFO - step: 72920 loss: 16.3612 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.8564 +[titan] 2025-07-10 05:37:55,031 - root - INFO - lr: 7.6852e-05 gnorm: 1.12 [16:02:52< 5:57:34] +[titan] 2025-07-10 05:37:58,978 - root - INFO - step: 72925 loss: 16.2412 memory: 44.58GiB(31.99%) tps: 83,017 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7548 +[titan] 2025-07-10 05:37:58,978 - root - INFO - lr: 7.6836e-05 gnorm: 0.94 [16:02:56< 5:57:30] +[titan] 2025-07-10 05:38:02,900 - root - INFO - step: 72930 loss: 16.2015 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7209 +[titan] 2025-07-10 05:38:02,901 - root - INFO - lr: 7.6820e-05 gnorm: 0.96 [16:03:00< 5:57:26] +[titan] 2025-07-10 05:38:06,832 - root - INFO - step: 72935 loss: 16.1469 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.6837 +[titan] 2025-07-10 05:38:06,832 - root - INFO - lr: 7.6804e-05 gnorm: 0.88 [16:03:04< 5:57:22] +[titan] 2025-07-10 05:38:10,730 - root - INFO - step: 72940 loss: 16.3647 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8666 +[titan] 2025-07-10 05:38:10,730 - root - INFO - lr: 7.6787e-05 gnorm: 0.90 [16:03:08< 5:57:19] +[titan] 2025-07-10 05:38:14,636 - root - INFO - step: 72945 loss: 16.2108 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7274 +[titan] 2025-07-10 05:38:14,636 - root - INFO - lr: 7.6771e-05 gnorm: 0.95 [16:03:12< 5:57:15] +[titan] 2025-07-10 05:38:17,782 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:38:18,569 - root - INFO - step: 72950 loss: 16.1275 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6601 +[titan] 2025-07-10 05:38:18,569 - root - INFO - lr: 7.6755e-05 gnorm: 0.97 [16:03:16< 5:57:11] +[titan] 2025-07-10 05:38:22,491 - root - INFO - step: 72955 loss: 16.5578 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5547 global_avg_mtp_loss: 14.0031 +[titan] 2025-07-10 05:38:22,491 - root - INFO - lr: 7.6739e-05 gnorm: 0.98 [16:03:20< 5:57:07] +[titan] 2025-07-10 05:38:26,404 - root - INFO - step: 72960 loss: 16.4112 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8955 +[titan] 2025-07-10 05:38:26,404 - root - INFO - lr: 7.6722e-05 gnorm: 0.94 [16:03:24< 5:57:03] +[titan] 2025-07-10 05:38:30,323 - root - INFO - step: 72965 loss: 16.4103 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8939 +[titan] 2025-07-10 05:38:30,323 - root - INFO - lr: 7.6706e-05 gnorm: 0.92 [16:03:28< 5:56:59] +[titan] 2025-07-10 05:38:34,241 - root - INFO - step: 72970 loss: 16.1851 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7075 +[titan] 2025-07-10 05:38:34,241 - root - INFO - lr: 7.6690e-05 gnorm: 0.90 [16:03:32< 5:56:55] +[titan] 2025-07-10 05:38:38,163 - root - INFO - step: 72975 loss: 16.2946 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.8123 +[titan] 2025-07-10 05:38:38,163 - root - INFO - lr: 7.6674e-05 gnorm: 0.90 [16:03:36< 5:56:51] +[titan] 2025-07-10 05:38:42,057 - root - INFO - step: 72980 loss: 16.0931 memory: 44.58GiB(31.99%) tps: 84,155 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6269 +[titan] 2025-07-10 05:38:42,057 - root - INFO - lr: 7.6658e-05 gnorm: 0.97 [16:03:39< 5:56:47] +[titan] 2025-07-10 05:38:45,962 - root - INFO - step: 72985 loss: 16.1678 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 05:38:45,962 - root - INFO - lr: 7.6641e-05 gnorm: 0.99 [16:03:43< 5:56:43] +[titan] 2025-07-10 05:38:49,858 - root - INFO - step: 72990 loss: 16.5285 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9942 +[titan] 2025-07-10 05:38:49,859 - root - INFO - lr: 7.6625e-05 gnorm: 0.97 [16:03:47< 5:56:39] +[titan] 2025-07-10 05:38:53,764 - root - INFO - step: 72995 loss: 16.2515 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7731 +[titan] 2025-07-10 05:38:53,764 - root - INFO - lr: 7.6609e-05 gnorm: 0.90 [16:03:51< 5:56:35] +[titan] 2025-07-10 05:38:56,927 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:38:57,724 - root - INFO - step: 73000 loss: 16.2048 memory: 44.58GiB(31.99%) tps: 82,748 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 05:38:57,725 - root - INFO - lr: 7.6593e-05 gnorm: 0.92 [16:03:55< 5:56:31] +[titan] 2025-07-10 05:39:01,628 - root - INFO - step: 73005 loss: 16.2757 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7905 +[titan] 2025-07-10 05:39:01,628 - root - INFO - lr: 7.6577e-05 gnorm: 0.95 [16:03:59< 5:56:27] +[titan] 2025-07-10 05:39:05,525 - root - INFO - step: 73010 loss: 16.2755 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.7725 +[titan] 2025-07-10 05:39:05,526 - root - INFO - lr: 7.6561e-05 gnorm: 0.92 [16:04:03< 5:56:23] +[titan] 2025-07-10 05:39:09,426 - root - INFO - step: 73015 loss: 16.1664 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.7030 +[titan] 2025-07-10 05:39:09,426 - root - INFO - lr: 7.6544e-05 gnorm: 0.88 [16:04:07< 5:56:19] +[titan] 2025-07-10 05:39:13,380 - root - INFO - step: 73020 loss: 16.2526 memory: 44.58GiB(31.99%) tps: 82,881 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7718 +[titan] 2025-07-10 05:39:13,380 - root - INFO - lr: 7.6528e-05 gnorm: 0.89 [16:04:11< 5:56:15] +[titan] 2025-07-10 05:39:17,275 - root - INFO - step: 73025 loss: 16.1461 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.6878 +[titan] 2025-07-10 05:39:17,275 - root - INFO - lr: 7.6512e-05 gnorm: 0.89 [16:04:15< 5:56:11] +[titan] 2025-07-10 05:39:21,191 - root - INFO - step: 73030 loss: 16.1312 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6681 +[titan] 2025-07-10 05:39:21,191 - root - INFO - lr: 7.6496e-05 gnorm: 0.95 [16:04:19< 5:56:07] +[titan] 2025-07-10 05:39:25,098 - root - INFO - step: 73035 loss: 16.4538 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9304 +[titan] 2025-07-10 05:39:25,098 - root - INFO - lr: 7.6480e-05 gnorm: 0.91 [16:04:23< 5:56:03] +[titan] 2025-07-10 05:39:28,996 - root - INFO - step: 73040 loss: 16.2090 memory: 44.58GiB(31.99%) tps: 84,084 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7336 +[titan] 2025-07-10 05:39:28,996 - root - INFO - lr: 7.6463e-05 gnorm: 1.07 [16:04:26< 5:55:59] +[titan] 2025-07-10 05:39:32,886 - root - INFO - step: 73045 loss: 16.4111 memory: 44.58GiB(31.99%) tps: 84,245 tflops: 290.74 mfu: 29.40% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8982 +[titan] 2025-07-10 05:39:32,886 - root - INFO - lr: 7.6447e-05 gnorm: 0.92 [16:04:30< 5:55:55] +[titan] 2025-07-10 05:39:36,010 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:39:36,796 - root - INFO - step: 73050 loss: 16.2593 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7801 +[titan] 2025-07-10 05:39:36,796 - root - INFO - lr: 7.6431e-05 gnorm: 0.92 [16:04:34< 5:55:51] +[titan] 2025-07-10 05:39:40,731 - root - INFO - step: 73055 loss: 16.2060 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 05:39:40,731 - root - INFO - lr: 7.6415e-05 gnorm: 0.92 [16:04:38< 5:55:47] +[titan] 2025-07-10 05:39:44,641 - root - INFO - step: 73060 loss: 16.5720 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5565 global_avg_mtp_loss: 14.0155 +[titan] 2025-07-10 05:39:44,641 - root - INFO - lr: 7.6399e-05 gnorm: 0.94 [16:04:42< 5:55:43] +[titan] 2025-07-10 05:39:48,552 - root - INFO - step: 73065 loss: 16.3901 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.8877 +[titan] 2025-07-10 05:39:48,552 - root - INFO - lr: 7.6383e-05 gnorm: 0.91 [16:04:46< 5:55:39] +[titan] 2025-07-10 05:39:52,489 - root - INFO - step: 73070 loss: 16.2797 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.7733 +[titan] 2025-07-10 05:39:52,490 - root - INFO - lr: 7.6367e-05 gnorm: 0.87 [16:04:50< 5:55:35] +[titan] 2025-07-10 05:39:56,383 - root - INFO - step: 73075 loss: 16.0308 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4366 global_avg_mtp_loss: 13.5942 +[titan] 2025-07-10 05:39:56,383 - root - INFO - lr: 7.6350e-05 gnorm: 0.93 [16:04:54< 5:55:31] +[titan] 2025-07-10 05:40:00,290 - root - INFO - step: 73080 loss: 16.2615 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.7568 +[titan] 2025-07-10 05:40:00,290 - root - INFO - lr: 7.6334e-05 gnorm: 1.07 [16:04:58< 5:55:27] +[titan] 2025-07-10 05:40:04,241 - root - INFO - step: 73085 loss: 16.3614 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 05:40:04,241 - root - INFO - lr: 7.6318e-05 gnorm: 1.06 [16:05:02< 5:55:23] +[titan] 2025-07-10 05:40:08,173 - root - INFO - step: 73090 loss: 16.0533 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.5956 +[titan] 2025-07-10 05:40:08,173 - root - INFO - lr: 7.6302e-05 gnorm: 0.95 [16:05:06< 5:55:19] +[titan] 2025-07-10 05:40:12,108 - root - INFO - step: 73095 loss: 16.4466 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9261 +[titan] 2025-07-10 05:40:12,108 - root - INFO - lr: 7.6286e-05 gnorm: 0.94 [16:05:10< 5:55:15] +[titan] 2025-07-10 05:40:15,275 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:40:16,063 - root - INFO - step: 73100 loss: 16.2374 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7571 +[titan] 2025-07-10 05:40:16,063 - root - INFO - lr: 7.6270e-05 gnorm: 0.93 [16:05:13< 5:55:11] +[titan] 2025-07-10 05:40:20,036 - root - INFO - step: 73105 loss: 16.4208 memory: 44.58GiB(31.99%) tps: 82,479 tflops: 284.65 mfu: 28.78% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.8857 +[titan] 2025-07-10 05:40:20,036 - root - INFO - lr: 7.6253e-05 gnorm: 0.96 [16:05:17< 5:55:07] +[titan] 2025-07-10 05:40:23,976 - root - INFO - step: 73110 loss: 16.2178 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7468 +[titan] 2025-07-10 05:40:23,976 - root - INFO - lr: 7.6237e-05 gnorm: 0.88 [16:05:21< 5:55:03] +[titan] 2025-07-10 05:40:27,898 - root - INFO - step: 73115 loss: 16.3530 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8525 +[titan] 2025-07-10 05:40:27,898 - root - INFO - lr: 7.6221e-05 gnorm: 0.91 [16:05:25< 5:54:59] +[titan] 2025-07-10 05:40:31,809 - root - INFO - step: 73120 loss: 16.2944 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.7938 +[titan] 2025-07-10 05:40:31,809 - root - INFO - lr: 7.6205e-05 gnorm: 0.93 [16:05:29< 5:54:55] +[titan] 2025-07-10 05:40:35,754 - root - INFO - step: 73125 loss: 16.2439 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7559 +[titan] 2025-07-10 05:40:35,754 - root - INFO - lr: 7.6189e-05 gnorm: 0.95 [16:05:33< 5:54:51] +[titan] 2025-07-10 05:40:39,650 - root - INFO - step: 73130 loss: 16.5052 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5294 global_avg_mtp_loss: 13.9758 +[titan] 2025-07-10 05:40:39,651 - root - INFO - lr: 7.6173e-05 gnorm: 0.94 [16:05:37< 5:54:47] +[titan] 2025-07-10 05:40:43,602 - root - INFO - step: 73135 loss: 16.3762 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8614 +[titan] 2025-07-10 05:40:43,602 - root - INFO - lr: 7.6157e-05 gnorm: 0.90 [16:05:41< 5:54:43] +[titan] 2025-07-10 05:40:47,518 - root - INFO - step: 73140 loss: 16.3094 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.8219 +[titan] 2025-07-10 05:40:47,518 - root - INFO - lr: 7.6141e-05 gnorm: 0.88 [16:05:45< 5:54:39] +[titan] 2025-07-10 05:40:51,438 - root - INFO - step: 73145 loss: 16.3110 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8134 +[titan] 2025-07-10 05:40:51,438 - root - INFO - lr: 7.6124e-05 gnorm: 0.89 [16:05:49< 5:54:35] +[titan] 2025-07-10 05:40:54,570 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:40:55,354 - root - INFO - step: 73150 loss: 16.1226 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6639 +[titan] 2025-07-10 05:40:55,354 - root - INFO - lr: 7.6108e-05 gnorm: 0.93 [16:05:53< 5:54:31] +[titan] 2025-07-10 05:40:59,302 - root - INFO - step: 73155 loss: 16.2246 memory: 44.58GiB(31.99%) tps: 83,003 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7327 +[titan] 2025-07-10 05:40:59,302 - root - INFO - lr: 7.6092e-05 gnorm: 0.90 [16:05:57< 5:54:28] +[titan] 2025-07-10 05:41:03,221 - root - INFO - step: 73160 loss: 16.1968 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7068 +[titan] 2025-07-10 05:41:03,221 - root - INFO - lr: 7.6076e-05 gnorm: 0.99 [16:06:01< 5:54:24] +[titan] 2025-07-10 05:41:07,159 - root - INFO - step: 73165 loss: 16.0253 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5851 +[titan] 2025-07-10 05:41:07,159 - root - INFO - lr: 7.6060e-05 gnorm: 0.95 [16:06:05< 5:54:20] +[titan] 2025-07-10 05:41:11,069 - root - INFO - step: 73170 loss: 16.2759 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.7698 +[titan] 2025-07-10 05:41:11,069 - root - INFO - lr: 7.6044e-05 gnorm: 0.94 [16:06:08< 5:54:16] +[titan] 2025-07-10 05:41:15,004 - root - INFO - step: 73175 loss: 16.2472 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7696 +[titan] 2025-07-10 05:41:15,004 - root - INFO - lr: 7.6028e-05 gnorm: 0.96 [16:06:12< 5:54:12] +[titan] 2025-07-10 05:41:18,940 - root - INFO - step: 73180 loss: 16.4341 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 05:41:18,940 - root - INFO - lr: 7.6012e-05 gnorm: 0.94 [16:06:16< 5:54:08] +[titan] 2025-07-10 05:41:22,904 - root - INFO - step: 73185 loss: 15.8995 memory: 44.58GiB(31.99%) tps: 82,669 tflops: 285.30 mfu: 28.85% global_avg_ntp_loss: 2.4238 global_avg_mtp_loss: 13.4757 +[titan] 2025-07-10 05:41:22,904 - root - INFO - lr: 7.5995e-05 gnorm: 0.93 [16:06:20< 5:54:04] +[titan] 2025-07-10 05:41:26,834 - root - INFO - step: 73190 loss: 16.3552 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8395 +[titan] 2025-07-10 05:41:26,834 - root - INFO - lr: 7.5979e-05 gnorm: 0.95 [16:06:24< 5:54:00] +[titan] 2025-07-10 05:41:30,772 - root - INFO - step: 73195 loss: 16.2077 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.7261 +[titan] 2025-07-10 05:41:30,772 - root - INFO - lr: 7.5963e-05 gnorm: 0.91 [16:06:28< 5:53:56] +[titan] 2025-07-10 05:41:33,904 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:41:34,698 - root - INFO - step: 73200 loss: 16.3596 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8590 +[titan] 2025-07-10 05:41:34,698 - root - INFO - lr: 7.5947e-05 gnorm: 0.93 [16:06:32< 5:53:52] +[titan] 2025-07-10 05:41:38,623 - root - INFO - step: 73205 loss: 16.3273 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.8480 +[titan] 2025-07-10 05:41:38,623 - root - INFO - lr: 7.5931e-05 gnorm: 0.90 [16:06:36< 5:53:48] +[titan] 2025-07-10 05:41:42,537 - root - INFO - step: 73210 loss: 16.5931 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 14.0444 +[titan] 2025-07-10 05:41:42,537 - root - INFO - lr: 7.5915e-05 gnorm: 0.91 [16:06:40< 5:53:44] +[titan] 2025-07-10 05:41:46,578 - root - INFO - step: 73215 loss: 16.1399 memory: 44.58GiB(31.99%) tps: 81,102 tflops: 279.90 mfu: 28.30% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 05:41:46,578 - root - INFO - lr: 7.5899e-05 gnorm: 0.92 [16:06:44< 5:53:40] +[titan] 2025-07-10 05:41:47,520 - root - INFO - Dumping profiler traces at step 73216 +[titan] 2025-07-10 05:41:47,553 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:41:50,765 - root - INFO - step: 73220 loss: 16.2635 memory: 44.58GiB(31.99%) tps: 78,257 tflops: 270.08 mfu: 27.31% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7682 +[titan] 2025-07-10 05:41:50,766 - root - INFO - lr: 7.5883e-05 gnorm: 0.90 [16:06:48< 5:53:36] +[titan] 2025-07-10 05:41:54,686 - root - INFO - step: 73225 loss: 16.1436 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.6645 +[titan] 2025-07-10 05:41:54,686 - root - INFO - lr: 7.5867e-05 gnorm: 0.91 [16:06:52< 5:53:32] +[titan] 2025-07-10 05:41:58,633 - root - INFO - step: 73230 loss: 16.2241 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7335 +[titan] 2025-07-10 05:41:58,633 - root - INFO - lr: 7.5851e-05 gnorm: 0.95 [16:06:56< 5:53:28] +[titan] 2025-07-10 05:42:02,542 - root - INFO - step: 73235 loss: 16.4074 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.8892 +[titan] 2025-07-10 05:42:02,543 - root - INFO - lr: 7.5835e-05 gnorm: 0.89 [16:07:00< 5:53:24] +[titan] 2025-07-10 05:42:06,466 - root - INFO - step: 73240 loss: 16.4031 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.8896 +[titan] 2025-07-10 05:42:06,466 - root - INFO - lr: 7.5818e-05 gnorm: 0.97 [16:07:04< 5:53:20] +[titan] 2025-07-10 05:42:10,407 - root - INFO - step: 73245 loss: 16.3948 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8817 +[titan] 2025-07-10 05:42:10,408 - root - INFO - lr: 7.5802e-05 gnorm: 0.94 [16:07:08< 5:53:16] +[titan] 2025-07-10 05:42:13,512 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:42:14,304 - root - INFO - step: 73250 loss: 16.2539 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7723 +[titan] 2025-07-10 05:42:14,304 - root - INFO - lr: 7.5786e-05 gnorm: 0.92 [16:07:12< 5:53:12] +[titan] 2025-07-10 05:42:18,227 - root - INFO - step: 73255 loss: 16.5212 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.9881 +[titan] 2025-07-10 05:42:18,227 - root - INFO - lr: 7.5770e-05 gnorm: 0.89 [16:07:16< 5:53:08] +[titan] 2025-07-10 05:42:22,139 - root - INFO - step: 73260 loss: 16.1087 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4424 global_avg_mtp_loss: 13.6663 +[titan] 2025-07-10 05:42:22,139 - root - INFO - lr: 7.5754e-05 gnorm: 0.91 [16:07:20< 5:53:04] +[titan] 2025-07-10 05:42:26,079 - root - INFO - step: 73265 loss: 16.0143 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4332 global_avg_mtp_loss: 13.5811 +[titan] 2025-07-10 05:42:26,079 - root - INFO - lr: 7.5738e-05 gnorm: 0.93 [16:07:23< 5:53:00] +[titan] 2025-07-10 05:42:29,978 - root - INFO - step: 73270 loss: 15.9207 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4344 global_avg_mtp_loss: 13.4862 +[titan] 2025-07-10 05:42:29,979 - root - INFO - lr: 7.5722e-05 gnorm: 0.93 [16:07:27< 5:52:56] +[titan] 2025-07-10 05:42:33,924 - root - INFO - step: 73275 loss: 16.2621 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7795 +[titan] 2025-07-10 05:42:33,924 - root - INFO - lr: 7.5706e-05 gnorm: 0.91 [16:07:31< 5:52:52] +[titan] 2025-07-10 05:42:37,820 - root - INFO - step: 73280 loss: 16.4283 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.9200 +[titan] 2025-07-10 05:42:37,821 - root - INFO - lr: 7.5690e-05 gnorm: 0.98 [16:07:35< 5:52:48] +[titan] 2025-07-10 05:42:41,739 - root - INFO - step: 73285 loss: 16.3786 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8783 +[titan] 2025-07-10 05:42:41,740 - root - INFO - lr: 7.5674e-05 gnorm: 0.96 [16:07:39< 5:52:44] +[titan] 2025-07-10 05:42:45,677 - root - INFO - step: 73290 loss: 16.3022 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8120 +[titan] 2025-07-10 05:42:45,677 - root - INFO - lr: 7.5658e-05 gnorm: 0.94 [16:07:43< 5:52:40] +[titan] 2025-07-10 05:42:49,568 - root - INFO - step: 73295 loss: 16.3190 memory: 44.58GiB(31.99%) tps: 84,211 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8252 +[titan] 2025-07-10 05:42:49,568 - root - INFO - lr: 7.5642e-05 gnorm: 0.95 [16:07:47< 5:52:36] +[titan] 2025-07-10 05:42:52,710 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:42:53,497 - root - INFO - step: 73300 loss: 16.3050 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8072 +[titan] 2025-07-10 05:42:53,497 - root - INFO - lr: 7.5626e-05 gnorm: 0.99 [16:07:51< 5:52:32] +[titan] 2025-07-10 05:42:57,480 - root - INFO - step: 73305 loss: 16.3133 memory: 44.58GiB(31.99%) tps: 82,278 tflops: 283.95 mfu: 28.71% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8141 +[titan] 2025-07-10 05:42:57,480 - root - INFO - lr: 7.5610e-05 gnorm: 0.93 [16:07:55< 5:52:28] +[titan] 2025-07-10 05:43:01,418 - root - INFO - step: 73310 loss: 16.2652 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7840 +[titan] 2025-07-10 05:43:01,418 - root - INFO - lr: 7.5594e-05 gnorm: 0.97 [16:07:59< 5:52:24] +[titan] 2025-07-10 05:43:05,381 - root - INFO - step: 73315 loss: 16.1984 memory: 44.58GiB(31.99%) tps: 82,705 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7251 +[titan] 2025-07-10 05:43:05,381 - root - INFO - lr: 7.5578e-05 gnorm: 0.91 [16:08:03< 5:52:20] +[titan] 2025-07-10 05:43:09,317 - root - INFO - step: 73320 loss: 16.1241 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6500 +[titan] 2025-07-10 05:43:09,317 - root - INFO - lr: 7.5561e-05 gnorm: 0.92 [16:08:07< 5:52:17] +[titan] 2025-07-10 05:43:13,258 - root - INFO - step: 73325 loss: 16.4075 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8947 +[titan] 2025-07-10 05:43:13,259 - root - INFO - lr: 7.5545e-05 gnorm: 0.96 [16:08:11< 5:52:13] +[titan] 2025-07-10 05:43:17,214 - root - INFO - step: 73330 loss: 16.3365 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8421 +[titan] 2025-07-10 05:43:17,214 - root - INFO - lr: 7.5529e-05 gnorm: 0.95 [16:08:15< 5:52:09] +[titan] 2025-07-10 05:43:21,139 - root - INFO - step: 73335 loss: 16.1933 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-10 05:43:21,140 - root - INFO - lr: 7.5513e-05 gnorm: 0.90 [16:08:19< 5:52:05] +[titan] 2025-07-10 05:43:25,071 - root - INFO - step: 73340 loss: 16.5175 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5325 global_avg_mtp_loss: 13.9849 +[titan] 2025-07-10 05:43:25,071 - root - INFO - lr: 7.5497e-05 gnorm: 0.93 [16:08:22< 5:52:01] +[titan] 2025-07-10 05:43:29,027 - root - INFO - step: 73345 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 82,841 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8700 +[titan] 2025-07-10 05:43:29,027 - root - INFO - lr: 7.5481e-05 gnorm: 0.92 [16:08:26< 5:51:57] +[titan] 2025-07-10 05:43:32,164 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:43:32,949 - root - INFO - step: 73350 loss: 16.4968 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9705 +[titan] 2025-07-10 05:43:32,949 - root - INFO - lr: 7.5465e-05 gnorm: 1.14 [16:08:30< 5:51:53] +[titan] 2025-07-10 05:43:36,871 - root - INFO - step: 73355 loss: 16.3404 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8328 +[titan] 2025-07-10 05:43:36,871 - root - INFO - lr: 7.5449e-05 gnorm: 0.95 [16:08:34< 5:51:49] +[titan] 2025-07-10 05:43:40,805 - root - INFO - step: 73360 loss: 16.1965 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4691 global_avg_mtp_loss: 13.7274 +[titan] 2025-07-10 05:43:40,805 - root - INFO - lr: 7.5433e-05 gnorm: 0.95 [16:08:38< 5:51:45] +[titan] 2025-07-10 05:43:44,722 - root - INFO - step: 73365 loss: 16.3122 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.7973 +[titan] 2025-07-10 05:43:44,723 - root - INFO - lr: 7.5417e-05 gnorm: 0.94 [16:08:42< 5:51:41] +[titan] 2025-07-10 05:43:48,630 - root - INFO - step: 73370 loss: 16.3753 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.8537 +[titan] 2025-07-10 05:43:48,631 - root - INFO - lr: 7.5401e-05 gnorm: 0.92 [16:08:46< 5:51:37] +[titan] 2025-07-10 05:43:52,564 - root - INFO - step: 73375 loss: 16.4500 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5387 global_avg_mtp_loss: 13.9113 +[titan] 2025-07-10 05:43:52,564 - root - INFO - lr: 7.5385e-05 gnorm: 0.97 [16:08:50< 5:51:33] +[titan] 2025-07-10 05:43:56,494 - root - INFO - step: 73380 loss: 16.4815 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.5222 global_avg_mtp_loss: 13.9593 +[titan] 2025-07-10 05:43:56,494 - root - INFO - lr: 7.5369e-05 gnorm: 0.90 [16:08:54< 5:51:29] +[titan] 2025-07-10 05:44:00,388 - root - INFO - step: 73385 loss: 16.3366 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8316 +[titan] 2025-07-10 05:44:00,388 - root - INFO - lr: 7.5353e-05 gnorm: 0.89 [16:08:58< 5:51:25] +[titan] 2025-07-10 05:44:04,285 - root - INFO - step: 73390 loss: 16.3369 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8422 +[titan] 2025-07-10 05:44:04,286 - root - INFO - lr: 7.5337e-05 gnorm: 0.98 [16:09:02< 5:51:21] +[titan] 2025-07-10 05:44:08,187 - root - INFO - step: 73395 loss: 16.5512 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 14.0099 +[titan] 2025-07-10 05:44:08,188 - root - INFO - lr: 7.5321e-05 gnorm: 0.98 [16:09:06< 5:51:17] +[titan] 2025-07-10 05:44:11,295 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:44:12,096 - root - INFO - step: 73400 loss: 16.1275 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6645 +[titan] 2025-07-10 05:44:12,096 - root - INFO - lr: 7.5305e-05 gnorm: 0.94 [16:09:09< 5:51:13] +[titan] 2025-07-10 05:44:16,061 - root - INFO - step: 73405 loss: 15.8793 memory: 44.58GiB(31.99%) tps: 82,644 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.4243 global_avg_mtp_loss: 13.4550 +[titan] 2025-07-10 05:44:16,061 - root - INFO - lr: 7.5289e-05 gnorm: 0.94 [16:09:13< 5:51:09] +[titan] 2025-07-10 05:44:19,986 - root - INFO - step: 73410 loss: 16.4302 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.9195 +[titan] 2025-07-10 05:44:19,986 - root - INFO - lr: 7.5273e-05 gnorm: 0.93 [16:09:17< 5:51:05] +[titan] 2025-07-10 05:44:23,903 - root - INFO - step: 73415 loss: 16.6200 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5579 global_avg_mtp_loss: 14.0620 +[titan] 2025-07-10 05:44:23,904 - root - INFO - lr: 7.5257e-05 gnorm: 0.93 [16:09:21< 5:51:01] +[titan] 2025-07-10 05:44:27,835 - root - INFO - step: 73420 loss: 16.2483 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7574 +[titan] 2025-07-10 05:44:27,835 - root - INFO - lr: 7.5241e-05 gnorm: 0.91 [16:09:25< 5:50:57] +[titan] 2025-07-10 05:44:31,739 - root - INFO - step: 73425 loss: 16.5207 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 13.9841 +[titan] 2025-07-10 05:44:31,739 - root - INFO - lr: 7.5225e-05 gnorm: 0.90 [16:09:29< 5:50:53] +[titan] 2025-07-10 05:44:35,640 - root - INFO - step: 73430 loss: 16.3944 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.8670 +[titan] 2025-07-10 05:44:35,641 - root - INFO - lr: 7.5209e-05 gnorm: 0.91 [16:09:33< 5:50:49] +[titan] 2025-07-10 05:44:39,545 - root - INFO - step: 73435 loss: 16.2846 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.8073 +[titan] 2025-07-10 05:44:39,545 - root - INFO - lr: 7.5193e-05 gnorm: 0.90 [16:09:37< 5:50:45] +[titan] 2025-07-10 05:44:43,466 - root - INFO - step: 73440 loss: 16.3508 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8461 +[titan] 2025-07-10 05:44:43,467 - root - INFO - lr: 7.5177e-05 gnorm: 0.92 [16:09:41< 5:50:41] +[titan] 2025-07-10 05:44:47,424 - root - INFO - step: 73445 loss: 15.6988 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.3908 global_avg_mtp_loss: 13.3080 +[titan] 2025-07-10 05:44:47,424 - root - INFO - lr: 7.5161e-05 gnorm: 0.97 [16:09:45< 5:50:37] +[titan] 2025-07-10 05:44:50,596 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:44:51,395 - root - INFO - step: 73450 loss: 16.1859 memory: 44.58GiB(31.99%) tps: 82,508 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.7141 +[titan] 2025-07-10 05:44:51,396 - root - INFO - lr: 7.5145e-05 gnorm: 0.96 [16:09:49< 5:50:33] +[titan] 2025-07-10 05:44:55,304 - root - INFO - step: 73455 loss: 16.3082 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.8221 +[titan] 2025-07-10 05:44:55,304 - root - INFO - lr: 7.5129e-05 gnorm: 0.98 [16:09:53< 5:50:29] +[titan] 2025-07-10 05:44:59,262 - root - INFO - step: 73460 loss: 16.5589 memory: 44.58GiB(31.99%) tps: 82,806 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.5642 global_avg_mtp_loss: 13.9946 +[titan] 2025-07-10 05:44:59,262 - root - INFO - lr: 7.5113e-05 gnorm: 0.96 [16:09:57< 5:50:25] +[titan] 2025-07-10 05:45:03,178 - root - INFO - step: 73465 loss: 16.4215 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.8969 +[titan] 2025-07-10 05:45:03,179 - root - INFO - lr: 7.5097e-05 gnorm: 1.00 [16:10:01< 5:50:21] +[titan] 2025-07-10 05:45:07,072 - root - INFO - step: 73470 loss: 16.5611 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5471 global_avg_mtp_loss: 14.0140 +[titan] 2025-07-10 05:45:07,072 - root - INFO - lr: 7.5081e-05 gnorm: 0.91 [16:10:04< 5:50:17] +[titan] 2025-07-10 05:45:11,003 - root - INFO - step: 73475 loss: 16.2878 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7950 +[titan] 2025-07-10 05:45:11,003 - root - INFO - lr: 7.5065e-05 gnorm: 0.92 [16:10:08< 5:50:13] +[titan] 2025-07-10 05:45:14,921 - root - INFO - step: 73480 loss: 16.2114 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7366 +[titan] 2025-07-10 05:45:14,921 - root - INFO - lr: 7.5049e-05 gnorm: 0.98 [16:10:12< 5:50:09] +[titan] 2025-07-10 05:45:18,853 - root - INFO - step: 73485 loss: 16.1753 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7029 +[titan] 2025-07-10 05:45:18,854 - root - INFO - lr: 7.5033e-05 gnorm: 1.41 [16:10:16< 5:50:05] +[titan] 2025-07-10 05:45:22,762 - root - INFO - step: 73490 loss: 16.2875 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.7915 +[titan] 2025-07-10 05:45:22,762 - root - INFO - lr: 7.5017e-05 gnorm: 0.89 [16:10:20< 5:50:01] +[titan] 2025-07-10 05:45:26,676 - root - INFO - step: 73495 loss: 16.3488 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8509 +[titan] 2025-07-10 05:45:26,676 - root - INFO - lr: 7.5001e-05 gnorm: 0.94 [16:10:24< 5:49:57] +[titan] 2025-07-10 05:45:29,818 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:45:30,605 - root - INFO - step: 73500 loss: 16.2175 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7391 +[titan] 2025-07-10 05:45:30,605 - root - INFO - lr: 7.4985e-05 gnorm: 0.93 [16:10:28< 5:49:53] +[titan] 2025-07-10 05:45:34,530 - root - INFO - step: 73505 loss: 16.2499 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7640 +[titan] 2025-07-10 05:45:34,531 - root - INFO - lr: 7.4969e-05 gnorm: 0.92 [16:10:32< 5:49:49] +[titan] 2025-07-10 05:45:38,490 - root - INFO - step: 73510 loss: 16.1054 memory: 44.58GiB(31.99%) tps: 82,761 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4495 global_avg_mtp_loss: 13.6559 +[titan] 2025-07-10 05:45:38,490 - root - INFO - lr: 7.4953e-05 gnorm: 0.92 [16:10:36< 5:49:46] +[titan] 2025-07-10 05:45:42,412 - root - INFO - step: 73515 loss: 16.1321 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4679 global_avg_mtp_loss: 13.6642 +[titan] 2025-07-10 05:45:42,413 - root - INFO - lr: 7.4937e-05 gnorm: 0.93 [16:10:40< 5:49:42] +[titan] 2025-07-10 05:45:46,350 - root - INFO - step: 73520 loss: 16.4960 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9719 +[titan] 2025-07-10 05:45:46,350 - root - INFO - lr: 7.4921e-05 gnorm: 0.92 [16:10:44< 5:49:38] +[titan] 2025-07-10 05:45:50,288 - root - INFO - step: 73525 loss: 16.5179 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.5331 global_avg_mtp_loss: 13.9848 +[titan] 2025-07-10 05:45:50,288 - root - INFO - lr: 7.4906e-05 gnorm: 0.95 [16:10:48< 5:49:34] +[titan] 2025-07-10 05:45:54,276 - root - INFO - step: 73530 loss: 16.4901 memory: 44.58GiB(31.99%) tps: 82,177 tflops: 283.61 mfu: 28.68% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9553 +[titan] 2025-07-10 05:45:54,276 - root - INFO - lr: 7.4890e-05 gnorm: 0.95 [16:10:52< 5:49:30] +[titan] 2025-07-10 05:45:58,202 - root - INFO - step: 73535 loss: 16.4616 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9350 +[titan] 2025-07-10 05:45:58,202 - root - INFO - lr: 7.4874e-05 gnorm: 0.94 [16:10:56< 5:49:26] +[titan] 2025-07-10 05:46:02,112 - root - INFO - step: 73540 loss: 15.8413 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4178 global_avg_mtp_loss: 13.4235 +[titan] 2025-07-10 05:46:02,112 - root - INFO - lr: 7.4858e-05 gnorm: 1.02 [16:10:59< 5:49:22] +[titan] 2025-07-10 05:46:06,105 - root - INFO - step: 73545 loss: 16.5270 memory: 44.58GiB(31.99%) tps: 82,074 tflops: 283.25 mfu: 28.64% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 13.9852 +[titan] 2025-07-10 05:46:06,105 - root - INFO - lr: 7.4842e-05 gnorm: 0.94 [16:11:03< 5:49:18] +[titan] 2025-07-10 05:46:09,225 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:46:10,022 - root - INFO - step: 73550 loss: 15.8779 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4260 global_avg_mtp_loss: 13.4520 +[titan] 2025-07-10 05:46:10,023 - root - INFO - lr: 7.4826e-05 gnorm: 0.97 [16:11:07< 5:49:14] +[titan] 2025-07-10 05:46:13,978 - root - INFO - step: 73555 loss: 16.2922 memory: 44.58GiB(31.99%) tps: 82,843 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.7980 +[titan] 2025-07-10 05:46:13,978 - root - INFO - lr: 7.4810e-05 gnorm: 0.91 [16:11:11< 5:49:10] +[titan] 2025-07-10 05:46:17,889 - root - INFO - step: 73560 loss: 16.3426 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8481 +[titan] 2025-07-10 05:46:17,889 - root - INFO - lr: 7.4794e-05 gnorm: 0.93 [16:11:15< 5:49:06] +[titan] 2025-07-10 05:46:21,803 - root - INFO - step: 73565 loss: 16.3127 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8217 +[titan] 2025-07-10 05:46:21,804 - root - INFO - lr: 7.4778e-05 gnorm: 0.97 [16:11:19< 5:49:02] +[titan] 2025-07-10 05:46:25,755 - root - INFO - step: 73570 loss: 16.3372 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8291 +[titan] 2025-07-10 05:46:25,755 - root - INFO - lr: 7.4762e-05 gnorm: 0.94 [16:11:23< 5:48:58] +[titan] 2025-07-10 05:46:29,675 - root - INFO - step: 73575 loss: 15.9348 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4388 global_avg_mtp_loss: 13.4960 +[titan] 2025-07-10 05:46:29,675 - root - INFO - lr: 7.4746e-05 gnorm: 0.95 [16:11:27< 5:48:54] +[titan] 2025-07-10 05:46:33,594 - root - INFO - step: 73580 loss: 16.1753 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.7130 +[titan] 2025-07-10 05:46:33,594 - root - INFO - lr: 7.4730e-05 gnorm: 0.92 [16:11:31< 5:48:50] +[titan] 2025-07-10 05:46:37,513 - root - INFO - step: 73585 loss: 16.1303 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4691 global_avg_mtp_loss: 13.6612 +[titan] 2025-07-10 05:46:37,513 - root - INFO - lr: 7.4714e-05 gnorm: 0.92 [16:11:35< 5:48:46] +[titan] 2025-07-10 05:46:41,467 - root - INFO - step: 73590 loss: 16.1568 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.6880 +[titan] 2025-07-10 05:46:41,467 - root - INFO - lr: 7.4698e-05 gnorm: 0.92 [16:11:39< 5:48:42] +[titan] 2025-07-10 05:46:45,448 - root - INFO - step: 73595 loss: 16.7030 memory: 44.58GiB(31.99%) tps: 82,312 tflops: 284.07 mfu: 28.72% global_avg_ntp_loss: 2.5615 global_avg_mtp_loss: 14.1415 +[titan] 2025-07-10 05:46:45,448 - root - INFO - lr: 7.4682e-05 gnorm: 0.91 [16:11:43< 5:48:38] +[titan] 2025-07-10 05:46:48,579 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:46:49,377 - root - INFO - step: 73600 loss: 16.4157 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9029 +[titan] 2025-07-10 05:46:49,377 - root - INFO - lr: 7.4667e-05 gnorm: 0.94 [16:11:47< 5:48:34] +[titan] 2025-07-10 05:46:53,291 - root - INFO - step: 73605 loss: 16.3528 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8525 +[titan] 2025-07-10 05:46:53,292 - root - INFO - lr: 7.4651e-05 gnorm: 0.93 [16:11:51< 5:48:30] +[titan] 2025-07-10 05:46:57,252 - root - INFO - step: 73610 loss: 16.2794 memory: 44.58GiB(31.99%) tps: 82,745 tflops: 285.57 mfu: 28.87% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7869 +[titan] 2025-07-10 05:46:57,252 - root - INFO - lr: 7.4635e-05 gnorm: 0.92 [16:11:55< 5:48:26] +[titan] 2025-07-10 05:47:01,195 - root - INFO - step: 73615 loss: 16.4980 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9568 +[titan] 2025-07-10 05:47:01,195 - root - INFO - lr: 7.4619e-05 gnorm: 0.94 [16:11:59< 5:48:22] +[titan] 2025-07-10 05:47:05,107 - root - INFO - step: 73620 loss: 16.3014 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.8104 +[titan] 2025-07-10 05:47:05,107 - root - INFO - lr: 7.4603e-05 gnorm: 0.94 [16:12:02< 5:48:18] +[titan] 2025-07-10 05:47:09,028 - root - INFO - step: 73625 loss: 16.2194 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7394 +[titan] 2025-07-10 05:47:09,029 - root - INFO - lr: 7.4587e-05 gnorm: 0.96 [16:12:06< 5:48:14] +[titan] 2025-07-10 05:47:12,942 - root - INFO - step: 73630 loss: 15.9839 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4367 global_avg_mtp_loss: 13.5472 +[titan] 2025-07-10 05:47:12,942 - root - INFO - lr: 7.4571e-05 gnorm: 0.93 [16:12:10< 5:48:10] +[titan] 2025-07-10 05:47:16,858 - root - INFO - step: 73635 loss: 16.0469 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4592 global_avg_mtp_loss: 13.5876 +[titan] 2025-07-10 05:47:16,858 - root - INFO - lr: 7.4555e-05 gnorm: 0.95 [16:12:14< 5:48:06] +[titan] 2025-07-10 05:47:20,784 - root - INFO - step: 73640 loss: 16.5905 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5424 global_avg_mtp_loss: 14.0482 +[titan] 2025-07-10 05:47:20,785 - root - INFO - lr: 7.4539e-05 gnorm: 0.92 [16:12:18< 5:48:02] +[titan] 2025-07-10 05:47:24,687 - root - INFO - step: 73645 loss: 16.3656 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8625 +[titan] 2025-07-10 05:47:24,687 - root - INFO - lr: 7.4523e-05 gnorm: 0.96 [16:12:22< 5:47:58] +[titan] 2025-07-10 05:47:27,848 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:47:28,635 - root - INFO - step: 73650 loss: 16.3267 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.8405 +[titan] 2025-07-10 05:47:28,635 - root - INFO - lr: 7.4507e-05 gnorm: 0.92 [16:12:26< 5:47:54] +[titan] 2025-07-10 05:47:32,541 - root - INFO - step: 73655 loss: 16.3373 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8279 +[titan] 2025-07-10 05:47:32,541 - root - INFO - lr: 7.4492e-05 gnorm: 0.94 [16:12:30< 5:47:50] +[titan] 2025-07-10 05:47:36,470 - root - INFO - step: 73660 loss: 16.1494 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.6749 +[titan] 2025-07-10 05:47:36,470 - root - INFO - lr: 7.4476e-05 gnorm: 0.93 [16:12:34< 5:47:46] +[titan] 2025-07-10 05:47:40,404 - root - INFO - step: 73665 loss: 16.2624 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.7654 +[titan] 2025-07-10 05:47:40,405 - root - INFO - lr: 7.4460e-05 gnorm: 0.97 [16:12:38< 5:47:42] +[titan] 2025-07-10 05:47:44,336 - root - INFO - step: 73670 loss: 16.2565 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7743 +[titan] 2025-07-10 05:47:44,336 - root - INFO - lr: 7.4444e-05 gnorm: 0.89 [16:12:42< 5:47:38] +[titan] 2025-07-10 05:47:48,269 - root - INFO - step: 73675 loss: 16.4924 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.9650 +[titan] 2025-07-10 05:47:48,270 - root - INFO - lr: 7.4428e-05 gnorm: 0.93 [16:12:46< 5:47:34] +[titan] 2025-07-10 05:47:52,192 - root - INFO - step: 73680 loss: 16.0641 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.5886 +[titan] 2025-07-10 05:47:52,192 - root - INFO - lr: 7.4412e-05 gnorm: 2.90 [16:12:50< 5:47:30] +[titan] 2025-07-10 05:47:56,088 - root - INFO - step: 73685 loss: 16.3312 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8366 +[titan] 2025-07-10 05:47:56,088 - root - INFO - lr: 7.4396e-05 gnorm: 0.97 [16:12:53< 5:47:26] +[titan] 2025-07-10 05:47:59,984 - root - INFO - step: 73690 loss: 16.2779 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7892 +[titan] 2025-07-10 05:47:59,984 - root - INFO - lr: 7.4380e-05 gnorm: 0.95 [16:12:57< 5:47:23] +[titan] 2025-07-10 05:48:03,937 - root - INFO - step: 73695 loss: 16.3840 memory: 44.58GiB(31.99%) tps: 82,903 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.8529 +[titan] 2025-07-10 05:48:03,937 - root - INFO - lr: 7.4364e-05 gnorm: 1.00 [16:13:01< 5:47:19] +[titan] 2025-07-10 05:48:07,046 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:48:07,839 - root - INFO - step: 73700 loss: 16.5117 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9774 +[titan] 2025-07-10 05:48:07,840 - root - INFO - lr: 7.4349e-05 gnorm: 0.91 [16:13:05< 5:47:15] +[titan] 2025-07-10 05:48:11,785 - root - INFO - step: 73705 loss: 16.0077 memory: 44.58GiB(31.99%) tps: 83,049 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.4510 global_avg_mtp_loss: 13.5567 +[titan] 2025-07-10 05:48:11,786 - root - INFO - lr: 7.4333e-05 gnorm: 0.96 [16:13:09< 5:47:11] +[titan] 2025-07-10 05:48:15,738 - root - INFO - step: 73710 loss: 16.2325 memory: 44.58GiB(31.99%) tps: 82,919 tflops: 286.17 mfu: 28.93% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7449 +[titan] 2025-07-10 05:48:15,738 - root - INFO - lr: 7.4317e-05 gnorm: 0.95 [16:13:13< 5:47:07] +[titan] 2025-07-10 05:48:19,652 - root - INFO - step: 73715 loss: 16.6756 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.5704 global_avg_mtp_loss: 14.1052 +[titan] 2025-07-10 05:48:19,652 - root - INFO - lr: 7.4301e-05 gnorm: 0.92 [16:13:17< 5:47:03] +[titan] 2025-07-10 05:48:23,554 - root - INFO - step: 73720 loss: 16.2258 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7446 +[titan] 2025-07-10 05:48:23,555 - root - INFO - lr: 7.4285e-05 gnorm: 0.94 [16:13:21< 5:46:59] +[titan] 2025-07-10 05:48:27,600 - root - INFO - step: 73725 loss: 16.4096 memory: 44.58GiB(31.99%) tps: 81,008 tflops: 279.57 mfu: 28.27% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.9013 +[titan] 2025-07-10 05:48:27,600 - root - INFO - lr: 7.4269e-05 gnorm: 0.93 [16:13:25< 5:46:55] +[titan] 2025-07-10 05:48:30,102 - root - INFO - Dumping profiler traces at step 73728 +[titan] 2025-07-10 05:48:30,135 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:48:31,713 - root - INFO - step: 73730 loss: 16.3372 memory: 44.58GiB(31.99%) tps: 79,669 tflops: 274.95 mfu: 27.80% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8310 +[titan] 2025-07-10 05:48:31,714 - root - INFO - lr: 7.4253e-05 gnorm: 0.89 [16:13:29< 5:46:51] +[titan] 2025-07-10 05:48:35,620 - root - INFO - step: 73735 loss: 16.4162 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.9026 +[titan] 2025-07-10 05:48:35,620 - root - INFO - lr: 7.4238e-05 gnorm: 0.92 [16:13:33< 5:46:47] +[titan] 2025-07-10 05:48:39,560 - root - INFO - step: 73740 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8468 +[titan] 2025-07-10 05:48:39,560 - root - INFO - lr: 7.4222e-05 gnorm: 0.89 [16:13:37< 5:46:43] +[titan] 2025-07-10 05:48:43,502 - root - INFO - step: 73745 loss: 16.1395 memory: 44.58GiB(31.99%) tps: 83,122 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.6785 +[titan] 2025-07-10 05:48:43,503 - root - INFO - lr: 7.4206e-05 gnorm: 0.98 [16:13:41< 5:46:39] +[titan] 2025-07-10 05:48:46,603 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:48:47,393 - root - INFO - step: 73750 loss: 16.3165 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.5036 global_avg_mtp_loss: 13.8129 +[titan] 2025-07-10 05:48:47,393 - root - INFO - lr: 7.4190e-05 gnorm: 0.96 [16:13:45< 5:46:35] +[titan] 2025-07-10 05:48:51,295 - root - INFO - step: 73755 loss: 16.4037 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.8781 +[titan] 2025-07-10 05:48:51,296 - root - INFO - lr: 7.4174e-05 gnorm: 0.93 [16:13:49< 5:46:31] +[titan] 2025-07-10 05:48:55,213 - root - INFO - step: 73760 loss: 16.3973 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.8810 +[titan] 2025-07-10 05:48:55,213 - root - INFO - lr: 7.4158e-05 gnorm: 0.93 [16:13:53< 5:46:27] +[titan] 2025-07-10 05:48:59,174 - root - INFO - step: 73765 loss: 16.2853 memory: 44.58GiB(31.99%) tps: 82,726 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.7940 +[titan] 2025-07-10 05:48:59,174 - root - INFO - lr: 7.4142e-05 gnorm: 1.02 [16:13:57< 5:46:23] +[titan] 2025-07-10 05:49:03,073 - root - INFO - step: 73770 loss: 16.1341 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6679 +[titan] 2025-07-10 05:49:03,073 - root - INFO - lr: 7.4127e-05 gnorm: 0.98 [16:14:00< 5:46:19] +[titan] 2025-07-10 05:49:06,982 - root - INFO - step: 73775 loss: 16.3599 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 05:49:06,982 - root - INFO - lr: 7.4111e-05 gnorm: 0.99 [16:14:04< 5:46:15] +[titan] 2025-07-10 05:49:10,896 - root - INFO - step: 73780 loss: 16.4901 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-10 05:49:10,897 - root - INFO - lr: 7.4095e-05 gnorm: 0.97 [16:14:08< 5:46:11] +[titan] 2025-07-10 05:49:14,816 - root - INFO - step: 73785 loss: 16.3823 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.8614 +[titan] 2025-07-10 05:49:14,816 - root - INFO - lr: 7.4079e-05 gnorm: 0.93 [16:14:12< 5:46:07] +[titan] 2025-07-10 05:49:18,758 - root - INFO - step: 73790 loss: 16.1402 memory: 44.58GiB(31.99%) tps: 83,122 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6801 +[titan] 2025-07-10 05:49:18,758 - root - INFO - lr: 7.4063e-05 gnorm: 0.95 [16:14:16< 5:46:03] +[titan] 2025-07-10 05:49:22,689 - root - INFO - step: 73795 loss: 16.4485 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5216 global_avg_mtp_loss: 13.9269 +[titan] 2025-07-10 05:49:22,689 - root - INFO - lr: 7.4047e-05 gnorm: 0.95 [16:14:20< 5:45:59] +[titan] 2025-07-10 05:49:25,824 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:49:26,634 - root - INFO - step: 73800 loss: 16.3278 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8368 +[titan] 2025-07-10 05:49:26,635 - root - INFO - lr: 7.4032e-05 gnorm: 0.91 [16:14:24< 5:45:55] +[titan] 2025-07-10 05:49:30,539 - root - INFO - step: 73805 loss: 16.2171 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7379 +[titan] 2025-07-10 05:49:30,539 - root - INFO - lr: 7.4016e-05 gnorm: 0.90 [16:14:28< 5:45:51] +[titan] 2025-07-10 05:49:34,480 - root - INFO - step: 73810 loss: 16.2198 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7308 +[titan] 2025-07-10 05:49:34,480 - root - INFO - lr: 7.4000e-05 gnorm: 0.92 [16:14:32< 5:45:47] +[titan] 2025-07-10 05:49:38,406 - root - INFO - step: 73815 loss: 16.1314 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.6522 +[titan] 2025-07-10 05:49:38,407 - root - INFO - lr: 7.3984e-05 gnorm: 0.96 [16:14:36< 5:45:43] +[titan] 2025-07-10 05:49:42,325 - root - INFO - step: 73820 loss: 16.2845 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7954 +[titan] 2025-07-10 05:49:42,325 - root - INFO - lr: 7.3968e-05 gnorm: 0.89 [16:14:40< 5:45:39] +[titan] 2025-07-10 05:49:46,239 - root - INFO - step: 73825 loss: 16.1798 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.7033 +[titan] 2025-07-10 05:49:46,239 - root - INFO - lr: 7.3953e-05 gnorm: 0.96 [16:14:44< 5:45:35] +[titan] 2025-07-10 05:49:50,178 - root - INFO - step: 73830 loss: 16.4527 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9111 +[titan] 2025-07-10 05:49:50,178 - root - INFO - lr: 7.3937e-05 gnorm: 0.88 [16:14:48< 5:45:31] +[titan] 2025-07-10 05:49:54,111 - root - INFO - step: 73835 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 83,334 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8440 +[titan] 2025-07-10 05:49:54,111 - root - INFO - lr: 7.3921e-05 gnorm: 0.94 [16:14:51< 5:45:27] +[titan] 2025-07-10 05:49:58,014 - root - INFO - step: 73840 loss: 16.3548 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8523 +[titan] 2025-07-10 05:49:58,014 - root - INFO - lr: 7.3905e-05 gnorm: 0.94 [16:14:55< 5:45:23] +[titan] 2025-07-10 05:50:01,929 - root - INFO - step: 73845 loss: 16.0730 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4541 global_avg_mtp_loss: 13.6189 +[titan] 2025-07-10 05:50:01,929 - root - INFO - lr: 7.3889e-05 gnorm: 1.01 [16:14:59< 5:45:19] +[titan] 2025-07-10 05:50:05,064 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:50:05,868 - root - INFO - step: 73850 loss: 16.5280 memory: 44.58GiB(31.99%) tps: 83,187 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 13.9922 +[titan] 2025-07-10 05:50:05,868 - root - INFO - lr: 7.3873e-05 gnorm: 1.01 [16:15:03< 5:45:15] +[titan] 2025-07-10 05:50:09,790 - root - INFO - step: 73855 loss: 16.5407 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 13.9990 +[titan] 2025-07-10 05:50:09,791 - root - INFO - lr: 7.3858e-05 gnorm: 0.93 [16:15:07< 5:45:11] +[titan] 2025-07-10 05:50:13,717 - root - INFO - step: 73860 loss: 16.1009 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6390 +[titan] 2025-07-10 05:50:13,717 - root - INFO - lr: 7.3842e-05 gnorm: 0.92 [16:15:11< 5:45:07] +[titan] 2025-07-10 05:50:17,635 - root - INFO - step: 73865 loss: 16.0849 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 05:50:17,635 - root - INFO - lr: 7.3826e-05 gnorm: 0.94 [16:15:15< 5:45:04] +[titan] 2025-07-10 05:50:21,575 - root - INFO - step: 73870 loss: 16.1889 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7083 +[titan] 2025-07-10 05:50:21,575 - root - INFO - lr: 7.3810e-05 gnorm: 1.00 [16:15:19< 5:45:00] +[titan] 2025-07-10 05:50:25,504 - root - INFO - step: 73875 loss: 16.4839 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5299 global_avg_mtp_loss: 13.9540 +[titan] 2025-07-10 05:50:25,504 - root - INFO - lr: 7.3794e-05 gnorm: 0.93 [16:15:23< 5:44:56] +[titan] 2025-07-10 05:50:29,426 - root - INFO - step: 73880 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8079 +[titan] 2025-07-10 05:50:29,426 - root - INFO - lr: 7.3779e-05 gnorm: 0.94 [16:15:27< 5:44:52] +[titan] 2025-07-10 05:50:33,357 - root - INFO - step: 73885 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7150 +[titan] 2025-07-10 05:50:33,357 - root - INFO - lr: 7.3763e-05 gnorm: 0.94 [16:15:31< 5:44:48] +[titan] 2025-07-10 05:50:37,295 - root - INFO - step: 73890 loss: 16.2874 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7958 +[titan] 2025-07-10 05:50:37,296 - root - INFO - lr: 7.3747e-05 gnorm: 0.92 [16:15:35< 5:44:44] +[titan] 2025-07-10 05:50:41,206 - root - INFO - step: 73895 loss: 16.4817 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9561 +[titan] 2025-07-10 05:50:41,207 - root - INFO - lr: 7.3731e-05 gnorm: 0.95 [16:15:39< 5:44:40] +[titan] 2025-07-10 05:50:44,331 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:50:45,130 - root - INFO - step: 73900 loss: 16.2659 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.7679 +[titan] 2025-07-10 05:50:45,130 - root - INFO - lr: 7.3716e-05 gnorm: 0.93 [16:15:42< 5:44:36] +[titan] 2025-07-10 05:50:49,031 - root - INFO - step: 73905 loss: 16.1567 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6878 +[titan] 2025-07-10 05:50:49,032 - root - INFO - lr: 7.3700e-05 gnorm: 0.95 [16:15:46< 5:44:32] +[titan] 2025-07-10 05:50:52,946 - root - INFO - step: 73910 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 05:50:52,946 - root - INFO - lr: 7.3684e-05 gnorm: 0.94 [16:15:50< 5:44:28] +[titan] 2025-07-10 05:50:56,852 - root - INFO - step: 73915 loss: 16.3201 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8226 +[titan] 2025-07-10 05:50:56,852 - root - INFO - lr: 7.3668e-05 gnorm: 0.91 [16:15:54< 5:44:24] +[titan] 2025-07-10 05:51:00,774 - root - INFO - step: 73920 loss: 16.3730 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8746 +[titan] 2025-07-10 05:51:00,774 - root - INFO - lr: 7.3652e-05 gnorm: 0.96 [16:15:58< 5:44:20] +[titan] 2025-07-10 05:51:04,678 - root - INFO - step: 73925 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8434 +[titan] 2025-07-10 05:51:04,678 - root - INFO - lr: 7.3637e-05 gnorm: 0.93 [16:16:02< 5:44:16] +[titan] 2025-07-10 05:51:08,580 - root - INFO - step: 73930 loss: 16.2235 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.7489 +[titan] 2025-07-10 05:51:08,581 - root - INFO - lr: 7.3621e-05 gnorm: 0.93 [16:16:06< 5:44:12] +[titan] 2025-07-10 05:51:12,523 - root - INFO - step: 73935 loss: 16.1480 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6814 +[titan] 2025-07-10 05:51:12,523 - root - INFO - lr: 7.3605e-05 gnorm: 0.99 [16:16:10< 5:44:08] +[titan] 2025-07-10 05:51:16,441 - root - INFO - step: 73940 loss: 16.2231 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 05:51:16,441 - root - INFO - lr: 7.3589e-05 gnorm: 0.95 [16:16:14< 5:44:04] +[titan] 2025-07-10 05:51:20,435 - root - INFO - step: 73945 loss: 16.1886 memory: 44.58GiB(31.99%) tps: 82,038 tflops: 283.12 mfu: 28.63% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.7154 +[titan] 2025-07-10 05:51:20,435 - root - INFO - lr: 7.3574e-05 gnorm: 0.96 [16:16:18< 5:44:00] +[titan] 2025-07-10 05:51:23,564 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:51:24,357 - root - INFO - step: 73950 loss: 16.3452 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.8613 +[titan] 2025-07-10 05:51:24,357 - root - INFO - lr: 7.3558e-05 gnorm: 0.95 [16:16:22< 5:43:56] +[titan] 2025-07-10 05:51:28,262 - root - INFO - step: 73955 loss: 16.1671 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6996 +[titan] 2025-07-10 05:51:28,262 - root - INFO - lr: 7.3542e-05 gnorm: 0.97 [16:16:26< 5:43:52] +[titan] 2025-07-10 05:51:32,214 - root - INFO - step: 73960 loss: 16.6253 memory: 44.58GiB(31.99%) tps: 82,919 tflops: 286.17 mfu: 28.94% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 14.0798 +[titan] 2025-07-10 05:51:32,214 - root - INFO - lr: 7.3526e-05 gnorm: 0.90 [16:16:30< 5:43:48] +[titan] 2025-07-10 05:51:36,119 - root - INFO - step: 73965 loss: 16.5479 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5461 global_avg_mtp_loss: 14.0018 +[titan] 2025-07-10 05:51:36,119 - root - INFO - lr: 7.3511e-05 gnorm: 0.94 [16:16:33< 5:43:44] +[titan] 2025-07-10 05:51:40,065 - root - INFO - step: 73970 loss: 16.4109 memory: 44.58GiB(31.99%) tps: 83,042 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8933 +[titan] 2025-07-10 05:51:40,065 - root - INFO - lr: 7.3495e-05 gnorm: 0.95 [16:16:37< 5:43:40] +[titan] 2025-07-10 05:51:44,002 - root - INFO - step: 73975 loss: 16.2818 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7966 +[titan] 2025-07-10 05:51:44,002 - root - INFO - lr: 7.3479e-05 gnorm: 0.94 [16:16:41< 5:43:36] +[titan] 2025-07-10 05:51:47,944 - root - INFO - step: 73980 loss: 16.5523 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5426 global_avg_mtp_loss: 14.0097 +[titan] 2025-07-10 05:51:47,945 - root - INFO - lr: 7.3463e-05 gnorm: 0.95 [16:16:45< 5:43:32] +[titan] 2025-07-10 05:51:51,865 - root - INFO - step: 73985 loss: 16.2746 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7846 +[titan] 2025-07-10 05:51:51,865 - root - INFO - lr: 7.3448e-05 gnorm: 0.97 [16:16:49< 5:43:28] +[titan] 2025-07-10 05:51:55,781 - root - INFO - step: 73990 loss: 16.3369 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.8490 +[titan] 2025-07-10 05:51:55,782 - root - INFO - lr: 7.3432e-05 gnorm: 0.98 [16:16:53< 5:43:24] +[titan] 2025-07-10 05:51:59,727 - root - INFO - step: 73995 loss: 16.2689 memory: 44.58GiB(31.99%) tps: 83,058 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.7793 +[titan] 2025-07-10 05:51:59,727 - root - INFO - lr: 7.3416e-05 gnorm: 0.92 [16:16:57< 5:43:20] +[titan] 2025-07-10 05:52:02,843 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:52:03,630 - root - INFO - step: 74000 loss: 16.4845 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.9631 +[titan] 2025-07-10 05:52:03,630 - root - INFO - lr: 7.3400e-05 gnorm: 0.87 [16:17:01< 5:43:16] +[titan] 2025-07-10 05:52:07,556 - root - INFO - step: 74005 loss: 16.6319 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5507 global_avg_mtp_loss: 14.0811 +[titan] 2025-07-10 05:52:07,557 - root - INFO - lr: 7.3385e-05 gnorm: 0.95 [16:17:05< 5:43:12] +[titan] 2025-07-10 05:52:11,478 - root - INFO - step: 74010 loss: 16.2951 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.8064 +[titan] 2025-07-10 05:52:11,478 - root - INFO - lr: 7.3369e-05 gnorm: 0.96 [16:17:09< 5:43:08] +[titan] 2025-07-10 05:52:15,416 - root - INFO - step: 74015 loss: 16.1459 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 05:52:15,417 - root - INFO - lr: 7.3353e-05 gnorm: 0.90 [16:17:13< 5:43:04] +[titan] 2025-07-10 05:52:19,360 - root - INFO - step: 74020 loss: 16.3608 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8652 +[titan] 2025-07-10 05:52:19,361 - root - INFO - lr: 7.3337e-05 gnorm: 0.88 [16:17:17< 5:43:00] +[titan] 2025-07-10 05:52:23,305 - root - INFO - step: 74025 loss: 16.0385 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4446 global_avg_mtp_loss: 13.5940 +[titan] 2025-07-10 05:52:23,305 - root - INFO - lr: 7.3322e-05 gnorm: 0.97 [16:17:21< 5:42:56] +[titan] 2025-07-10 05:52:27,238 - root - INFO - step: 74030 loss: 16.0999 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4578 global_avg_mtp_loss: 13.6421 +[titan] 2025-07-10 05:52:27,238 - root - INFO - lr: 7.3306e-05 gnorm: 0.94 [16:17:25< 5:42:52] +[titan] 2025-07-10 05:52:31,177 - root - INFO - step: 74035 loss: 16.0741 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4502 global_avg_mtp_loss: 13.6238 +[titan] 2025-07-10 05:52:31,177 - root - INFO - lr: 7.3290e-05 gnorm: 0.97 [16:17:29< 5:42:48] +[titan] 2025-07-10 05:52:35,087 - root - INFO - step: 74040 loss: 16.3510 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8377 +[titan] 2025-07-10 05:52:35,087 - root - INFO - lr: 7.3275e-05 gnorm: 0.90 [16:17:32< 5:42:44] +[titan] 2025-07-10 05:52:39,036 - root - INFO - step: 74045 loss: 16.4867 memory: 44.58GiB(31.99%) tps: 82,994 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9594 +[titan] 2025-07-10 05:52:39,036 - root - INFO - lr: 7.3259e-05 gnorm: 0.88 [16:17:36< 5:42:40] +[titan] 2025-07-10 05:52:42,174 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:52:42,956 - root - INFO - step: 74050 loss: 16.2200 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.7529 +[titan] 2025-07-10 05:52:42,957 - root - INFO - lr: 7.3243e-05 gnorm: 0.92 [16:17:40< 5:42:37] +[titan] 2025-07-10 05:52:46,888 - root - INFO - step: 74055 loss: 16.4215 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8995 +[titan] 2025-07-10 05:52:46,889 - root - INFO - lr: 7.3227e-05 gnorm: 0.91 [16:17:44< 5:42:33] +[titan] 2025-07-10 05:52:50,828 - root - INFO - step: 74060 loss: 16.1386 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.6704 +[titan] 2025-07-10 05:52:50,828 - root - INFO - lr: 7.3212e-05 gnorm: 0.92 [16:17:48< 5:42:29] +[titan] 2025-07-10 05:52:54,723 - root - INFO - step: 74065 loss: 16.1770 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.7098 +[titan] 2025-07-10 05:52:54,724 - root - INFO - lr: 7.3196e-05 gnorm: 0.88 [16:17:52< 5:42:25] +[titan] 2025-07-10 05:52:58,651 - root - INFO - step: 74070 loss: 16.4658 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5359 global_avg_mtp_loss: 13.9299 +[titan] 2025-07-10 05:52:58,652 - root - INFO - lr: 7.3180e-05 gnorm: 0.92 [16:17:56< 5:42:21] +[titan] 2025-07-10 05:53:02,584 - root - INFO - step: 74075 loss: 16.6059 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5634 global_avg_mtp_loss: 14.0424 +[titan] 2025-07-10 05:53:02,584 - root - INFO - lr: 7.3165e-05 gnorm: 0.96 [16:18:00< 5:42:17] +[titan] 2025-07-10 05:53:06,490 - root - INFO - step: 74080 loss: 16.2298 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4730 global_avg_mtp_loss: 13.7568 +[titan] 2025-07-10 05:53:06,490 - root - INFO - lr: 7.3149e-05 gnorm: 0.96 [16:18:04< 5:42:13] +[titan] 2025-07-10 05:53:10,385 - root - INFO - step: 74085 loss: 16.5353 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5277 global_avg_mtp_loss: 14.0076 +[titan] 2025-07-10 05:53:10,386 - root - INFO - lr: 7.3133e-05 gnorm: 0.91 [16:18:08< 5:42:09] +[titan] 2025-07-10 05:53:14,278 - root - INFO - step: 74090 loss: 16.1841 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7035 +[titan] 2025-07-10 05:53:14,278 - root - INFO - lr: 7.3118e-05 gnorm: 0.99 [16:18:12< 5:42:05] +[titan] 2025-07-10 05:53:18,177 - root - INFO - step: 74095 loss: 16.4208 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.5406 global_avg_mtp_loss: 13.8802 +[titan] 2025-07-10 05:53:18,177 - root - INFO - lr: 7.3102e-05 gnorm: 0.95 [16:18:15< 5:42:01] +[titan] 2025-07-10 05:53:21,297 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:53:22,085 - root - INFO - step: 74100 loss: 16.3319 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.8431 +[titan] 2025-07-10 05:53:22,086 - root - INFO - lr: 7.3086e-05 gnorm: 0.92 [16:18:19< 5:41:57] +[titan] 2025-07-10 05:53:25,995 - root - INFO - step: 74105 loss: 16.1918 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.7074 +[titan] 2025-07-10 05:53:25,996 - root - INFO - lr: 7.3070e-05 gnorm: 0.91 [16:18:23< 5:41:53] +[titan] 2025-07-10 05:53:29,913 - root - INFO - step: 74110 loss: 16.1590 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.6890 +[titan] 2025-07-10 05:53:29,913 - root - INFO - lr: 7.3055e-05 gnorm: 0.96 [16:18:27< 5:41:49] +[titan] 2025-07-10 05:53:33,811 - root - INFO - step: 74115 loss: 16.2727 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7905 +[titan] 2025-07-10 05:53:33,812 - root - INFO - lr: 7.3039e-05 gnorm: 0.93 [16:18:31< 5:41:45] +[titan] 2025-07-10 05:53:37,751 - root - INFO - step: 74120 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.8535 +[titan] 2025-07-10 05:53:37,752 - root - INFO - lr: 7.3023e-05 gnorm: 0.96 [16:18:35< 5:41:41] +[titan] 2025-07-10 05:53:41,675 - root - INFO - step: 74125 loss: 16.4929 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.9654 +[titan] 2025-07-10 05:53:41,675 - root - INFO - lr: 7.3008e-05 gnorm: 0.95 [16:18:39< 5:41:37] +[titan] 2025-07-10 05:53:45,597 - root - INFO - step: 74130 loss: 16.2045 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7401 +[titan] 2025-07-10 05:53:45,597 - root - INFO - lr: 7.2992e-05 gnorm: 0.88 [16:18:43< 5:41:33] +[titan] 2025-07-10 05:53:49,510 - root - INFO - step: 74135 loss: 15.8400 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4104 global_avg_mtp_loss: 13.4297 +[titan] 2025-07-10 05:53:49,510 - root - INFO - lr: 7.2976e-05 gnorm: 1.00 [16:18:47< 5:41:29] +[titan] 2025-07-10 05:53:53,419 - root - INFO - step: 74140 loss: 16.1760 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.6987 +[titan] 2025-07-10 05:53:53,419 - root - INFO - lr: 7.2961e-05 gnorm: 1.03 [16:18:51< 5:41:25] +[titan] 2025-07-10 05:53:57,326 - root - INFO - step: 74145 loss: 16.5488 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5317 global_avg_mtp_loss: 14.0171 +[titan] 2025-07-10 05:53:57,327 - root - INFO - lr: 7.2945e-05 gnorm: 0.91 [16:18:55< 5:41:21] +[titan] 2025-07-10 05:54:00,448 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:54:01,233 - root - INFO - step: 74150 loss: 16.3081 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8034 +[titan] 2025-07-10 05:54:01,234 - root - INFO - lr: 7.2929e-05 gnorm: 0.93 [16:18:59< 5:41:17] +[titan] 2025-07-10 05:54:05,124 - root - INFO - step: 74155 loss: 16.1525 memory: 44.58GiB(31.99%) tps: 84,239 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.4714 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 05:54:05,124 - root - INFO - lr: 7.2914e-05 gnorm: 0.94 [16:19:02< 5:41:13] +[titan] 2025-07-10 05:54:09,020 - root - INFO - step: 74160 loss: 16.2103 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7368 +[titan] 2025-07-10 05:54:09,020 - root - INFO - lr: 7.2898e-05 gnorm: 0.98 [16:19:06< 5:41:09] +[titan] 2025-07-10 05:54:12,919 - root - INFO - step: 74165 loss: 16.0909 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6332 +[titan] 2025-07-10 05:54:12,920 - root - INFO - lr: 7.2882e-05 gnorm: 0.96 [16:19:10< 5:41:05] +[titan] 2025-07-10 05:54:16,809 - root - INFO - step: 74170 loss: 16.3262 memory: 44.58GiB(31.99%) tps: 84,263 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8338 +[titan] 2025-07-10 05:54:16,809 - root - INFO - lr: 7.2867e-05 gnorm: 0.97 [16:19:14< 5:41:01] +[titan] 2025-07-10 05:54:20,738 - root - INFO - step: 74175 loss: 16.2332 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7511 +[titan] 2025-07-10 05:54:20,738 - root - INFO - lr: 7.2851e-05 gnorm: 0.96 [16:19:18< 5:40:57] +[titan] 2025-07-10 05:54:24,647 - root - INFO - step: 74180 loss: 16.3358 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8412 +[titan] 2025-07-10 05:54:24,648 - root - INFO - lr: 7.2835e-05 gnorm: 0.93 [16:19:22< 5:40:53] +[titan] 2025-07-10 05:54:28,595 - root - INFO - step: 74185 loss: 16.4312 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.9272 +[titan] 2025-07-10 05:54:28,596 - root - INFO - lr: 7.2820e-05 gnorm: 0.94 [16:19:26< 5:40:49] +[titan] 2025-07-10 05:54:32,500 - root - INFO - step: 74190 loss: 16.3928 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 05:54:32,501 - root - INFO - lr: 7.2804e-05 gnorm: 0.96 [16:19:30< 5:40:45] +[titan] 2025-07-10 05:54:36,453 - root - INFO - step: 74195 loss: 16.2172 memory: 44.58GiB(31.99%) tps: 82,918 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7448 +[titan] 2025-07-10 05:54:36,453 - root - INFO - lr: 7.2789e-05 gnorm: 0.94 [16:19:34< 5:40:41] +[titan] 2025-07-10 05:54:39,581 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:54:40,375 - root - INFO - step: 74200 loss: 16.4454 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9228 +[titan] 2025-07-10 05:54:40,375 - root - INFO - lr: 7.2773e-05 gnorm: 0.96 [16:19:38< 5:40:37] +[titan] 2025-07-10 05:54:44,296 - root - INFO - step: 74205 loss: 16.0943 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6327 +[titan] 2025-07-10 05:54:44,296 - root - INFO - lr: 7.2757e-05 gnorm: 0.97 [16:19:42< 5:40:33] +[titan] 2025-07-10 05:54:48,209 - root - INFO - step: 74210 loss: 16.4440 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.9312 +[titan] 2025-07-10 05:54:48,209 - root - INFO - lr: 7.2742e-05 gnorm: 0.92 [16:19:46< 5:40:29] +[titan] 2025-07-10 05:54:52,130 - root - INFO - step: 74215 loss: 16.2491 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7657 +[titan] 2025-07-10 05:54:52,130 - root - INFO - lr: 7.2726e-05 gnorm: 0.93 [16:19:49< 5:40:25] +[titan] 2025-07-10 05:54:56,046 - root - INFO - step: 74220 loss: 16.3054 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8122 +[titan] 2025-07-10 05:54:56,047 - root - INFO - lr: 7.2710e-05 gnorm: 0.94 [16:19:53< 5:40:21] +[titan] 2025-07-10 05:54:59,957 - root - INFO - step: 74225 loss: 16.0482 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4406 global_avg_mtp_loss: 13.6076 +[titan] 2025-07-10 05:54:59,957 - root - INFO - lr: 7.2695e-05 gnorm: 0.94 [16:19:57< 5:40:17] +[titan] 2025-07-10 05:55:03,883 - root - INFO - step: 74230 loss: 16.2785 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.7841 +[titan] 2025-07-10 05:55:03,884 - root - INFO - lr: 7.2679e-05 gnorm: 0.95 [16:20:01< 5:40:13] +[titan] 2025-07-10 05:55:07,797 - root - INFO - step: 74235 loss: 16.3454 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8474 +[titan] 2025-07-10 05:55:07,797 - root - INFO - lr: 7.2663e-05 gnorm: 0.95 [16:20:05< 5:40:09] +[titan] 2025-07-10 05:55:11,808 - root - INFO - step: 74240 loss: 16.3948 memory: 44.58GiB(31.99%) tps: 81,702 tflops: 281.97 mfu: 28.51% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.8596 +[titan] 2025-07-10 05:55:11,808 - root - INFO - lr: 7.2648e-05 gnorm: 1.01 [16:20:09< 5:40:05] +[titan] 2025-07-10 05:55:11,963 - root - INFO - Dumping profiler traces at step 74240 +[titan] 2025-07-10 05:55:11,997 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 05:55:15,917 - root - INFO - step: 74245 loss: 16.6918 memory: 44.58GiB(31.99%) tps: 79,746 tflops: 275.22 mfu: 27.83% global_avg_ntp_loss: 2.5731 global_avg_mtp_loss: 14.1187 +[titan] 2025-07-10 05:55:15,918 - root - INFO - lr: 7.2632e-05 gnorm: 0.93 [16:20:13< 5:40:02] +[titan] 2025-07-10 05:55:19,025 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:55:19,819 - root - INFO - step: 74250 loss: 16.1581 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.6914 +[titan] 2025-07-10 05:55:19,819 - root - INFO - lr: 7.2617e-05 gnorm: 0.93 [16:20:17< 5:39:58] +[titan] 2025-07-10 05:55:23,728 - root - INFO - step: 74255 loss: 16.2593 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 05:55:23,728 - root - INFO - lr: 7.2601e-05 gnorm: 0.91 [16:20:21< 5:39:54] +[titan] 2025-07-10 05:55:27,626 - root - INFO - step: 74260 loss: 16.2800 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.7841 +[titan] 2025-07-10 05:55:27,627 - root - INFO - lr: 7.2585e-05 gnorm: 0.91 [16:20:25< 5:39:50] +[titan] 2025-07-10 05:55:31,526 - root - INFO - step: 74265 loss: 16.1969 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7202 +[titan] 2025-07-10 05:55:31,526 - root - INFO - lr: 7.2570e-05 gnorm: 0.90 [16:20:29< 5:39:46] +[titan] 2025-07-10 05:55:35,411 - root - INFO - step: 74270 loss: 16.2906 memory: 44.58GiB(31.99%) tps: 84,350 tflops: 291.11 mfu: 29.43% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.7990 +[titan] 2025-07-10 05:55:35,411 - root - INFO - lr: 7.2554e-05 gnorm: 0.92 [16:20:33< 5:39:42] +[titan] 2025-07-10 05:55:39,299 - root - INFO - step: 74275 loss: 16.2275 memory: 44.58GiB(31.99%) tps: 84,279 tflops: 290.86 mfu: 29.41% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7351 +[titan] 2025-07-10 05:55:39,300 - root - INFO - lr: 7.2538e-05 gnorm: 0.94 [16:20:37< 5:39:38] +[titan] 2025-07-10 05:55:43,211 - root - INFO - step: 74280 loss: 16.2950 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.8061 +[titan] 2025-07-10 05:55:43,211 - root - INFO - lr: 7.2523e-05 gnorm: 0.97 [16:20:41< 5:39:34] +[titan] 2025-07-10 05:55:47,108 - root - INFO - step: 74285 loss: 16.3935 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8933 +[titan] 2025-07-10 05:55:47,109 - root - INFO - lr: 7.2507e-05 gnorm: 0.96 [16:20:44< 5:39:30] +[titan] 2025-07-10 05:55:51,007 - root - INFO - step: 74290 loss: 15.6672 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.3854 global_avg_mtp_loss: 13.2818 +[titan] 2025-07-10 05:55:51,007 - root - INFO - lr: 7.2492e-05 gnorm: 1.13 [16:20:48< 5:39:26] +[titan] 2025-07-10 05:55:54,930 - root - INFO - step: 74295 loss: 16.2162 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7267 +[titan] 2025-07-10 05:55:54,931 - root - INFO - lr: 7.2476e-05 gnorm: 0.98 [16:20:52< 5:39:22] +[titan] 2025-07-10 05:55:58,050 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:55:58,840 - root - INFO - step: 74300 loss: 16.1310 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 05:55:58,840 - root - INFO - lr: 7.2460e-05 gnorm: 0.94 [16:20:56< 5:39:18] +[titan] 2025-07-10 05:56:02,776 - root - INFO - step: 74305 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8099 +[titan] 2025-07-10 05:56:02,776 - root - INFO - lr: 7.2445e-05 gnorm: 0.97 [16:21:00< 5:39:14] +[titan] 2025-07-10 05:56:06,684 - root - INFO - step: 74310 loss: 16.3564 memory: 44.58GiB(31.99%) tps: 83,871 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-10 05:56:06,684 - root - INFO - lr: 7.2429e-05 gnorm: 1.01 [16:21:04< 5:39:10] +[titan] 2025-07-10 05:56:10,586 - root - INFO - step: 74315 loss: 16.3270 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.8379 +[titan] 2025-07-10 05:56:10,586 - root - INFO - lr: 7.2414e-05 gnorm: 0.94 [16:21:08< 5:39:06] +[titan] 2025-07-10 05:56:14,478 - root - INFO - step: 74320 loss: 16.1522 memory: 44.58GiB(31.99%) tps: 84,200 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.6795 +[titan] 2025-07-10 05:56:14,478 - root - INFO - lr: 7.2398e-05 gnorm: 0.94 [16:21:12< 5:39:02] +[titan] 2025-07-10 05:56:18,374 - root - INFO - step: 74325 loss: 16.4148 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5227 global_avg_mtp_loss: 13.8921 +[titan] 2025-07-10 05:56:18,374 - root - INFO - lr: 7.2383e-05 gnorm: 0.95 [16:21:16< 5:38:58] +[titan] 2025-07-10 05:56:22,287 - root - INFO - step: 74330 loss: 16.2866 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 05:56:22,287 - root - INFO - lr: 7.2367e-05 gnorm: 0.92 [16:21:20< 5:38:54] +[titan] 2025-07-10 05:56:26,196 - root - INFO - step: 74335 loss: 16.4149 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8992 +[titan] 2025-07-10 05:56:26,196 - root - INFO - lr: 7.2351e-05 gnorm: 0.96 [16:21:23< 5:38:50] +[titan] 2025-07-10 05:56:30,164 - root - INFO - step: 74340 loss: 16.2870 memory: 44.58GiB(31.99%) tps: 82,602 tflops: 285.07 mfu: 28.82% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.7908 +[titan] 2025-07-10 05:56:30,164 - root - INFO - lr: 7.2336e-05 gnorm: 0.94 [16:21:27< 5:38:46] +[titan] 2025-07-10 05:56:34,072 - root - INFO - step: 74345 loss: 16.0735 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.6167 +[titan] 2025-07-10 05:56:34,072 - root - INFO - lr: 7.2320e-05 gnorm: 0.95 [16:21:31< 5:38:42] +[titan] 2025-07-10 05:56:37,193 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:56:38,002 - root - INFO - step: 74350 loss: 16.2289 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7483 +[titan] 2025-07-10 05:56:38,002 - root - INFO - lr: 7.2305e-05 gnorm: 0.96 [16:21:35< 5:38:38] +[titan] 2025-07-10 05:56:41,936 - root - INFO - step: 74355 loss: 16.1328 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.6539 +[titan] 2025-07-10 05:56:41,936 - root - INFO - lr: 7.2289e-05 gnorm: 0.95 [16:21:39< 5:38:34] +[titan] 2025-07-10 05:56:45,856 - root - INFO - step: 74360 loss: 16.1741 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7054 +[titan] 2025-07-10 05:56:45,856 - root - INFO - lr: 7.2274e-05 gnorm: 0.92 [16:21:43< 5:38:30] +[titan] 2025-07-10 05:56:49,756 - root - INFO - step: 74365 loss: 16.3515 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8440 +[titan] 2025-07-10 05:56:49,757 - root - INFO - lr: 7.2258e-05 gnorm: 0.96 [16:21:47< 5:38:26] +[titan] 2025-07-10 05:56:53,684 - root - INFO - step: 74370 loss: 16.3386 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8197 +[titan] 2025-07-10 05:56:53,685 - root - INFO - lr: 7.2242e-05 gnorm: 0.99 [16:21:51< 5:38:22] +[titan] 2025-07-10 05:56:57,592 - root - INFO - step: 74375 loss: 16.4119 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-10 05:56:57,592 - root - INFO - lr: 7.2227e-05 gnorm: 0.95 [16:21:55< 5:38:18] +[titan] 2025-07-10 05:57:01,511 - root - INFO - step: 74380 loss: 16.3013 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 05:57:01,511 - root - INFO - lr: 7.2211e-05 gnorm: 0.94 [16:21:59< 5:38:14] +[titan] 2025-07-10 05:57:05,421 - root - INFO - step: 74385 loss: 16.2395 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7612 +[titan] 2025-07-10 05:57:05,421 - root - INFO - lr: 7.2196e-05 gnorm: 0.94 [16:22:03< 5:38:10] +[titan] 2025-07-10 05:57:09,330 - root - INFO - step: 74390 loss: 16.1749 memory: 44.58GiB(31.99%) tps: 83,835 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.6993 +[titan] 2025-07-10 05:57:09,330 - root - INFO - lr: 7.2180e-05 gnorm: 1.01 [16:22:07< 5:38:06] +[titan] 2025-07-10 05:57:13,265 - root - INFO - step: 74395 loss: 16.0200 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4390 global_avg_mtp_loss: 13.5810 +[titan] 2025-07-10 05:57:13,265 - root - INFO - lr: 7.2165e-05 gnorm: 1.06 [16:22:11< 5:38:02] +[titan] 2025-07-10 05:57:16,419 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:57:17,212 - root - INFO - step: 74400 loss: 16.3320 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.8369 +[titan] 2025-07-10 05:57:17,212 - root - INFO - lr: 7.2149e-05 gnorm: 0.97 [16:22:15< 5:37:58] +[titan] 2025-07-10 05:57:21,140 - root - INFO - step: 74405 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7893 +[titan] 2025-07-10 05:57:21,141 - root - INFO - lr: 7.2133e-05 gnorm: 0.94 [16:22:18< 5:37:54] +[titan] 2025-07-10 05:57:25,058 - root - INFO - step: 74410 loss: 16.5026 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5412 global_avg_mtp_loss: 13.9614 +[titan] 2025-07-10 05:57:25,058 - root - INFO - lr: 7.2118e-05 gnorm: 0.96 [16:22:22< 5:37:50] +[titan] 2025-07-10 05:57:29,001 - root - INFO - step: 74415 loss: 15.6928 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.3841 global_avg_mtp_loss: 13.3086 +[titan] 2025-07-10 05:57:29,001 - root - INFO - lr: 7.2102e-05 gnorm: 0.96 [16:22:26< 5:37:46] +[titan] 2025-07-10 05:57:32,930 - root - INFO - step: 74420 loss: 16.1671 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.7087 +[titan] 2025-07-10 05:57:32,930 - root - INFO - lr: 7.2087e-05 gnorm: 0.91 [16:22:30< 5:37:42] +[titan] 2025-07-10 05:57:36,853 - root - INFO - step: 74425 loss: 16.2330 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7549 +[titan] 2025-07-10 05:57:36,853 - root - INFO - lr: 7.2071e-05 gnorm: 0.93 [16:22:34< 5:37:38] +[titan] 2025-07-10 05:57:40,811 - root - INFO - step: 74430 loss: 16.3439 memory: 44.58GiB(31.99%) tps: 82,803 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8459 +[titan] 2025-07-10 05:57:40,811 - root - INFO - lr: 7.2056e-05 gnorm: 0.91 [16:22:38< 5:37:34] +[titan] 2025-07-10 05:57:44,706 - root - INFO - step: 74435 loss: 16.1105 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.6519 +[titan] 2025-07-10 05:57:44,706 - root - INFO - lr: 7.2040e-05 gnorm: 1.06 [16:22:42< 5:37:30] +[titan] 2025-07-10 05:57:48,631 - root - INFO - step: 74440 loss: 16.0741 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4505 global_avg_mtp_loss: 13.6236 +[titan] 2025-07-10 05:57:48,631 - root - INFO - lr: 7.2025e-05 gnorm: 0.92 [16:22:46< 5:37:26] +[titan] 2025-07-10 05:57:52,544 - root - INFO - step: 74445 loss: 16.3208 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.7852 +[titan] 2025-07-10 05:57:52,545 - root - INFO - lr: 7.2009e-05 gnorm: 0.96 [16:22:50< 5:37:22] +[titan] 2025-07-10 05:57:55,670 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:57:56,479 - root - INFO - step: 74450 loss: 16.4116 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.8968 +[titan] 2025-07-10 05:57:56,479 - root - INFO - lr: 7.1994e-05 gnorm: 0.96 [16:22:54< 5:37:18] +[titan] 2025-07-10 05:58:00,391 - root - INFO - step: 74455 loss: 16.3762 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.8706 +[titan] 2025-07-10 05:58:00,391 - root - INFO - lr: 7.1978e-05 gnorm: 0.93 [16:22:58< 5:37:15] +[titan] 2025-07-10 05:58:04,327 - root - INFO - step: 74460 loss: 16.4813 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5390 global_avg_mtp_loss: 13.9422 +[titan] 2025-07-10 05:58:04,327 - root - INFO - lr: 7.1963e-05 gnorm: 0.95 [16:23:02< 5:37:11] +[titan] 2025-07-10 05:58:08,252 - root - INFO - step: 74465 loss: 15.9983 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4384 global_avg_mtp_loss: 13.5599 +[titan] 2025-07-10 05:58:08,253 - root - INFO - lr: 7.1947e-05 gnorm: 0.94 [16:23:06< 5:37:07] +[titan] 2025-07-10 05:58:12,152 - root - INFO - step: 74470 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9707 +[titan] 2025-07-10 05:58:12,152 - root - INFO - lr: 7.1932e-05 gnorm: 0.98 [16:23:09< 5:37:03] +[titan] 2025-07-10 05:58:16,048 - root - INFO - step: 74475 loss: 16.7475 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.5833 global_avg_mtp_loss: 14.1642 +[titan] 2025-07-10 05:58:16,048 - root - INFO - lr: 7.1916e-05 gnorm: 0.90 [16:23:13< 5:36:59] +[titan] 2025-07-10 05:58:19,970 - root - INFO - step: 74480 loss: 16.2297 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7446 +[titan] 2025-07-10 05:58:19,970 - root - INFO - lr: 7.1901e-05 gnorm: 0.93 [16:23:17< 5:36:55] +[titan] 2025-07-10 05:58:23,899 - root - INFO - step: 74485 loss: 16.2992 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8070 +[titan] 2025-07-10 05:58:23,899 - root - INFO - lr: 7.1885e-05 gnorm: 0.91 [16:23:21< 5:36:51] +[titan] 2025-07-10 05:58:27,831 - root - INFO - step: 74490 loss: 16.3011 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8023 +[titan] 2025-07-10 05:58:27,831 - root - INFO - lr: 7.1870e-05 gnorm: 0.95 [16:23:25< 5:36:47] +[titan] 2025-07-10 05:58:31,738 - root - INFO - step: 74495 loss: 16.0953 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.6393 +[titan] 2025-07-10 05:58:31,738 - root - INFO - lr: 7.1854e-05 gnorm: 0.95 [16:23:29< 5:36:43] +[titan] 2025-07-10 05:58:34,884 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:58:35,681 - root - INFO - step: 74500 loss: 16.3827 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8672 +[titan] 2025-07-10 05:58:35,681 - root - INFO - lr: 7.1839e-05 gnorm: 0.95 [16:23:33< 5:36:39] +[titan] 2025-07-10 05:58:39,646 - root - INFO - step: 74505 loss: 16.3776 memory: 44.58GiB(31.99%) tps: 82,657 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8590 +[titan] 2025-07-10 05:58:39,646 - root - INFO - lr: 7.1823e-05 gnorm: 1.00 [16:23:37< 5:36:35] +[titan] 2025-07-10 05:58:43,578 - root - INFO - step: 74510 loss: 16.2374 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.7461 +[titan] 2025-07-10 05:58:43,578 - root - INFO - lr: 7.1808e-05 gnorm: 0.96 [16:23:41< 5:36:31] +[titan] 2025-07-10 05:58:47,471 - root - INFO - step: 74515 loss: 16.3904 memory: 44.58GiB(31.99%) tps: 84,176 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8803 +[titan] 2025-07-10 05:58:47,471 - root - INFO - lr: 7.1792e-05 gnorm: 0.98 [16:23:45< 5:36:27] +[titan] 2025-07-10 05:58:51,372 - root - INFO - step: 74520 loss: 16.1929 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.7277 +[titan] 2025-07-10 05:58:51,373 - root - INFO - lr: 7.1777e-05 gnorm: 0.97 [16:23:49< 5:36:23] +[titan] 2025-07-10 05:58:55,284 - root - INFO - step: 74525 loss: 16.3114 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8200 +[titan] 2025-07-10 05:58:55,285 - root - INFO - lr: 7.1761e-05 gnorm: 0.99 [16:23:53< 5:36:19] +[titan] 2025-07-10 05:58:59,206 - root - INFO - step: 74530 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.7733 +[titan] 2025-07-10 05:58:59,206 - root - INFO - lr: 7.1746e-05 gnorm: 1.26 [16:23:56< 5:36:15] +[titan] 2025-07-10 05:59:03,127 - root - INFO - step: 74535 loss: 16.1634 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.6960 +[titan] 2025-07-10 05:59:03,127 - root - INFO - lr: 7.1730e-05 gnorm: 0.93 [16:24:00< 5:36:11] +[titan] 2025-07-10 05:59:07,045 - root - INFO - step: 74540 loss: 16.2769 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7866 +[titan] 2025-07-10 05:59:07,045 - root - INFO - lr: 7.1715e-05 gnorm: 0.95 [16:24:04< 5:36:07] +[titan] 2025-07-10 05:59:10,988 - root - INFO - step: 74545 loss: 16.2426 memory: 44.58GiB(31.99%) tps: 83,110 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7590 +[titan] 2025-07-10 05:59:10,988 - root - INFO - lr: 7.1699e-05 gnorm: 0.89 [16:24:08< 5:36:03] +[titan] 2025-07-10 05:59:14,143 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:59:14,942 - root - INFO - step: 74550 loss: 16.3565 memory: 44.58GiB(31.99%) tps: 82,868 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8576 +[titan] 2025-07-10 05:59:14,943 - root - INFO - lr: 7.1684e-05 gnorm: 0.91 [16:24:12< 5:35:59] +[titan] 2025-07-10 05:59:18,848 - root - INFO - step: 74555 loss: 16.1172 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.6662 +[titan] 2025-07-10 05:59:18,848 - root - INFO - lr: 7.1668e-05 gnorm: 0.96 [16:24:16< 5:35:55] +[titan] 2025-07-10 05:59:22,807 - root - INFO - step: 74560 loss: 16.3750 memory: 44.58GiB(31.99%) tps: 82,770 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8644 +[titan] 2025-07-10 05:59:22,807 - root - INFO - lr: 7.1653e-05 gnorm: 0.99 [16:24:20< 5:35:51] +[titan] 2025-07-10 05:59:26,740 - root - INFO - step: 74565 loss: 16.4167 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.9086 +[titan] 2025-07-10 05:59:26,740 - root - INFO - lr: 7.1637e-05 gnorm: 0.93 [16:24:24< 5:35:47] +[titan] 2025-07-10 05:59:30,697 - root - INFO - step: 74570 loss: 16.3307 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8349 +[titan] 2025-07-10 05:59:30,697 - root - INFO - lr: 7.1622e-05 gnorm: 0.93 [16:24:28< 5:35:43] +[titan] 2025-07-10 05:59:34,619 - root - INFO - step: 74575 loss: 16.2584 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7779 +[titan] 2025-07-10 05:59:34,620 - root - INFO - lr: 7.1606e-05 gnorm: 0.91 [16:24:32< 5:35:39] +[titan] 2025-07-10 05:59:38,572 - root - INFO - step: 74580 loss: 16.4368 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.9161 +[titan] 2025-07-10 05:59:38,572 - root - INFO - lr: 7.1591e-05 gnorm: 0.93 [16:24:36< 5:35:35] +[titan] 2025-07-10 05:59:42,485 - root - INFO - step: 74585 loss: 16.1006 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6360 +[titan] 2025-07-10 05:59:42,485 - root - INFO - lr: 7.1575e-05 gnorm: 0.96 [16:24:40< 5:35:31] +[titan] 2025-07-10 05:59:46,395 - root - INFO - step: 74590 loss: 16.4224 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.8959 +[titan] 2025-07-10 05:59:46,396 - root - INFO - lr: 7.1560e-05 gnorm: 0.95 [16:24:44< 5:35:27] +[titan] 2025-07-10 05:59:50,350 - root - INFO - step: 74595 loss: 16.2224 memory: 44.58GiB(31.99%) tps: 82,865 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7525 +[titan] 2025-07-10 05:59:50,350 - root - INFO - lr: 7.1544e-05 gnorm: 0.95 [16:24:48< 5:35:23] +[titan] 2025-07-10 05:59:53,505 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 05:59:54,302 - root - INFO - step: 74600 loss: 16.1700 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.6975 +[titan] 2025-07-10 05:59:54,303 - root - INFO - lr: 7.1529e-05 gnorm: 0.96 [16:24:52< 5:35:19] +[titan] 2025-07-10 05:59:58,257 - root - INFO - step: 74605 loss: 16.2672 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7833 +[titan] 2025-07-10 05:59:58,257 - root - INFO - lr: 7.1514e-05 gnorm: 1.06 [16:24:56< 5:35:15] +[titan] 2025-07-10 06:00:02,184 - root - INFO - step: 74610 loss: 16.4336 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.9305 +[titan] 2025-07-10 06:00:02,184 - root - INFO - lr: 7.1498e-05 gnorm: 0.90 [16:24:59< 5:35:11] +[titan] 2025-07-10 06:00:06,107 - root - INFO - step: 74615 loss: 16.3875 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8776 +[titan] 2025-07-10 06:00:06,107 - root - INFO - lr: 7.1483e-05 gnorm: 0.94 [16:25:03< 5:35:07] +[titan] 2025-07-10 06:00:10,006 - root - INFO - step: 74620 loss: 16.2777 memory: 44.58GiB(31.99%) tps: 84,045 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7946 +[titan] 2025-07-10 06:00:10,006 - root - INFO - lr: 7.1467e-05 gnorm: 0.95 [16:25:07< 5:35:03] +[titan] 2025-07-10 06:00:13,916 - root - INFO - step: 74625 loss: 16.1316 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.6594 +[titan] 2025-07-10 06:00:13,916 - root - INFO - lr: 7.1452e-05 gnorm: 1.05 [16:25:11< 5:34:59] +[titan] 2025-07-10 06:00:17,893 - root - INFO - step: 74630 loss: 16.3104 memory: 44.58GiB(31.99%) tps: 82,391 tflops: 284.35 mfu: 28.75% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.7956 +[titan] 2025-07-10 06:00:17,893 - root - INFO - lr: 7.1436e-05 gnorm: 0.95 [16:25:15< 5:34:55] +[titan] 2025-07-10 06:00:21,798 - root - INFO - step: 74635 loss: 16.0819 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.6318 +[titan] 2025-07-10 06:00:21,798 - root - INFO - lr: 7.1421e-05 gnorm: 0.90 [16:25:19< 5:34:52] +[titan] 2025-07-10 06:00:25,756 - root - INFO - step: 74640 loss: 16.5148 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5413 global_avg_mtp_loss: 13.9734 +[titan] 2025-07-10 06:00:25,756 - root - INFO - lr: 7.1405e-05 gnorm: 0.91 [16:25:23< 5:34:48] +[titan] 2025-07-10 06:00:29,686 - root - INFO - step: 74645 loss: 16.4214 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.9076 +[titan] 2025-07-10 06:00:29,686 - root - INFO - lr: 7.1390e-05 gnorm: 0.93 [16:25:27< 5:34:44] +[titan] 2025-07-10 06:00:32,800 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:00:33,589 - root - INFO - step: 74650 loss: 16.2979 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.8063 +[titan] 2025-07-10 06:00:33,589 - root - INFO - lr: 7.1375e-05 gnorm: 0.93 [16:25:31< 5:34:40] +[titan] 2025-07-10 06:00:37,510 - root - INFO - step: 74655 loss: 15.8955 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4214 global_avg_mtp_loss: 13.4741 +[titan] 2025-07-10 06:00:37,511 - root - INFO - lr: 7.1359e-05 gnorm: 0.95 [16:25:35< 5:34:36] +[titan] 2025-07-10 06:00:41,425 - root - INFO - step: 74660 loss: 15.9486 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4304 global_avg_mtp_loss: 13.5182 +[titan] 2025-07-10 06:00:41,425 - root - INFO - lr: 7.1344e-05 gnorm: 0.92 [16:25:39< 5:34:32] +[titan] 2025-07-10 06:00:45,312 - root - INFO - step: 74665 loss: 16.2674 memory: 44.58GiB(31.99%) tps: 84,308 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7891 +[titan] 2025-07-10 06:00:45,312 - root - INFO - lr: 7.1328e-05 gnorm: 0.93 [16:25:43< 5:34:28] +[titan] 2025-07-10 06:00:49,213 - root - INFO - step: 74670 loss: 16.5325 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5388 global_avg_mtp_loss: 13.9937 +[titan] 2025-07-10 06:00:49,213 - root - INFO - lr: 7.1313e-05 gnorm: 0.92 [16:25:46< 5:34:24] +[titan] 2025-07-10 06:00:53,169 - root - INFO - step: 74675 loss: 16.4916 memory: 44.58GiB(31.99%) tps: 82,829 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 06:00:53,169 - root - INFO - lr: 7.1297e-05 gnorm: 0.95 [16:25:50< 5:34:20] +[titan] 2025-07-10 06:00:57,085 - root - INFO - step: 74680 loss: 16.0834 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6237 +[titan] 2025-07-10 06:00:57,085 - root - INFO - lr: 7.1282e-05 gnorm: 0.97 [16:25:54< 5:34:16] +[titan] 2025-07-10 06:01:00,990 - root - INFO - step: 74685 loss: 16.5041 memory: 44.58GiB(31.99%) tps: 83,914 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5389 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-10 06:01:00,990 - root - INFO - lr: 7.1267e-05 gnorm: 0.92 [16:25:58< 5:34:12] +[titan] 2025-07-10 06:01:04,909 - root - INFO - step: 74690 loss: 16.2716 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 06:01:04,910 - root - INFO - lr: 7.1251e-05 gnorm: 0.97 [16:26:02< 5:34:08] +[titan] 2025-07-10 06:01:08,823 - root - INFO - step: 74695 loss: 16.1678 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.6970 +[titan] 2025-07-10 06:01:08,823 - root - INFO - lr: 7.1236e-05 gnorm: 0.91 [16:26:06< 5:34:04] +[titan] 2025-07-10 06:01:11,942 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:01:12,726 - root - INFO - step: 74700 loss: 16.0934 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 06:01:12,727 - root - INFO - lr: 7.1220e-05 gnorm: 0.98 [16:26:10< 5:34:00] +[titan] 2025-07-10 06:01:16,640 - root - INFO - step: 74705 loss: 16.5201 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.5405 global_avg_mtp_loss: 13.9796 +[titan] 2025-07-10 06:01:16,640 - root - INFO - lr: 7.1205e-05 gnorm: 1.09 [16:26:14< 5:33:56] +[titan] 2025-07-10 06:01:20,544 - root - INFO - step: 74710 loss: 16.4360 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9145 +[titan] 2025-07-10 06:01:20,544 - root - INFO - lr: 7.1190e-05 gnorm: 0.94 [16:26:18< 5:33:52] +[titan] 2025-07-10 06:01:24,477 - root - INFO - step: 74715 loss: 16.3875 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8773 +[titan] 2025-07-10 06:01:24,477 - root - INFO - lr: 7.1174e-05 gnorm: 0.97 [16:26:22< 5:33:48] +[titan] 2025-07-10 06:01:28,395 - root - INFO - step: 74720 loss: 16.1562 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4714 global_avg_mtp_loss: 13.6848 +[titan] 2025-07-10 06:01:28,395 - root - INFO - lr: 7.1159e-05 gnorm: 0.92 [16:26:26< 5:33:44] +[titan] 2025-07-10 06:01:32,326 - root - INFO - step: 74725 loss: 16.2787 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 06:01:32,326 - root - INFO - lr: 7.1143e-05 gnorm: 0.91 [16:26:30< 5:33:40] +[titan] 2025-07-10 06:01:36,239 - root - INFO - step: 74730 loss: 16.2391 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7630 +[titan] 2025-07-10 06:01:36,239 - root - INFO - lr: 7.1128e-05 gnorm: 0.90 [16:26:33< 5:33:36] +[titan] 2025-07-10 06:01:40,151 - root - INFO - step: 74735 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6904 +[titan] 2025-07-10 06:01:40,151 - root - INFO - lr: 7.1113e-05 gnorm: 0.89 [16:26:37< 5:33:32] +[titan] 2025-07-10 06:01:44,074 - root - INFO - step: 74740 loss: 16.4775 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5373 global_avg_mtp_loss: 13.9402 +[titan] 2025-07-10 06:01:44,075 - root - INFO - lr: 7.1097e-05 gnorm: 0.96 [16:26:41< 5:33:28] +[titan] 2025-07-10 06:01:47,999 - root - INFO - step: 74745 loss: 16.4681 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9471 +[titan] 2025-07-10 06:01:47,999 - root - INFO - lr: 7.1082e-05 gnorm: 0.92 [16:26:45< 5:33:24] +[titan] 2025-07-10 06:01:51,214 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:01:52,001 - root - INFO - step: 74750 loss: 16.3146 memory: 44.58GiB(31.99%) tps: 81,883 tflops: 282.59 mfu: 28.57% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.8222 +[titan] 2025-07-10 06:01:52,001 - root - INFO - lr: 7.1066e-05 gnorm: 1.00 [16:26:49< 5:33:20] +[titan] 2025-07-10 06:01:53,737 - root - INFO - Dumping profiler traces at step 74752 +[titan] 2025-07-10 06:01:53,770 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:01:56,124 - root - INFO - step: 74755 loss: 16.4535 memory: 44.58GiB(31.99%) tps: 79,488 tflops: 274.33 mfu: 27.74% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 06:01:56,124 - root - INFO - lr: 7.1051e-05 gnorm: 0.99 [16:26:53< 5:33:16] +[titan] 2025-07-10 06:02:00,038 - root - INFO - step: 74760 loss: 16.0024 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.5556 +[titan] 2025-07-10 06:02:00,039 - root - INFO - lr: 7.1036e-05 gnorm: 1.00 [16:26:57< 5:33:12] +[titan] 2025-07-10 06:02:03,948 - root - INFO - step: 74765 loss: 16.2244 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7347 +[titan] 2025-07-10 06:02:03,948 - root - INFO - lr: 7.1020e-05 gnorm: 0.92 [16:27:01< 5:33:08] +[titan] 2025-07-10 06:02:07,864 - root - INFO - step: 74770 loss: 16.5212 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5517 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 06:02:07,864 - root - INFO - lr: 7.1005e-05 gnorm: 0.98 [16:27:05< 5:33:04] +[titan] 2025-07-10 06:02:11,794 - root - INFO - step: 74775 loss: 16.2926 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.7880 +[titan] 2025-07-10 06:02:11,794 - root - INFO - lr: 7.0989e-05 gnorm: 0.97 [16:27:09< 5:33:00] +[titan] 2025-07-10 06:02:15,726 - root - INFO - step: 74780 loss: 16.2759 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7920 +[titan] 2025-07-10 06:02:15,726 - root - INFO - lr: 7.0974e-05 gnorm: 0.93 [16:27:13< 5:32:56] +[titan] 2025-07-10 06:02:19,627 - root - INFO - step: 74785 loss: 16.2483 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.7711 +[titan] 2025-07-10 06:02:19,628 - root - INFO - lr: 7.0959e-05 gnorm: 0.96 [16:27:17< 5:32:52] +[titan] 2025-07-10 06:02:23,557 - root - INFO - step: 74790 loss: 15.9154 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.4774 +[titan] 2025-07-10 06:02:23,557 - root - INFO - lr: 7.0943e-05 gnorm: 0.92 [16:27:21< 5:32:48] +[titan] 2025-07-10 06:02:27,492 - root - INFO - step: 74795 loss: 16.3959 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.8832 +[titan] 2025-07-10 06:02:27,492 - root - INFO - lr: 7.0928e-05 gnorm: 0.94 [16:27:25< 5:32:44] +[titan] 2025-07-10 06:02:30,615 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:02:31,406 - root - INFO - step: 74800 loss: 16.2310 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7469 +[titan] 2025-07-10 06:02:31,406 - root - INFO - lr: 7.0913e-05 gnorm: 0.95 [16:27:29< 5:32:40] +[titan] 2025-07-10 06:02:35,316 - root - INFO - step: 74805 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8216 +[titan] 2025-07-10 06:02:35,317 - root - INFO - lr: 7.0897e-05 gnorm: 0.88 [16:27:33< 5:32:36] +[titan] 2025-07-10 06:02:39,248 - root - INFO - step: 74810 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7103 +[titan] 2025-07-10 06:02:39,248 - root - INFO - lr: 7.0882e-05 gnorm: 0.97 [16:27:37< 5:32:32] +[titan] 2025-07-10 06:02:43,148 - root - INFO - step: 74815 loss: 16.3245 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.8236 +[titan] 2025-07-10 06:02:43,149 - root - INFO - lr: 7.0867e-05 gnorm: 0.94 [16:27:40< 5:32:29] +[titan] 2025-07-10 06:02:47,036 - root - INFO - step: 74820 loss: 16.3083 memory: 44.58GiB(31.99%) tps: 84,306 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8122 +[titan] 2025-07-10 06:02:47,036 - root - INFO - lr: 7.0851e-05 gnorm: 0.96 [16:27:44< 5:32:25] +[titan] 2025-07-10 06:02:50,950 - root - INFO - step: 74825 loss: 16.3230 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.8324 +[titan] 2025-07-10 06:02:50,951 - root - INFO - lr: 7.0836e-05 gnorm: 0.92 [16:27:48< 5:32:21] +[titan] 2025-07-10 06:02:54,867 - root - INFO - step: 74830 loss: 16.1690 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4920 global_avg_mtp_loss: 13.6769 +[titan] 2025-07-10 06:02:54,867 - root - INFO - lr: 7.0821e-05 gnorm: 0.93 [16:27:52< 5:32:17] +[titan] 2025-07-10 06:02:58,779 - root - INFO - step: 74835 loss: 16.3057 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8105 +[titan] 2025-07-10 06:02:58,780 - root - INFO - lr: 7.0805e-05 gnorm: 0.99 [16:27:56< 5:32:13] +[titan] 2025-07-10 06:03:02,698 - root - INFO - step: 74840 loss: 16.5551 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5549 global_avg_mtp_loss: 14.0002 +[titan] 2025-07-10 06:03:02,699 - root - INFO - lr: 7.0790e-05 gnorm: 0.93 [16:28:00< 5:32:09] +[titan] 2025-07-10 06:03:06,592 - root - INFO - step: 74845 loss: 16.1752 memory: 44.58GiB(31.99%) tps: 84,158 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7010 +[titan] 2025-07-10 06:03:06,593 - root - INFO - lr: 7.0774e-05 gnorm: 0.90 [16:28:04< 5:32:05] +[titan] 2025-07-10 06:03:09,700 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:03:10,502 - root - INFO - step: 74850 loss: 16.1492 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6872 +[titan] 2025-07-10 06:03:10,502 - root - INFO - lr: 7.0759e-05 gnorm: 0.98 [16:28:08< 5:32:01] +[titan] 2025-07-10 06:03:14,412 - root - INFO - step: 74855 loss: 16.3007 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 06:03:14,413 - root - INFO - lr: 7.0744e-05 gnorm: 0.94 [16:28:12< 5:31:57] +[titan] 2025-07-10 06:03:18,347 - root - INFO - step: 74860 loss: 16.3097 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.8238 +[titan] 2025-07-10 06:03:18,347 - root - INFO - lr: 7.0728e-05 gnorm: 0.91 [16:28:16< 5:31:53] +[titan] 2025-07-10 06:03:22,268 - root - INFO - step: 74865 loss: 16.2051 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7268 +[titan] 2025-07-10 06:03:22,269 - root - INFO - lr: 7.0713e-05 gnorm: 0.92 [16:28:20< 5:31:49] +[titan] 2025-07-10 06:03:26,174 - root - INFO - step: 74870 loss: 16.1818 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.7102 +[titan] 2025-07-10 06:03:26,175 - root - INFO - lr: 7.0698e-05 gnorm: 0.93 [16:28:23< 5:31:45] +[titan] 2025-07-10 06:03:30,078 - root - INFO - step: 74875 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8171 +[titan] 2025-07-10 06:03:30,079 - root - INFO - lr: 7.0683e-05 gnorm: 0.93 [16:28:27< 5:31:41] +[titan] 2025-07-10 06:03:34,009 - root - INFO - step: 74880 loss: 16.4053 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8966 +[titan] 2025-07-10 06:03:34,009 - root - INFO - lr: 7.0667e-05 gnorm: 1.00 [16:28:31< 5:31:37] +[titan] 2025-07-10 06:03:37,944 - root - INFO - step: 74885 loss: 16.3962 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-10 06:03:37,944 - root - INFO - lr: 7.0652e-05 gnorm: 0.94 [16:28:35< 5:31:33] +[titan] 2025-07-10 06:03:41,843 - root - INFO - step: 74890 loss: 16.3974 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8805 +[titan] 2025-07-10 06:03:41,843 - root - INFO - lr: 7.0637e-05 gnorm: 0.89 [16:28:39< 5:31:29] +[titan] 2025-07-10 06:03:45,807 - root - INFO - step: 74895 loss: 16.3488 memory: 44.58GiB(31.99%) tps: 82,671 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8406 +[titan] 2025-07-10 06:03:45,807 - root - INFO - lr: 7.0621e-05 gnorm: 0.93 [16:28:43< 5:31:25] +[titan] 2025-07-10 06:03:48,951 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:03:49,735 - root - INFO - step: 74900 loss: 16.2996 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8051 +[titan] 2025-07-10 06:03:49,736 - root - INFO - lr: 7.0606e-05 gnorm: 0.93 [16:28:47< 5:31:21] +[titan] 2025-07-10 06:03:53,645 - root - INFO - step: 74905 loss: 16.4219 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.9028 +[titan] 2025-07-10 06:03:53,646 - root - INFO - lr: 7.0591e-05 gnorm: 0.98 [16:28:51< 5:31:17] +[titan] 2025-07-10 06:03:57,567 - root - INFO - step: 74910 loss: 16.2680 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7830 +[titan] 2025-07-10 06:03:57,568 - root - INFO - lr: 7.0575e-05 gnorm: 0.92 [16:28:55< 5:31:13] +[titan] 2025-07-10 06:04:01,493 - root - INFO - step: 74915 loss: 16.2343 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7601 +[titan] 2025-07-10 06:04:01,493 - root - INFO - lr: 7.0560e-05 gnorm: 0.98 [16:28:59< 5:31:09] +[titan] 2025-07-10 06:04:05,481 - root - INFO - step: 74920 loss: 16.0564 memory: 44.58GiB(31.99%) tps: 82,173 tflops: 283.59 mfu: 28.67% global_avg_ntp_loss: 2.4554 global_avg_mtp_loss: 13.6009 +[titan] 2025-07-10 06:04:05,481 - root - INFO - lr: 7.0545e-05 gnorm: 0.94 [16:29:03< 5:31:05] +[titan] 2025-07-10 06:04:09,406 - root - INFO - step: 74925 loss: 16.2975 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.8109 +[titan] 2025-07-10 06:04:09,406 - root - INFO - lr: 7.0529e-05 gnorm: 0.92 [16:29:07< 5:31:01] +[titan] 2025-07-10 06:04:13,320 - root - INFO - step: 74930 loss: 16.2597 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7731 +[titan] 2025-07-10 06:04:13,321 - root - INFO - lr: 7.0514e-05 gnorm: 0.90 [16:29:11< 5:30:57] +[titan] 2025-07-10 06:04:17,235 - root - INFO - step: 74935 loss: 16.2688 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7870 +[titan] 2025-07-10 06:04:17,236 - root - INFO - lr: 7.0499e-05 gnorm: 0.95 [16:29:14< 5:30:53] +[titan] 2025-07-10 06:04:21,153 - root - INFO - step: 74940 loss: 16.0876 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6349 +[titan] 2025-07-10 06:04:21,153 - root - INFO - lr: 7.0483e-05 gnorm: 0.94 [16:29:18< 5:30:49] +[titan] 2025-07-10 06:04:25,072 - root - INFO - step: 74945 loss: 16.2459 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.7613 +[titan] 2025-07-10 06:04:25,072 - root - INFO - lr: 7.0468e-05 gnorm: 0.94 [16:29:22< 5:30:45] +[titan] 2025-07-10 06:04:28,199 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:04:28,992 - root - INFO - step: 74950 loss: 16.1918 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7210 +[titan] 2025-07-10 06:04:28,992 - root - INFO - lr: 7.0453e-05 gnorm: 0.98 [16:29:26< 5:30:41] +[titan] 2025-07-10 06:04:32,928 - root - INFO - step: 74955 loss: 16.3543 memory: 44.58GiB(31.99%) tps: 83,256 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 06:04:32,928 - root - INFO - lr: 7.0438e-05 gnorm: 0.91 [16:29:30< 5:30:37] +[titan] 2025-07-10 06:04:36,846 - root - INFO - step: 74960 loss: 16.3397 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-10 06:04:36,846 - root - INFO - lr: 7.0422e-05 gnorm: 0.93 [16:29:34< 5:30:33] +[titan] 2025-07-10 06:04:40,743 - root - INFO - step: 74965 loss: 16.4960 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5266 global_avg_mtp_loss: 13.9694 +[titan] 2025-07-10 06:04:40,743 - root - INFO - lr: 7.0407e-05 gnorm: 1.02 [16:29:38< 5:30:29] +[titan] 2025-07-10 06:04:44,658 - root - INFO - step: 74970 loss: 16.3397 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8423 +[titan] 2025-07-10 06:04:44,658 - root - INFO - lr: 7.0392e-05 gnorm: 1.00 [16:29:42< 5:30:25] +[titan] 2025-07-10 06:04:48,563 - root - INFO - step: 74975 loss: 16.1177 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6608 +[titan] 2025-07-10 06:04:48,563 - root - INFO - lr: 7.0376e-05 gnorm: 0.95 [16:29:46< 5:30:21] +[titan] 2025-07-10 06:04:52,481 - root - INFO - step: 74980 loss: 16.2184 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7465 +[titan] 2025-07-10 06:04:52,481 - root - INFO - lr: 7.0361e-05 gnorm: 0.95 [16:29:50< 5:30:17] +[titan] 2025-07-10 06:04:56,393 - root - INFO - step: 74985 loss: 16.3014 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.8145 +[titan] 2025-07-10 06:04:56,393 - root - INFO - lr: 7.0346e-05 gnorm: 0.96 [16:29:54< 5:30:13] +[titan] 2025-07-10 06:05:00,339 - root - INFO - step: 74990 loss: 16.2249 memory: 44.58GiB(31.99%) tps: 83,056 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7431 +[titan] 2025-07-10 06:05:00,339 - root - INFO - lr: 7.0331e-05 gnorm: 0.95 [16:29:58< 5:30:09] +[titan] 2025-07-10 06:05:04,262 - root - INFO - step: 74995 loss: 16.2338 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7451 +[titan] 2025-07-10 06:05:04,263 - root - INFO - lr: 7.0315e-05 gnorm: 0.92 [16:30:01< 5:30:05] +[titan] 2025-07-10 06:05:07,409 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:05:08,207 - root - INFO - step: 75000 loss: 16.3130 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4893 global_avg_mtp_loss: 13.8237 +[titan] 2025-07-10 06:05:08,208 - root - INFO - lr: 7.0300e-05 gnorm: 0.95 [16:30:05< 5:30:01] +[titan] 2025-07-10 06:05:12,143 - root - INFO - step: 75005 loss: 16.3605 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8617 +[titan] 2025-07-10 06:05:12,144 - root - INFO - lr: 7.0285e-05 gnorm: 0.95 [16:30:09< 5:29:58] +[titan] 2025-07-10 06:05:16,117 - root - INFO - step: 75010 loss: 16.1943 memory: 44.58GiB(31.99%) tps: 82,474 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7260 +[titan] 2025-07-10 06:05:16,117 - root - INFO - lr: 7.0270e-05 gnorm: 0.98 [16:30:13< 5:29:54] +[titan] 2025-07-10 06:05:20,027 - root - INFO - step: 75015 loss: 15.8382 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4161 global_avg_mtp_loss: 13.4222 +[titan] 2025-07-10 06:05:20,027 - root - INFO - lr: 7.0254e-05 gnorm: 1.06 [16:30:17< 5:29:50] +[titan] 2025-07-10 06:05:23,938 - root - INFO - step: 75020 loss: 16.2208 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7331 +[titan] 2025-07-10 06:05:23,938 - root - INFO - lr: 7.0239e-05 gnorm: 0.96 [16:30:21< 5:29:46] +[titan] 2025-07-10 06:05:27,870 - root - INFO - step: 75025 loss: 16.2681 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.7729 +[titan] 2025-07-10 06:05:27,871 - root - INFO - lr: 7.0224e-05 gnorm: 0.98 [16:30:25< 5:29:42] +[titan] 2025-07-10 06:05:31,770 - root - INFO - step: 75030 loss: 16.2307 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7417 +[titan] 2025-07-10 06:05:31,770 - root - INFO - lr: 7.0209e-05 gnorm: 0.96 [16:30:29< 5:29:38] +[titan] 2025-07-10 06:05:35,689 - root - INFO - step: 75035 loss: 16.3684 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8694 +[titan] 2025-07-10 06:05:35,689 - root - INFO - lr: 7.0193e-05 gnorm: 0.92 [16:30:33< 5:29:34] +[titan] 2025-07-10 06:05:39,638 - root - INFO - step: 75040 loss: 16.3190 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.8287 +[titan] 2025-07-10 06:05:39,638 - root - INFO - lr: 7.0178e-05 gnorm: 0.97 [16:30:37< 5:29:30] +[titan] 2025-07-10 06:05:43,560 - root - INFO - step: 75045 loss: 16.3310 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8235 +[titan] 2025-07-10 06:05:43,560 - root - INFO - lr: 7.0163e-05 gnorm: 0.98 [16:30:41< 5:29:26] +[titan] 2025-07-10 06:05:46,674 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:05:47,458 - root - INFO - step: 75050 loss: 16.3826 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8654 +[titan] 2025-07-10 06:05:47,458 - root - INFO - lr: 7.0148e-05 gnorm: 0.93 [16:30:45< 5:29:22] +[titan] 2025-07-10 06:05:51,377 - root - INFO - step: 75055 loss: 16.3626 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8584 +[titan] 2025-07-10 06:05:51,377 - root - INFO - lr: 7.0132e-05 gnorm: 0.94 [16:30:49< 5:29:18] +[titan] 2025-07-10 06:05:55,348 - root - INFO - step: 75060 loss: 16.1863 memory: 44.58GiB(31.99%) tps: 82,519 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7203 +[titan] 2025-07-10 06:05:55,349 - root - INFO - lr: 7.0117e-05 gnorm: 1.00 [16:30:53< 5:29:14] +[titan] 2025-07-10 06:05:59,258 - root - INFO - step: 75065 loss: 16.4066 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.8869 +[titan] 2025-07-10 06:05:59,258 - root - INFO - lr: 7.0102e-05 gnorm: 0.94 [16:30:56< 5:29:10] +[titan] 2025-07-10 06:06:03,215 - root - INFO - step: 75070 loss: 16.4183 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5250 global_avg_mtp_loss: 13.8933 +[titan] 2025-07-10 06:06:03,215 - root - INFO - lr: 7.0087e-05 gnorm: 0.97 [16:31:00< 5:29:06] +[titan] 2025-07-10 06:06:07,131 - root - INFO - step: 75075 loss: 16.0510 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.6035 +[titan] 2025-07-10 06:06:07,131 - root - INFO - lr: 7.0071e-05 gnorm: 0.97 [16:31:04< 5:29:02] +[titan] 2025-07-10 06:06:11,031 - root - INFO - step: 75080 loss: 16.2290 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.7645 +[titan] 2025-07-10 06:06:11,032 - root - INFO - lr: 7.0056e-05 gnorm: 0.91 [16:31:08< 5:28:58] +[titan] 2025-07-10 06:06:14,945 - root - INFO - step: 75085 loss: 16.5207 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 13.9811 +[titan] 2025-07-10 06:06:14,945 - root - INFO - lr: 7.0041e-05 gnorm: 0.92 [16:31:12< 5:28:54] +[titan] 2025-07-10 06:06:18,898 - root - INFO - step: 75090 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9156 +[titan] 2025-07-10 06:06:18,898 - root - INFO - lr: 7.0026e-05 gnorm: 0.93 [16:31:16< 5:28:50] +[titan] 2025-07-10 06:06:22,813 - root - INFO - step: 75095 loss: 16.0261 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.5770 +[titan] 2025-07-10 06:06:22,814 - root - INFO - lr: 7.0011e-05 gnorm: 0.93 [16:31:20< 5:28:46] +[titan] 2025-07-10 06:06:25,930 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:06:26,726 - root - INFO - step: 75100 loss: 16.1323 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6736 +[titan] 2025-07-10 06:06:26,726 - root - INFO - lr: 6.9995e-05 gnorm: 0.96 [16:31:24< 5:28:42] +[titan] 2025-07-10 06:06:30,662 - root - INFO - step: 75105 loss: 16.2929 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.8074 +[titan] 2025-07-10 06:06:30,662 - root - INFO - lr: 6.9980e-05 gnorm: 0.95 [16:31:28< 5:28:38] +[titan] 2025-07-10 06:06:34,577 - root - INFO - step: 75110 loss: 16.1586 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.6973 +[titan] 2025-07-10 06:06:34,577 - root - INFO - lr: 6.9965e-05 gnorm: 0.92 [16:31:32< 5:28:34] +[titan] 2025-07-10 06:06:38,534 - root - INFO - step: 75115 loss: 16.0269 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.4526 global_avg_mtp_loss: 13.5742 +[titan] 2025-07-10 06:06:38,534 - root - INFO - lr: 6.9950e-05 gnorm: 0.94 [16:31:36< 5:28:30] +[titan] 2025-07-10 06:06:42,435 - root - INFO - step: 75120 loss: 16.2131 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7407 +[titan] 2025-07-10 06:06:42,436 - root - INFO - lr: 6.9934e-05 gnorm: 0.97 [16:31:40< 5:28:26] +[titan] 2025-07-10 06:06:46,349 - root - INFO - step: 75125 loss: 16.2619 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 06:06:46,349 - root - INFO - lr: 6.9919e-05 gnorm: 0.95 [16:31:44< 5:28:22] +[titan] 2025-07-10 06:06:50,273 - root - INFO - step: 75130 loss: 16.3309 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8139 +[titan] 2025-07-10 06:06:50,273 - root - INFO - lr: 6.9904e-05 gnorm: 0.99 [16:31:47< 5:28:18] +[titan] 2025-07-10 06:06:54,184 - root - INFO - step: 75135 loss: 16.3502 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8549 +[titan] 2025-07-10 06:06:54,184 - root - INFO - lr: 6.9889e-05 gnorm: 0.93 [16:31:51< 5:28:14] +[titan] 2025-07-10 06:06:58,130 - root - INFO - step: 75140 loss: 16.0656 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.5987 +[titan] 2025-07-10 06:06:58,131 - root - INFO - lr: 6.9874e-05 gnorm: 0.91 [16:31:55< 5:28:10] +[titan] 2025-07-10 06:07:02,025 - root - INFO - step: 75145 loss: 16.3104 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8189 +[titan] 2025-07-10 06:07:02,025 - root - INFO - lr: 6.9858e-05 gnorm: 0.96 [16:31:59< 5:28:06] +[titan] 2025-07-10 06:07:05,135 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:07:05,928 - root - INFO - step: 75150 loss: 16.1604 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.6901 +[titan] 2025-07-10 06:07:05,928 - root - INFO - lr: 6.9843e-05 gnorm: 0.92 [16:32:03< 5:28:02] +[titan] 2025-07-10 06:07:09,856 - root - INFO - step: 75155 loss: 16.0905 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4536 global_avg_mtp_loss: 13.6369 +[titan] 2025-07-10 06:07:09,857 - root - INFO - lr: 6.9828e-05 gnorm: 0.98 [16:32:07< 5:27:58] +[titan] 2025-07-10 06:07:13,774 - root - INFO - step: 75160 loss: 16.0353 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4449 global_avg_mtp_loss: 13.5905 +[titan] 2025-07-10 06:07:13,774 - root - INFO - lr: 6.9813e-05 gnorm: 0.97 [16:32:11< 5:27:54] +[titan] 2025-07-10 06:07:17,683 - root - INFO - step: 75165 loss: 16.1899 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7099 +[titan] 2025-07-10 06:07:17,684 - root - INFO - lr: 6.9798e-05 gnorm: 0.91 [16:32:15< 5:27:50] +[titan] 2025-07-10 06:07:21,582 - root - INFO - step: 75170 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8731 +[titan] 2025-07-10 06:07:21,582 - root - INFO - lr: 6.9782e-05 gnorm: 0.93 [16:32:19< 5:27:46] +[titan] 2025-07-10 06:07:25,498 - root - INFO - step: 75175 loss: 16.4360 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5206 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 06:07:25,499 - root - INFO - lr: 6.9767e-05 gnorm: 0.96 [16:32:23< 5:27:42] +[titan] 2025-07-10 06:07:29,437 - root - INFO - step: 75180 loss: 16.3676 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4878 global_avg_mtp_loss: 13.8797 +[titan] 2025-07-10 06:07:29,437 - root - INFO - lr: 6.9752e-05 gnorm: 0.96 [16:32:27< 5:27:38] +[titan] 2025-07-10 06:07:33,340 - root - INFO - step: 75185 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.7964 +[titan] 2025-07-10 06:07:33,340 - root - INFO - lr: 6.9737e-05 gnorm: 0.92 [16:32:31< 5:27:34] +[titan] 2025-07-10 06:07:37,253 - root - INFO - step: 75190 loss: 16.5708 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 14.0400 +[titan] 2025-07-10 06:07:37,254 - root - INFO - lr: 6.9722e-05 gnorm: 1.00 [16:32:34< 5:27:31] +[titan] 2025-07-10 06:07:41,163 - root - INFO - step: 75195 loss: 16.5813 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5524 global_avg_mtp_loss: 14.0290 +[titan] 2025-07-10 06:07:41,163 - root - INFO - lr: 6.9707e-05 gnorm: 0.95 [16:32:38< 5:27:27] +[titan] 2025-07-10 06:07:44,273 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:07:45,058 - root - INFO - step: 75200 loss: 16.1438 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6740 +[titan] 2025-07-10 06:07:45,058 - root - INFO - lr: 6.9691e-05 gnorm: 0.92 [16:32:42< 5:27:23] +[titan] 2025-07-10 06:07:48,963 - root - INFO - step: 75205 loss: 16.4595 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9257 +[titan] 2025-07-10 06:07:48,964 - root - INFO - lr: 6.9676e-05 gnorm: 0.94 [16:32:46< 5:27:19] +[titan] 2025-07-10 06:07:52,872 - root - INFO - step: 75210 loss: 16.3426 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 06:07:52,872 - root - INFO - lr: 6.9661e-05 gnorm: 0.92 [16:32:50< 5:27:15] +[titan] 2025-07-10 06:07:56,809 - root - INFO - step: 75215 loss: 16.1328 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.6583 +[titan] 2025-07-10 06:07:56,809 - root - INFO - lr: 6.9646e-05 gnorm: 1.01 [16:32:54< 5:27:11] +[titan] 2025-07-10 06:08:00,755 - root - INFO - step: 75220 loss: 16.2643 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.7723 +[titan] 2025-07-10 06:08:00,755 - root - INFO - lr: 6.9631e-05 gnorm: 0.95 [16:32:58< 5:27:07] +[titan] 2025-07-10 06:08:04,682 - root - INFO - step: 75225 loss: 16.1072 memory: 44.58GiB(31.99%) tps: 83,451 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4506 global_avg_mtp_loss: 13.6566 +[titan] 2025-07-10 06:08:04,682 - root - INFO - lr: 6.9616e-05 gnorm: 0.94 [16:33:02< 5:27:03] +[titan] 2025-07-10 06:08:08,586 - root - INFO - step: 75230 loss: 16.3815 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8706 +[titan] 2025-07-10 06:08:08,587 - root - INFO - lr: 6.9600e-05 gnorm: 0.92 [16:33:06< 5:26:59] +[titan] 2025-07-10 06:08:12,490 - root - INFO - step: 75235 loss: 16.7076 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5761 global_avg_mtp_loss: 14.1315 +[titan] 2025-07-10 06:08:12,490 - root - INFO - lr: 6.9585e-05 gnorm: 1.02 [16:33:10< 5:26:55] +[titan] 2025-07-10 06:08:16,411 - root - INFO - step: 75240 loss: 16.5103 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9855 +[titan] 2025-07-10 06:08:16,411 - root - INFO - lr: 6.9570e-05 gnorm: 0.94 [16:33:14< 5:26:51] +[titan] 2025-07-10 06:08:20,332 - root - INFO - step: 75245 loss: 16.2804 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.8050 +[titan] 2025-07-10 06:08:20,333 - root - INFO - lr: 6.9555e-05 gnorm: 0.89 [16:33:18< 5:26:47] +[titan] 2025-07-10 06:08:23,466 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:08:24,263 - root - INFO - step: 75250 loss: 16.5109 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5309 global_avg_mtp_loss: 13.9799 +[titan] 2025-07-10 06:08:24,263 - root - INFO - lr: 6.9540e-05 gnorm: 0.98 [16:33:21< 5:26:43] +[titan] 2025-07-10 06:08:28,165 - root - INFO - step: 75255 loss: 16.4711 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.9589 +[titan] 2025-07-10 06:08:28,166 - root - INFO - lr: 6.9525e-05 gnorm: 0.98 [16:33:25< 5:26:39] +[titan] 2025-07-10 06:08:32,099 - root - INFO - step: 75260 loss: 16.3774 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8672 +[titan] 2025-07-10 06:08:32,099 - root - INFO - lr: 6.9510e-05 gnorm: 0.95 [16:33:29< 5:26:35] +[titan] 2025-07-10 06:08:35,459 - root - INFO - Dumping profiler traces at step 75264 +[titan] 2025-07-10 06:08:35,491 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:08:36,285 - root - INFO - step: 75265 loss: 16.2387 memory: 44.58GiB(31.99%) tps: 78,275 tflops: 270.14 mfu: 27.31% global_avg_ntp_loss: 2.4893 global_avg_mtp_loss: 13.7494 +[titan] 2025-07-10 06:08:36,286 - root - INFO - lr: 6.9494e-05 gnorm: 0.92 [16:33:33< 5:26:31] +[titan] 2025-07-10 06:08:40,221 - root - INFO - step: 75270 loss: 16.4339 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.9192 +[titan] 2025-07-10 06:08:40,221 - root - INFO - lr: 6.9479e-05 gnorm: 0.93 [16:33:37< 5:26:27] +[titan] 2025-07-10 06:08:44,128 - root - INFO - step: 75275 loss: 16.1224 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6584 +[titan] 2025-07-10 06:08:44,128 - root - INFO - lr: 6.9464e-05 gnorm: 0.90 [16:33:41< 5:26:23] +[titan] 2025-07-10 06:08:48,073 - root - INFO - step: 75280 loss: 16.1557 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.6824 +[titan] 2025-07-10 06:08:48,073 - root - INFO - lr: 6.9449e-05 gnorm: 0.95 [16:33:45< 5:26:19] +[titan] 2025-07-10 06:08:51,967 - root - INFO - step: 75285 loss: 16.2761 memory: 44.58GiB(31.99%) tps: 84,153 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.7874 +[titan] 2025-07-10 06:08:51,968 - root - INFO - lr: 6.9434e-05 gnorm: 0.94 [16:33:49< 5:26:15] +[titan] 2025-07-10 06:08:55,905 - root - INFO - step: 75290 loss: 15.9839 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4409 global_avg_mtp_loss: 13.5430 +[titan] 2025-07-10 06:08:55,906 - root - INFO - lr: 6.9419e-05 gnorm: 0.96 [16:33:53< 5:26:11] +[titan] 2025-07-10 06:08:59,883 - root - INFO - step: 75295 loss: 16.5492 memory: 44.58GiB(31.99%) tps: 82,387 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.5462 global_avg_mtp_loss: 14.0031 +[titan] 2025-07-10 06:08:59,883 - root - INFO - lr: 6.9404e-05 gnorm: 0.95 [16:33:57< 5:26:07] +[titan] 2025-07-10 06:09:03,008 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:09:03,801 - root - INFO - step: 75300 loss: 16.2558 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7718 +[titan] 2025-07-10 06:09:03,801 - root - INFO - lr: 6.9388e-05 gnorm: 0.93 [16:34:01< 5:26:03] +[titan] 2025-07-10 06:09:07,713 - root - INFO - step: 75305 loss: 16.2375 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7580 +[titan] 2025-07-10 06:09:07,713 - root - INFO - lr: 6.9373e-05 gnorm: 0.96 [16:34:05< 5:25:59] +[titan] 2025-07-10 06:09:11,630 - root - INFO - step: 75310 loss: 16.2807 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.7962 +[titan] 2025-07-10 06:09:11,631 - root - INFO - lr: 6.9358e-05 gnorm: 0.93 [16:34:09< 5:25:55] +[titan] 2025-07-10 06:09:15,590 - root - INFO - step: 75315 loss: 16.0040 memory: 44.58GiB(31.99%) tps: 82,761 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4445 global_avg_mtp_loss: 13.5595 +[titan] 2025-07-10 06:09:15,590 - root - INFO - lr: 6.9343e-05 gnorm: 0.96 [16:34:13< 5:25:51] +[titan] 2025-07-10 06:09:19,489 - root - INFO - step: 75320 loss: 16.4118 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.8904 +[titan] 2025-07-10 06:09:19,490 - root - INFO - lr: 6.9328e-05 gnorm: 0.98 [16:34:17< 5:25:47] +[titan] 2025-07-10 06:09:23,410 - root - INFO - step: 75325 loss: 16.7208 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.5628 global_avg_mtp_loss: 14.1579 +[titan] 2025-07-10 06:09:23,411 - root - INFO - lr: 6.9313e-05 gnorm: 0.95 [16:34:21< 5:25:43] +[titan] 2025-07-10 06:09:27,318 - root - INFO - step: 75330 loss: 16.4073 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5328 global_avg_mtp_loss: 13.8745 +[titan] 2025-07-10 06:09:27,318 - root - INFO - lr: 6.9298e-05 gnorm: 0.96 [16:34:25< 5:25:39] +[titan] 2025-07-10 06:09:31,276 - root - INFO - step: 75335 loss: 16.2200 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7394 +[titan] 2025-07-10 06:09:31,277 - root - INFO - lr: 6.9283e-05 gnorm: 1.02 [16:34:28< 5:25:35] +[titan] 2025-07-10 06:09:35,175 - root - INFO - step: 75340 loss: 16.3447 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8457 +[titan] 2025-07-10 06:09:35,176 - root - INFO - lr: 6.9268e-05 gnorm: 0.93 [16:34:32< 5:25:31] +[titan] 2025-07-10 06:09:39,122 - root - INFO - step: 75345 loss: 16.5158 memory: 44.58GiB(31.99%) tps: 83,045 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9838 +[titan] 2025-07-10 06:09:39,122 - root - INFO - lr: 6.9252e-05 gnorm: 0.98 [16:34:36< 5:25:27] +[titan] 2025-07-10 06:09:42,249 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:09:43,039 - root - INFO - step: 75350 loss: 16.1985 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7163 +[titan] 2025-07-10 06:09:43,039 - root - INFO - lr: 6.9237e-05 gnorm: 0.95 [16:34:40< 5:25:23] +[titan] 2025-07-10 06:09:46,950 - root - INFO - step: 75355 loss: 16.3244 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8219 +[titan] 2025-07-10 06:09:46,950 - root - INFO - lr: 6.9222e-05 gnorm: 0.93 [16:34:44< 5:25:19] +[titan] 2025-07-10 06:09:50,877 - root - INFO - step: 75360 loss: 16.3076 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.8076 +[titan] 2025-07-10 06:09:50,877 - root - INFO - lr: 6.9207e-05 gnorm: 0.94 [16:34:48< 5:25:16] +[titan] 2025-07-10 06:09:54,823 - root - INFO - step: 75365 loss: 15.9987 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.5512 +[titan] 2025-07-10 06:09:54,823 - root - INFO - lr: 6.9192e-05 gnorm: 1.01 [16:34:52< 5:25:12] +[titan] 2025-07-10 06:09:58,755 - root - INFO - step: 75370 loss: 16.3167 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4941 global_avg_mtp_loss: 13.8226 +[titan] 2025-07-10 06:09:58,755 - root - INFO - lr: 6.9177e-05 gnorm: 0.95 [16:34:56< 5:25:08] +[titan] 2025-07-10 06:10:02,650 - root - INFO - step: 75375 loss: 16.5044 memory: 44.58GiB(31.99%) tps: 84,133 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5447 global_avg_mtp_loss: 13.9597 +[titan] 2025-07-10 06:10:02,650 - root - INFO - lr: 6.9162e-05 gnorm: 0.98 [16:35:00< 5:25:04] +[titan] 2025-07-10 06:10:06,563 - root - INFO - step: 75380 loss: 16.3264 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8195 +[titan] 2025-07-10 06:10:06,563 - root - INFO - lr: 6.9147e-05 gnorm: 0.97 [16:35:04< 5:25:00] +[titan] 2025-07-10 06:10:10,480 - root - INFO - step: 75385 loss: 16.3863 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 06:10:10,480 - root - INFO - lr: 6.9132e-05 gnorm: 0.92 [16:35:08< 5:24:56] +[titan] 2025-07-10 06:10:14,381 - root - INFO - step: 75390 loss: 16.2677 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4774 global_avg_mtp_loss: 13.7904 +[titan] 2025-07-10 06:10:14,381 - root - INFO - lr: 6.9117e-05 gnorm: 0.98 [16:35:12< 5:24:52] +[titan] 2025-07-10 06:10:18,327 - root - INFO - step: 75395 loss: 16.3442 memory: 44.58GiB(31.99%) tps: 83,052 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8511 +[titan] 2025-07-10 06:10:18,327 - root - INFO - lr: 6.9102e-05 gnorm: 0.93 [16:35:16< 5:24:48] +[titan] 2025-07-10 06:10:21,447 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:10:22,231 - root - INFO - step: 75400 loss: 15.9325 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4280 global_avg_mtp_loss: 13.5044 +[titan] 2025-07-10 06:10:22,231 - root - INFO - lr: 6.9087e-05 gnorm: 0.95 [16:35:19< 5:24:44] +[titan] 2025-07-10 06:10:26,142 - root - INFO - step: 75405 loss: 16.5211 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9933 +[titan] 2025-07-10 06:10:26,142 - root - INFO - lr: 6.9071e-05 gnorm: 0.97 [16:35:23< 5:24:40] +[titan] 2025-07-10 06:10:30,056 - root - INFO - step: 75410 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.7421 +[titan] 2025-07-10 06:10:30,056 - root - INFO - lr: 6.9056e-05 gnorm: 0.95 [16:35:27< 5:24:36] +[titan] 2025-07-10 06:10:33,979 - root - INFO - step: 75415 loss: 16.1954 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7218 +[titan] 2025-07-10 06:10:33,979 - root - INFO - lr: 6.9041e-05 gnorm: 0.95 [16:35:31< 5:24:32] +[titan] 2025-07-10 06:10:37,917 - root - INFO - step: 75420 loss: 16.6701 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.5718 global_avg_mtp_loss: 14.0982 +[titan] 2025-07-10 06:10:37,917 - root - INFO - lr: 6.9026e-05 gnorm: 0.97 [16:35:35< 5:24:28] +[titan] 2025-07-10 06:10:41,828 - root - INFO - step: 75425 loss: 16.0956 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.6354 +[titan] 2025-07-10 06:10:41,829 - root - INFO - lr: 6.9011e-05 gnorm: 0.91 [16:35:39< 5:24:24] +[titan] 2025-07-10 06:10:45,760 - root - INFO - step: 75430 loss: 16.2101 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7355 +[titan] 2025-07-10 06:10:45,760 - root - INFO - lr: 6.8996e-05 gnorm: 0.95 [16:35:43< 5:24:20] +[titan] 2025-07-10 06:10:49,678 - root - INFO - step: 75435 loss: 16.3284 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.8445 +[titan] 2025-07-10 06:10:49,678 - root - INFO - lr: 6.8981e-05 gnorm: 1.04 [16:35:47< 5:24:16] +[titan] 2025-07-10 06:10:53,636 - root - INFO - step: 75440 loss: 16.1117 memory: 44.58GiB(31.99%) tps: 82,787 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.4465 global_avg_mtp_loss: 13.6651 +[titan] 2025-07-10 06:10:53,636 - root - INFO - lr: 6.8966e-05 gnorm: 0.94 [16:35:51< 5:24:12] +[titan] 2025-07-10 06:10:57,576 - root - INFO - step: 75445 loss: 16.1409 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6805 +[titan] 2025-07-10 06:10:57,576 - root - INFO - lr: 6.8951e-05 gnorm: 1.12 [16:35:55< 5:24:08] +[titan] 2025-07-10 06:11:00,706 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:11:01,516 - root - INFO - step: 75450 loss: 16.2560 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.7646 +[titan] 2025-07-10 06:11:01,516 - root - INFO - lr: 6.8936e-05 gnorm: 0.98 [16:35:59< 5:24:04] +[titan] 2025-07-10 06:11:05,434 - root - INFO - step: 75455 loss: 16.2781 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.7862 +[titan] 2025-07-10 06:11:05,434 - root - INFO - lr: 6.8921e-05 gnorm: 0.96 [16:36:03< 5:24:00] +[titan] 2025-07-10 06:11:09,363 - root - INFO - step: 75460 loss: 16.3756 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8629 +[titan] 2025-07-10 06:11:09,363 - root - INFO - lr: 6.8906e-05 gnorm: 0.94 [16:36:07< 5:23:56] +[titan] 2025-07-10 06:11:13,316 - root - INFO - step: 75465 loss: 16.6918 memory: 44.58GiB(31.99%) tps: 82,894 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.5663 global_avg_mtp_loss: 14.1255 +[titan] 2025-07-10 06:11:13,316 - root - INFO - lr: 6.8891e-05 gnorm: 1.01 [16:36:11< 5:23:52] +[titan] 2025-07-10 06:11:17,236 - root - INFO - step: 75470 loss: 16.3402 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 06:11:17,236 - root - INFO - lr: 6.8876e-05 gnorm: 0.96 [16:36:14< 5:23:48] +[titan] 2025-07-10 06:11:21,147 - root - INFO - step: 75475 loss: 16.3326 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8366 +[titan] 2025-07-10 06:11:21,147 - root - INFO - lr: 6.8861e-05 gnorm: 0.92 [16:36:18< 5:23:44] +[titan] 2025-07-10 06:11:25,090 - root - INFO - step: 75480 loss: 16.1600 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.6830 +[titan] 2025-07-10 06:11:25,090 - root - INFO - lr: 6.8846e-05 gnorm: 0.92 [16:36:22< 5:23:40] +[titan] 2025-07-10 06:11:29,020 - root - INFO - step: 75485 loss: 16.3342 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-10 06:11:29,020 - root - INFO - lr: 6.8831e-05 gnorm: 0.95 [16:36:26< 5:23:36] +[titan] 2025-07-10 06:11:32,940 - root - INFO - step: 75490 loss: 16.2826 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7979 +[titan] 2025-07-10 06:11:32,941 - root - INFO - lr: 6.8816e-05 gnorm: 0.97 [16:36:30< 5:23:32] +[titan] 2025-07-10 06:11:36,877 - root - INFO - step: 75495 loss: 16.0229 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4341 global_avg_mtp_loss: 13.5888 +[titan] 2025-07-10 06:11:36,877 - root - INFO - lr: 6.8801e-05 gnorm: 1.02 [16:36:34< 5:23:28] +[titan] 2025-07-10 06:11:40,008 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:11:40,800 - root - INFO - step: 75500 loss: 16.1225 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6512 +[titan] 2025-07-10 06:11:40,800 - root - INFO - lr: 6.8786e-05 gnorm: 0.95 [16:36:38< 5:23:24] +[titan] 2025-07-10 06:11:44,719 - root - INFO - step: 75505 loss: 16.2286 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 06:11:44,719 - root - INFO - lr: 6.8771e-05 gnorm: 0.92 [16:36:42< 5:23:20] +[titan] 2025-07-10 06:11:48,613 - root - INFO - step: 75510 loss: 16.4353 memory: 44.58GiB(31.99%) tps: 84,149 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.9175 +[titan] 2025-07-10 06:11:48,613 - root - INFO - lr: 6.8755e-05 gnorm: 0.92 [16:36:46< 5:23:16] +[titan] 2025-07-10 06:11:52,527 - root - INFO - step: 75515 loss: 16.4537 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.9380 +[titan] 2025-07-10 06:11:52,527 - root - INFO - lr: 6.8740e-05 gnorm: 0.92 [16:36:50< 5:23:12] +[titan] 2025-07-10 06:11:56,450 - root - INFO - step: 75520 loss: 16.3482 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8545 +[titan] 2025-07-10 06:11:56,450 - root - INFO - lr: 6.8725e-05 gnorm: 0.93 [16:36:54< 5:23:08] +[titan] 2025-07-10 06:12:00,355 - root - INFO - step: 75525 loss: 16.2706 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7721 +[titan] 2025-07-10 06:12:00,355 - root - INFO - lr: 6.8710e-05 gnorm: 0.96 [16:36:58< 5:23:04] +[titan] 2025-07-10 06:12:04,256 - root - INFO - step: 75530 loss: 16.5718 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 14.0368 +[titan] 2025-07-10 06:12:04,257 - root - INFO - lr: 6.8695e-05 gnorm: 0.91 [16:37:01< 5:23:00] +[titan] 2025-07-10 06:12:08,172 - root - INFO - step: 75535 loss: 16.1317 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4551 global_avg_mtp_loss: 13.6766 +[titan] 2025-07-10 06:12:08,172 - root - INFO - lr: 6.8680e-05 gnorm: 1.00 [16:37:05< 5:22:56] +[titan] 2025-07-10 06:12:12,091 - root - INFO - step: 75540 loss: 16.1917 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.7270 +[titan] 2025-07-10 06:12:12,091 - root - INFO - lr: 6.8665e-05 gnorm: 0.97 [16:37:09< 5:22:52] +[titan] 2025-07-10 06:12:16,003 - root - INFO - step: 75545 loss: 16.3163 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8232 +[titan] 2025-07-10 06:12:16,003 - root - INFO - lr: 6.8650e-05 gnorm: 0.93 [16:37:13< 5:22:49] +[titan] 2025-07-10 06:12:19,128 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:12:19,938 - root - INFO - step: 75550 loss: 16.1031 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4521 global_avg_mtp_loss: 13.6510 +[titan] 2025-07-10 06:12:19,939 - root - INFO - lr: 6.8635e-05 gnorm: 0.92 [16:37:17< 5:22:45] +[titan] 2025-07-10 06:12:23,860 - root - INFO - step: 75555 loss: 16.2082 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7335 +[titan] 2025-07-10 06:12:23,860 - root - INFO - lr: 6.8620e-05 gnorm: 0.94 [16:37:21< 5:22:41] +[titan] 2025-07-10 06:12:27,800 - root - INFO - step: 75560 loss: 16.4990 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.5329 global_avg_mtp_loss: 13.9661 +[titan] 2025-07-10 06:12:27,800 - root - INFO - lr: 6.8605e-05 gnorm: 0.96 [16:37:25< 5:22:37] +[titan] 2025-07-10 06:12:31,710 - root - INFO - step: 75565 loss: 16.0332 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.5859 +[titan] 2025-07-10 06:12:31,711 - root - INFO - lr: 6.8590e-05 gnorm: 0.97 [16:37:29< 5:22:33] +[titan] 2025-07-10 06:12:35,626 - root - INFO - step: 75570 loss: 16.2898 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7996 +[titan] 2025-07-10 06:12:35,627 - root - INFO - lr: 6.8575e-05 gnorm: 0.94 [16:37:33< 5:22:29] +[titan] 2025-07-10 06:12:39,569 - root - INFO - step: 75575 loss: 16.1435 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 06:12:39,569 - root - INFO - lr: 6.8560e-05 gnorm: 1.16 [16:37:37< 5:22:25] +[titan] 2025-07-10 06:12:43,502 - root - INFO - step: 75580 loss: 16.3596 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8650 +[titan] 2025-07-10 06:12:43,502 - root - INFO - lr: 6.8545e-05 gnorm: 0.93 [16:37:41< 5:22:21] +[titan] 2025-07-10 06:12:47,408 - root - INFO - step: 75585 loss: 16.4270 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.9041 +[titan] 2025-07-10 06:12:47,408 - root - INFO - lr: 6.8530e-05 gnorm: 0.99 [16:37:45< 5:22:17] +[titan] 2025-07-10 06:12:51,318 - root - INFO - step: 75590 loss: 16.6078 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0585 +[titan] 2025-07-10 06:12:51,318 - root - INFO - lr: 6.8515e-05 gnorm: 0.95 [16:37:48< 5:22:13] +[titan] 2025-07-10 06:12:55,275 - root - INFO - step: 75595 loss: 16.1169 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.6537 +[titan] 2025-07-10 06:12:55,275 - root - INFO - lr: 6.8501e-05 gnorm: 1.02 [16:37:52< 5:22:09] +[titan] 2025-07-10 06:12:58,415 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:12:59,221 - root - INFO - step: 75600 loss: 16.1730 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.7012 +[titan] 2025-07-10 06:12:59,221 - root - INFO - lr: 6.8486e-05 gnorm: 0.95 [16:37:56< 5:22:05] +[titan] 2025-07-10 06:13:03,159 - root - INFO - step: 75605 loss: 16.1437 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4555 global_avg_mtp_loss: 13.6882 +[titan] 2025-07-10 06:13:03,159 - root - INFO - lr: 6.8471e-05 gnorm: 0.93 [16:38:00< 5:22:01] +[titan] 2025-07-10 06:13:07,083 - root - INFO - step: 75610 loss: 16.4218 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 06:13:07,084 - root - INFO - lr: 6.8456e-05 gnorm: 0.93 [16:38:04< 5:21:57] +[titan] 2025-07-10 06:13:11,021 - root - INFO - step: 75615 loss: 16.4651 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9339 +[titan] 2025-07-10 06:13:11,022 - root - INFO - lr: 6.8441e-05 gnorm: 0.97 [16:38:08< 5:21:53] +[titan] 2025-07-10 06:13:15,008 - root - INFO - step: 75620 loss: 16.2756 memory: 44.58GiB(31.99%) tps: 82,213 tflops: 283.73 mfu: 28.69% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.7785 +[titan] 2025-07-10 06:13:15,008 - root - INFO - lr: 6.8426e-05 gnorm: 1.05 [16:38:12< 5:21:49] +[titan] 2025-07-10 06:13:18,949 - root - INFO - step: 75625 loss: 16.2431 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.7538 +[titan] 2025-07-10 06:13:18,950 - root - INFO - lr: 6.8411e-05 gnorm: 0.96 [16:38:16< 5:21:45] +[titan] 2025-07-10 06:13:22,868 - root - INFO - step: 75630 loss: 16.2433 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 06:13:22,869 - root - INFO - lr: 6.8396e-05 gnorm: 0.90 [16:38:20< 5:21:41] +[titan] 2025-07-10 06:13:26,767 - root - INFO - step: 75635 loss: 16.4126 memory: 44.58GiB(31.99%) tps: 84,051 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5159 global_avg_mtp_loss: 13.8968 +[titan] 2025-07-10 06:13:26,768 - root - INFO - lr: 6.8381e-05 gnorm: 0.97 [16:38:24< 5:21:37] +[titan] 2025-07-10 06:13:30,686 - root - INFO - step: 75640 loss: 16.3554 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 06:13:30,686 - root - INFO - lr: 6.8366e-05 gnorm: 0.89 [16:38:28< 5:21:33] +[titan] 2025-07-10 06:13:34,596 - root - INFO - step: 75645 loss: 16.0844 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4560 global_avg_mtp_loss: 13.6284 +[titan] 2025-07-10 06:13:34,596 - root - INFO - lr: 6.8351e-05 gnorm: 0.93 [16:38:32< 5:21:29] +[titan] 2025-07-10 06:13:37,726 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:13:38,518 - root - INFO - step: 75650 loss: 16.6116 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 14.0605 +[titan] 2025-07-10 06:13:38,518 - root - INFO - lr: 6.8336e-05 gnorm: 0.97 [16:38:36< 5:21:25] +[titan] 2025-07-10 06:13:42,443 - root - INFO - step: 75655 loss: 16.2688 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7763 +[titan] 2025-07-10 06:13:42,444 - root - INFO - lr: 6.8321e-05 gnorm: 0.92 [16:38:40< 5:21:21] +[titan] 2025-07-10 06:13:46,355 - root - INFO - step: 75660 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 06:13:46,356 - root - INFO - lr: 6.8306e-05 gnorm: 0.88 [16:38:44< 5:21:17] +[titan] 2025-07-10 06:13:50,258 - root - INFO - step: 75665 loss: 16.0202 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4455 global_avg_mtp_loss: 13.5747 +[titan] 2025-07-10 06:13:50,258 - root - INFO - lr: 6.8291e-05 gnorm: 0.95 [16:38:47< 5:21:13] +[titan] 2025-07-10 06:13:54,184 - root - INFO - step: 75670 loss: 16.1511 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4663 global_avg_mtp_loss: 13.6848 +[titan] 2025-07-10 06:13:54,184 - root - INFO - lr: 6.8276e-05 gnorm: 0.88 [16:38:51< 5:21:09] +[titan] 2025-07-10 06:13:58,110 - root - INFO - step: 75675 loss: 16.4722 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5343 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-10 06:13:58,110 - root - INFO - lr: 6.8261e-05 gnorm: 1.01 [16:38:55< 5:21:05] +[titan] 2025-07-10 06:14:02,014 - root - INFO - step: 75680 loss: 16.3899 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8822 +[titan] 2025-07-10 06:14:02,014 - root - INFO - lr: 6.8246e-05 gnorm: 0.93 [16:38:59< 5:21:01] +[titan] 2025-07-10 06:14:05,918 - root - INFO - step: 75685 loss: 15.9832 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4348 global_avg_mtp_loss: 13.5484 +[titan] 2025-07-10 06:14:05,918 - root - INFO - lr: 6.8231e-05 gnorm: 0.99 [16:39:03< 5:20:57] +[titan] 2025-07-10 06:14:09,832 - root - INFO - step: 75690 loss: 16.0516 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4541 global_avg_mtp_loss: 13.5975 +[titan] 2025-07-10 06:14:09,833 - root - INFO - lr: 6.8216e-05 gnorm: 0.92 [16:39:07< 5:20:53] +[titan] 2025-07-10 06:14:13,767 - root - INFO - step: 75695 loss: 16.2472 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.7746 +[titan] 2025-07-10 06:14:13,767 - root - INFO - lr: 6.8201e-05 gnorm: 0.92 [16:39:11< 5:20:49] +[titan] 2025-07-10 06:14:16,907 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:14:17,701 - root - INFO - step: 75700 loss: 16.3498 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8391 +[titan] 2025-07-10 06:14:17,701 - root - INFO - lr: 6.8186e-05 gnorm: 0.96 [16:39:15< 5:20:45] +[titan] 2025-07-10 06:14:21,630 - root - INFO - step: 75705 loss: 16.5424 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5417 global_avg_mtp_loss: 14.0007 +[titan] 2025-07-10 06:14:21,631 - root - INFO - lr: 6.8172e-05 gnorm: 0.96 [16:39:19< 5:20:41] +[titan] 2025-07-10 06:14:25,536 - root - INFO - step: 75710 loss: 16.5752 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5448 global_avg_mtp_loss: 14.0304 +[titan] 2025-07-10 06:14:25,537 - root - INFO - lr: 6.8157e-05 gnorm: 0.92 [16:39:23< 5:20:37] +[titan] 2025-07-10 06:14:29,478 - root - INFO - step: 75715 loss: 16.3783 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.8665 +[titan] 2025-07-10 06:14:29,478 - root - INFO - lr: 6.8142e-05 gnorm: 0.96 [16:39:27< 5:20:33] +[titan] 2025-07-10 06:14:33,376 - root - INFO - step: 75720 loss: 16.2291 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7454 +[titan] 2025-07-10 06:14:33,376 - root - INFO - lr: 6.8127e-05 gnorm: 0.97 [16:39:31< 5:20:30] +[titan] 2025-07-10 06:14:37,270 - root - INFO - step: 75725 loss: 16.0269 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4438 global_avg_mtp_loss: 13.5831 +[titan] 2025-07-10 06:14:37,270 - root - INFO - lr: 6.8112e-05 gnorm: 0.92 [16:39:34< 5:20:26] +[titan] 2025-07-10 06:14:41,189 - root - INFO - step: 75730 loss: 16.2021 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7333 +[titan] 2025-07-10 06:14:41,189 - root - INFO - lr: 6.8097e-05 gnorm: 0.94 [16:39:38< 5:20:22] +[titan] 2025-07-10 06:14:45,087 - root - INFO - step: 75735 loss: 16.2513 memory: 44.58GiB(31.99%) tps: 84,064 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7640 +[titan] 2025-07-10 06:14:45,087 - root - INFO - lr: 6.8082e-05 gnorm: 1.06 [16:39:42< 5:20:18] +[titan] 2025-07-10 06:14:49,016 - root - INFO - step: 75740 loss: 16.0873 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6237 +[titan] 2025-07-10 06:14:49,016 - root - INFO - lr: 6.8067e-05 gnorm: 0.93 [16:39:46< 5:20:14] +[titan] 2025-07-10 06:14:52,929 - root - INFO - step: 75745 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8614 +[titan] 2025-07-10 06:14:52,929 - root - INFO - lr: 6.8052e-05 gnorm: 0.94 [16:39:50< 5:20:10] +[titan] 2025-07-10 06:14:56,052 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:14:56,836 - root - INFO - step: 75750 loss: 16.3925 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.8752 +[titan] 2025-07-10 06:14:56,836 - root - INFO - lr: 6.8037e-05 gnorm: 0.95 [16:39:54< 5:20:06] +[titan] 2025-07-10 06:15:00,752 - root - INFO - step: 75755 loss: 16.1206 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4664 global_avg_mtp_loss: 13.6541 +[titan] 2025-07-10 06:15:00,753 - root - INFO - lr: 6.8022e-05 gnorm: 0.94 [16:39:58< 5:20:02] +[titan] 2025-07-10 06:15:04,711 - root - INFO - step: 75760 loss: 16.1033 memory: 44.58GiB(31.99%) tps: 82,783 tflops: 285.70 mfu: 28.89% global_avg_ntp_loss: 2.4495 global_avg_mtp_loss: 13.6538 +[titan] 2025-07-10 06:15:04,711 - root - INFO - lr: 6.8008e-05 gnorm: 0.93 [16:40:02< 5:19:58] +[titan] 2025-07-10 06:15:08,626 - root - INFO - step: 75765 loss: 16.0661 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4432 global_avg_mtp_loss: 13.6229 +[titan] 2025-07-10 06:15:08,626 - root - INFO - lr: 6.7993e-05 gnorm: 0.93 [16:40:06< 5:19:54] +[titan] 2025-07-10 06:15:12,569 - root - INFO - step: 75770 loss: 16.3738 memory: 44.58GiB(31.99%) tps: 83,099 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.8799 +[titan] 2025-07-10 06:15:12,570 - root - INFO - lr: 6.7978e-05 gnorm: 0.92 [16:40:10< 5:19:50] +[titan] 2025-07-10 06:15:16,595 - root - INFO - step: 75775 loss: 16.2389 memory: 44.58GiB(31.99%) tps: 81,411 tflops: 280.96 mfu: 28.41% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7593 +[titan] 2025-07-10 06:15:16,595 - root - INFO - lr: 6.7963e-05 gnorm: 0.93 [16:40:14< 5:19:46] +[titan] 2025-07-10 06:15:17,537 - root - INFO - Dumping profiler traces at step 75776 +[titan] 2025-07-10 06:15:17,569 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:15:20,706 - root - INFO - step: 75780 loss: 16.4691 memory: 44.58GiB(31.99%) tps: 79,719 tflops: 275.12 mfu: 27.82% global_avg_ntp_loss: 2.5375 global_avg_mtp_loss: 13.9316 +[titan] 2025-07-10 06:15:20,706 - root - INFO - lr: 6.7948e-05 gnorm: 0.96 [16:40:18< 5:19:42] +[titan] 2025-07-10 06:15:24,622 - root - INFO - step: 75785 loss: 16.2510 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7718 +[titan] 2025-07-10 06:15:24,622 - root - INFO - lr: 6.7933e-05 gnorm: 0.97 [16:40:22< 5:19:38] +[titan] 2025-07-10 06:15:28,585 - root - INFO - step: 75790 loss: 16.0320 memory: 44.58GiB(31.99%) tps: 82,702 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.4312 global_avg_mtp_loss: 13.6007 +[titan] 2025-07-10 06:15:28,585 - root - INFO - lr: 6.7918e-05 gnorm: 0.99 [16:40:26< 5:19:34] +[titan] 2025-07-10 06:15:32,496 - root - INFO - step: 75795 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5251 global_avg_mtp_loss: 13.9199 +[titan] 2025-07-10 06:15:32,496 - root - INFO - lr: 6.7903e-05 gnorm: 0.96 [16:40:30< 5:19:30] +[titan] 2025-07-10 06:15:35,616 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:15:36,412 - root - INFO - step: 75800 loss: 16.0317 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5883 +[titan] 2025-07-10 06:15:36,412 - root - INFO - lr: 6.7888e-05 gnorm: 0.98 [16:40:34< 5:19:26] +[titan] 2025-07-10 06:15:40,345 - root - INFO - step: 75805 loss: 16.4369 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.9213 +[titan] 2025-07-10 06:15:40,345 - root - INFO - lr: 6.7874e-05 gnorm: 0.98 [16:40:38< 5:19:22] +[titan] 2025-07-10 06:15:44,274 - root - INFO - step: 75810 loss: 16.3448 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.8322 +[titan] 2025-07-10 06:15:44,274 - root - INFO - lr: 6.7859e-05 gnorm: 0.97 [16:40:41< 5:19:18] +[titan] 2025-07-10 06:15:48,207 - root - INFO - step: 75815 loss: 16.1277 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.6643 +[titan] 2025-07-10 06:15:48,207 - root - INFO - lr: 6.7844e-05 gnorm: 0.96 [16:40:45< 5:19:14] +[titan] 2025-07-10 06:15:52,128 - root - INFO - step: 75820 loss: 16.4193 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5033 global_avg_mtp_loss: 13.9160 +[titan] 2025-07-10 06:15:52,128 - root - INFO - lr: 6.7829e-05 gnorm: 0.97 [16:40:49< 5:19:10] +[titan] 2025-07-10 06:15:56,030 - root - INFO - step: 75825 loss: 16.1781 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.7122 +[titan] 2025-07-10 06:15:56,030 - root - INFO - lr: 6.7814e-05 gnorm: 0.96 [16:40:53< 5:19:06] +[titan] 2025-07-10 06:15:59,949 - root - INFO - step: 75830 loss: 16.1982 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7275 +[titan] 2025-07-10 06:15:59,949 - root - INFO - lr: 6.7799e-05 gnorm: 0.94 [16:40:57< 5:19:02] +[titan] 2025-07-10 06:16:03,895 - root - INFO - step: 75835 loss: 16.5334 memory: 44.58GiB(31.99%) tps: 83,041 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5601 global_avg_mtp_loss: 13.9733 +[titan] 2025-07-10 06:16:03,896 - root - INFO - lr: 6.7784e-05 gnorm: 0.96 [16:41:01< 5:18:58] +[titan] 2025-07-10 06:16:07,806 - root - INFO - step: 75840 loss: 16.5348 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 14.0044 +[titan] 2025-07-10 06:16:07,806 - root - INFO - lr: 6.7769e-05 gnorm: 0.98 [16:41:05< 5:18:54] +[titan] 2025-07-10 06:16:11,717 - root - INFO - step: 75845 loss: 16.2328 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.7388 +[titan] 2025-07-10 06:16:11,717 - root - INFO - lr: 6.7755e-05 gnorm: 0.95 [16:41:09< 5:18:50] +[titan] 2025-07-10 06:16:14,919 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:16:15,712 - root - INFO - step: 75850 loss: 16.4478 memory: 44.58GiB(31.99%) tps: 82,036 tflops: 283.12 mfu: 28.63% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.9289 +[titan] 2025-07-10 06:16:15,712 - root - INFO - lr: 6.7740e-05 gnorm: 0.96 [16:41:13< 5:18:46] +[titan] 2025-07-10 06:16:19,616 - root - INFO - step: 75855 loss: 16.2732 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 06:16:19,616 - root - INFO - lr: 6.7725e-05 gnorm: 0.99 [16:41:17< 5:18:42] +[titan] 2025-07-10 06:16:23,531 - root - INFO - step: 75860 loss: 16.2410 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 06:16:23,531 - root - INFO - lr: 6.7710e-05 gnorm: 1.01 [16:41:21< 5:18:38] +[titan] 2025-07-10 06:16:27,458 - root - INFO - step: 75865 loss: 16.2746 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.7864 +[titan] 2025-07-10 06:16:27,458 - root - INFO - lr: 6.7695e-05 gnorm: 0.93 [16:41:25< 5:18:34] +[titan] 2025-07-10 06:16:31,380 - root - INFO - step: 75870 loss: 16.2143 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.7441 +[titan] 2025-07-10 06:16:31,381 - root - INFO - lr: 6.7680e-05 gnorm: 0.99 [16:41:29< 5:18:30] +[titan] 2025-07-10 06:16:35,308 - root - INFO - step: 75875 loss: 16.2785 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7938 +[titan] 2025-07-10 06:16:35,308 - root - INFO - lr: 6.7665e-05 gnorm: 0.94 [16:41:32< 5:18:26] +[titan] 2025-07-10 06:16:39,253 - root - INFO - step: 75880 loss: 16.4075 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8959 +[titan] 2025-07-10 06:16:39,253 - root - INFO - lr: 6.7651e-05 gnorm: 0.96 [16:41:36< 5:18:23] +[titan] 2025-07-10 06:16:43,209 - root - INFO - step: 75885 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.9149 +[titan] 2025-07-10 06:16:43,209 - root - INFO - lr: 6.7636e-05 gnorm: 0.94 [16:41:40< 5:18:19] +[titan] 2025-07-10 06:16:47,115 - root - INFO - step: 75890 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 06:16:47,115 - root - INFO - lr: 6.7621e-05 gnorm: 0.93 [16:41:44< 5:18:15] +[titan] 2025-07-10 06:16:51,044 - root - INFO - step: 75895 loss: 16.3812 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.8687 +[titan] 2025-07-10 06:16:51,044 - root - INFO - lr: 6.7606e-05 gnorm: 0.93 [16:41:48< 5:18:11] +[titan] 2025-07-10 06:16:54,146 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:16:54,928 - root - INFO - step: 75900 loss: 16.4193 memory: 44.58GiB(31.99%) tps: 84,371 tflops: 291.18 mfu: 29.44% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.9123 +[titan] 2025-07-10 06:16:54,929 - root - INFO - lr: 6.7591e-05 gnorm: 0.92 [16:41:52< 5:18:07] +[titan] 2025-07-10 06:16:58,843 - root - INFO - step: 75905 loss: 16.1246 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.6614 +[titan] 2025-07-10 06:16:58,843 - root - INFO - lr: 6.7576e-05 gnorm: 0.96 [16:41:56< 5:18:03] +[titan] 2025-07-10 06:17:02,811 - root - INFO - step: 75910 loss: 16.2447 memory: 44.58GiB(31.99%) tps: 82,575 tflops: 284.98 mfu: 28.81% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7646 +[titan] 2025-07-10 06:17:02,812 - root - INFO - lr: 6.7562e-05 gnorm: 0.95 [16:42:00< 5:17:59] +[titan] 2025-07-10 06:17:06,766 - root - INFO - step: 75915 loss: 16.4264 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.9245 +[titan] 2025-07-10 06:17:06,767 - root - INFO - lr: 6.7547e-05 gnorm: 1.03 [16:42:04< 5:17:55] +[titan] 2025-07-10 06:17:10,696 - root - INFO - step: 75920 loss: 16.1037 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6468 +[titan] 2025-07-10 06:17:10,696 - root - INFO - lr: 6.7532e-05 gnorm: 0.92 [16:42:08< 5:17:51] +[titan] 2025-07-10 06:17:14,593 - root - INFO - step: 75925 loss: 16.3915 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 06:17:14,593 - root - INFO - lr: 6.7517e-05 gnorm: 0.98 [16:42:12< 5:17:47] +[titan] 2025-07-10 06:17:18,509 - root - INFO - step: 75930 loss: 16.3293 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8203 +[titan] 2025-07-10 06:17:18,509 - root - INFO - lr: 6.7502e-05 gnorm: 1.00 [16:42:16< 5:17:43] +[titan] 2025-07-10 06:17:22,412 - root - INFO - step: 75935 loss: 16.3244 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8195 +[titan] 2025-07-10 06:17:22,412 - root - INFO - lr: 6.7488e-05 gnorm: 0.94 [16:42:20< 5:17:39] +[titan] 2025-07-10 06:17:26,320 - root - INFO - step: 75940 loss: 16.1859 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.7210 +[titan] 2025-07-10 06:17:26,320 - root - INFO - lr: 6.7473e-05 gnorm: 0.98 [16:42:23< 5:17:35] +[titan] 2025-07-10 06:17:30,242 - root - INFO - step: 75945 loss: 16.3829 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8881 +[titan] 2025-07-10 06:17:30,242 - root - INFO - lr: 6.7458e-05 gnorm: 0.97 [16:42:27< 5:17:31] +[titan] 2025-07-10 06:17:33,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:17:34,164 - root - INFO - step: 75950 loss: 16.2191 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7427 +[titan] 2025-07-10 06:17:34,164 - root - INFO - lr: 6.7443e-05 gnorm: 0.91 [16:42:31< 5:17:27] +[titan] 2025-07-10 06:17:38,060 - root - INFO - step: 75955 loss: 16.4553 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.9394 +[titan] 2025-07-10 06:17:38,060 - root - INFO - lr: 6.7428e-05 gnorm: 0.92 [16:42:35< 5:17:23] +[titan] 2025-07-10 06:17:41,960 - root - INFO - step: 75960 loss: 15.8326 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4174 global_avg_mtp_loss: 13.4152 +[titan] 2025-07-10 06:17:41,960 - root - INFO - lr: 6.7414e-05 gnorm: 0.91 [16:42:39< 5:17:19] +[titan] 2025-07-10 06:17:45,907 - root - INFO - step: 75965 loss: 16.4578 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.9446 +[titan] 2025-07-10 06:17:45,907 - root - INFO - lr: 6.7399e-05 gnorm: 0.92 [16:42:43< 5:17:15] +[titan] 2025-07-10 06:17:49,802 - root - INFO - step: 75970 loss: 16.1132 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6558 +[titan] 2025-07-10 06:17:49,802 - root - INFO - lr: 6.7384e-05 gnorm: 0.95 [16:42:47< 5:17:11] +[titan] 2025-07-10 06:17:53,696 - root - INFO - step: 75975 loss: 16.3206 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.8282 +[titan] 2025-07-10 06:17:53,696 - root - INFO - lr: 6.7369e-05 gnorm: 0.97 [16:42:51< 5:17:07] +[titan] 2025-07-10 06:17:57,601 - root - INFO - step: 75980 loss: 16.3786 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.8515 +[titan] 2025-07-10 06:17:57,602 - root - INFO - lr: 6.7354e-05 gnorm: 0.94 [16:42:55< 5:17:03] +[titan] 2025-07-10 06:18:01,558 - root - INFO - step: 75985 loss: 16.1113 memory: 44.58GiB(31.99%) tps: 82,829 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6626 +[titan] 2025-07-10 06:18:01,558 - root - INFO - lr: 6.7340e-05 gnorm: 0.98 [16:42:59< 5:16:59] +[titan] 2025-07-10 06:18:05,479 - root - INFO - step: 75990 loss: 16.1971 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.7246 +[titan] 2025-07-10 06:18:05,480 - root - INFO - lr: 6.7325e-05 gnorm: 0.93 [16:43:03< 5:16:55] +[titan] 2025-07-10 06:18:09,412 - root - INFO - step: 75995 loss: 16.4291 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9086 +[titan] 2025-07-10 06:18:09,412 - root - INFO - lr: 6.7310e-05 gnorm: 0.99 [16:43:07< 5:16:51] +[titan] 2025-07-10 06:18:12,550 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:18:13,341 - root - INFO - step: 76000 loss: 16.3439 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.8626 +[titan] 2025-07-10 06:18:13,342 - root - INFO - lr: 6.7295e-05 gnorm: 0.92 [16:43:10< 5:16:47] +[titan] 2025-07-10 06:18:17,248 - root - INFO - step: 76005 loss: 16.3758 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8659 +[titan] 2025-07-10 06:18:17,249 - root - INFO - lr: 6.7280e-05 gnorm: 0.94 [16:43:14< 5:16:43] +[titan] 2025-07-10 06:18:21,151 - root - INFO - step: 76010 loss: 16.1593 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6961 +[titan] 2025-07-10 06:18:21,151 - root - INFO - lr: 6.7266e-05 gnorm: 0.97 [16:43:18< 5:16:39] +[titan] 2025-07-10 06:18:25,078 - root - INFO - step: 76015 loss: 16.2186 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.7504 +[titan] 2025-07-10 06:18:25,078 - root - INFO - lr: 6.7251e-05 gnorm: 0.90 [16:43:22< 5:16:35] +[titan] 2025-07-10 06:18:29,011 - root - INFO - step: 76020 loss: 16.3930 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8850 +[titan] 2025-07-10 06:18:29,012 - root - INFO - lr: 6.7236e-05 gnorm: 0.98 [16:43:26< 5:16:31] +[titan] 2025-07-10 06:18:32,988 - root - INFO - step: 76025 loss: 16.3265 memory: 44.58GiB(31.99%) tps: 82,418 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8290 +[titan] 2025-07-10 06:18:32,988 - root - INFO - lr: 6.7221e-05 gnorm: 1.00 [16:43:30< 5:16:27] +[titan] 2025-07-10 06:18:36,922 - root - INFO - step: 76030 loss: 16.1609 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.6823 +[titan] 2025-07-10 06:18:36,923 - root - INFO - lr: 6.7207e-05 gnorm: 0.97 [16:43:34< 5:16:23] +[titan] 2025-07-10 06:18:40,837 - root - INFO - step: 76035 loss: 16.2036 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.7386 +[titan] 2025-07-10 06:18:40,837 - root - INFO - lr: 6.7192e-05 gnorm: 0.96 [16:43:38< 5:16:19] +[titan] 2025-07-10 06:18:44,811 - root - INFO - step: 76040 loss: 16.1648 memory: 44.58GiB(31.99%) tps: 82,469 tflops: 284.61 mfu: 28.78% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.7037 +[titan] 2025-07-10 06:18:44,811 - root - INFO - lr: 6.7177e-05 gnorm: 0.95 [16:43:42< 5:16:15] +[titan] 2025-07-10 06:18:48,728 - root - INFO - step: 76045 loss: 16.4183 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.8911 +[titan] 2025-07-10 06:18:48,729 - root - INFO - lr: 6.7162e-05 gnorm: 0.92 [16:43:46< 5:16:11] +[titan] 2025-07-10 06:18:51,858 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:18:52,646 - root - INFO - step: 76050 loss: 16.2444 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 06:18:52,646 - root - INFO - lr: 6.7147e-05 gnorm: 0.99 [16:43:50< 5:16:07] +[titan] 2025-07-10 06:18:56,595 - root - INFO - step: 76055 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7936 +[titan] 2025-07-10 06:18:56,596 - root - INFO - lr: 6.7133e-05 gnorm: 0.92 [16:43:54< 5:16:04] +[titan] 2025-07-10 06:19:00,525 - root - INFO - step: 76060 loss: 16.5016 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9779 +[titan] 2025-07-10 06:19:00,525 - root - INFO - lr: 6.7118e-05 gnorm: 0.94 [16:43:58< 5:16:00] +[titan] 2025-07-10 06:19:04,453 - root - INFO - step: 76065 loss: 16.1000 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.6398 +[titan] 2025-07-10 06:19:04,453 - root - INFO - lr: 6.7103e-05 gnorm: 0.95 [16:44:02< 5:15:56] +[titan] 2025-07-10 06:19:08,368 - root - INFO - step: 76070 loss: 16.2405 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7590 +[titan] 2025-07-10 06:19:08,369 - root - INFO - lr: 6.7088e-05 gnorm: 0.92 [16:44:06< 5:15:52] +[titan] 2025-07-10 06:19:12,293 - root - INFO - step: 76075 loss: 16.2970 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8032 +[titan] 2025-07-10 06:19:12,293 - root - INFO - lr: 6.7074e-05 gnorm: 0.93 [16:44:09< 5:15:48] +[titan] 2025-07-10 06:19:16,212 - root - INFO - step: 76080 loss: 16.0559 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4508 global_avg_mtp_loss: 13.6051 +[titan] 2025-07-10 06:19:16,212 - root - INFO - lr: 6.7059e-05 gnorm: 0.95 [16:44:13< 5:15:44] +[titan] 2025-07-10 06:19:20,170 - root - INFO - step: 76085 loss: 16.3772 memory: 44.58GiB(31.99%) tps: 82,793 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8703 +[titan] 2025-07-10 06:19:20,170 - root - INFO - lr: 6.7044e-05 gnorm: 0.90 [16:44:17< 5:15:40] +[titan] 2025-07-10 06:19:24,076 - root - INFO - step: 76090 loss: 16.0377 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.5952 +[titan] 2025-07-10 06:19:24,077 - root - INFO - lr: 6.7030e-05 gnorm: 0.94 [16:44:21< 5:15:36] +[titan] 2025-07-10 06:19:27,993 - root - INFO - step: 76095 loss: 15.9726 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4356 global_avg_mtp_loss: 13.5370 +[titan] 2025-07-10 06:19:27,993 - root - INFO - lr: 6.7015e-05 gnorm: 0.92 [16:44:25< 5:15:32] +[titan] 2025-07-10 06:19:31,122 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:19:31,924 - root - INFO - step: 76100 loss: 16.3407 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.8319 +[titan] 2025-07-10 06:19:31,924 - root - INFO - lr: 6.7000e-05 gnorm: 0.92 [16:44:29< 5:15:28] +[titan] 2025-07-10 06:19:35,858 - root - INFO - step: 76105 loss: 16.0465 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4514 global_avg_mtp_loss: 13.5951 +[titan] 2025-07-10 06:19:35,858 - root - INFO - lr: 6.6985e-05 gnorm: 0.96 [16:44:33< 5:15:24] +[titan] 2025-07-10 06:19:39,776 - root - INFO - step: 76110 loss: 16.2356 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7449 +[titan] 2025-07-10 06:19:39,776 - root - INFO - lr: 6.6971e-05 gnorm: 0.99 [16:44:37< 5:15:20] +[titan] 2025-07-10 06:19:43,693 - root - INFO - step: 76115 loss: 16.1424 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.6622 +[titan] 2025-07-10 06:19:43,693 - root - INFO - lr: 6.6956e-05 gnorm: 0.95 [16:44:41< 5:15:16] +[titan] 2025-07-10 06:19:47,637 - root - INFO - step: 76120 loss: 16.2501 memory: 44.58GiB(31.99%) tps: 83,078 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 06:19:47,638 - root - INFO - lr: 6.6941e-05 gnorm: 0.99 [16:44:45< 5:15:12] +[titan] 2025-07-10 06:19:51,535 - root - INFO - step: 76125 loss: 16.0590 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.4469 global_avg_mtp_loss: 13.6121 +[titan] 2025-07-10 06:19:51,535 - root - INFO - lr: 6.6926e-05 gnorm: 0.97 [16:44:49< 5:15:08] +[titan] 2025-07-10 06:19:55,476 - root - INFO - step: 76130 loss: 16.3803 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8664 +[titan] 2025-07-10 06:19:55,476 - root - INFO - lr: 6.6912e-05 gnorm: 0.92 [16:44:53< 5:15:04] +[titan] 2025-07-10 06:19:59,376 - root - INFO - step: 76135 loss: 16.3451 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4941 global_avg_mtp_loss: 13.8510 +[titan] 2025-07-10 06:19:59,376 - root - INFO - lr: 6.6897e-05 gnorm: 0.95 [16:44:57< 5:15:00] +[titan] 2025-07-10 06:20:03,321 - root - INFO - step: 76140 loss: 15.9805 memory: 44.58GiB(31.99%) tps: 83,068 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.4400 global_avg_mtp_loss: 13.5405 +[titan] 2025-07-10 06:20:03,321 - root - INFO - lr: 6.6882e-05 gnorm: 0.95 [16:45:00< 5:14:56] +[titan] 2025-07-10 06:20:07,273 - root - INFO - step: 76145 loss: 15.9664 memory: 44.58GiB(31.99%) tps: 82,928 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.4414 global_avg_mtp_loss: 13.5250 +[titan] 2025-07-10 06:20:07,273 - root - INFO - lr: 6.6868e-05 gnorm: 0.96 [16:45:04< 5:14:52] +[titan] 2025-07-10 06:20:10,383 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:20:11,175 - root - INFO - step: 76150 loss: 16.4623 memory: 44.58GiB(31.99%) tps: 83,978 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.9426 +[titan] 2025-07-10 06:20:11,175 - root - INFO - lr: 6.6853e-05 gnorm: 0.99 [16:45:08< 5:14:48] +[titan] 2025-07-10 06:20:15,109 - root - INFO - step: 76155 loss: 16.0765 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6124 +[titan] 2025-07-10 06:20:15,109 - root - INFO - lr: 6.6838e-05 gnorm: 0.96 [16:45:12< 5:14:44] +[titan] 2025-07-10 06:20:19,018 - root - INFO - step: 76160 loss: 16.1077 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4544 global_avg_mtp_loss: 13.6533 +[titan] 2025-07-10 06:20:19,018 - root - INFO - lr: 6.6823e-05 gnorm: 0.97 [16:45:16< 5:14:40] +[titan] 2025-07-10 06:20:22,932 - root - INFO - step: 76165 loss: 16.1755 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7111 +[titan] 2025-07-10 06:20:22,932 - root - INFO - lr: 6.6809e-05 gnorm: 0.98 [16:45:20< 5:14:36] +[titan] 2025-07-10 06:20:26,847 - root - INFO - step: 76170 loss: 16.3073 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8084 +[titan] 2025-07-10 06:20:26,848 - root - INFO - lr: 6.6794e-05 gnorm: 0.97 [16:45:24< 5:14:32] +[titan] 2025-07-10 06:20:30,751 - root - INFO - step: 76175 loss: 16.3477 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8418 +[titan] 2025-07-10 06:20:30,751 - root - INFO - lr: 6.6779e-05 gnorm: 0.99 [16:45:28< 5:14:28] +[titan] 2025-07-10 06:20:34,672 - root - INFO - step: 76180 loss: 16.1769 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 06:20:34,672 - root - INFO - lr: 6.6765e-05 gnorm: 0.98 [16:45:32< 5:14:24] +[titan] 2025-07-10 06:20:38,581 - root - INFO - step: 76185 loss: 16.3881 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8893 +[titan] 2025-07-10 06:20:38,581 - root - INFO - lr: 6.6750e-05 gnorm: 0.88 [16:45:36< 5:14:20] +[titan] 2025-07-10 06:20:42,505 - root - INFO - step: 76190 loss: 16.1831 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7071 +[titan] 2025-07-10 06:20:42,505 - root - INFO - lr: 6.6735e-05 gnorm: 0.93 [16:45:40< 5:14:16] +[titan] 2025-07-10 06:20:46,414 - root - INFO - step: 76195 loss: 16.1136 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.6522 +[titan] 2025-07-10 06:20:46,414 - root - INFO - lr: 6.6721e-05 gnorm: 0.95 [16:45:44< 5:14:12] +[titan] 2025-07-10 06:20:49,538 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:20:50,326 - root - INFO - step: 76200 loss: 15.9201 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4355 global_avg_mtp_loss: 13.4847 +[titan] 2025-07-10 06:20:50,326 - root - INFO - lr: 6.6706e-05 gnorm: 0.99 [16:45:47< 5:14:08] +[titan] 2025-07-10 06:20:54,256 - root - INFO - step: 76205 loss: 16.4408 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.9310 +[titan] 2025-07-10 06:20:54,256 - root - INFO - lr: 6.6691e-05 gnorm: 0.93 [16:45:51< 5:14:04] +[titan] 2025-07-10 06:20:58,199 - root - INFO - step: 76210 loss: 16.0960 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6363 +[titan] 2025-07-10 06:20:58,199 - root - INFO - lr: 6.6677e-05 gnorm: 1.02 [16:45:55< 5:14:00] +[titan] 2025-07-10 06:21:02,142 - root - INFO - step: 76215 loss: 16.3442 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8388 +[titan] 2025-07-10 06:21:02,142 - root - INFO - lr: 6.6662e-05 gnorm: 0.95 [16:45:59< 5:13:56] +[titan] 2025-07-10 06:21:06,038 - root - INFO - step: 76220 loss: 16.2745 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 06:21:06,038 - root - INFO - lr: 6.6647e-05 gnorm: 0.99 [16:46:03< 5:13:52] +[titan] 2025-07-10 06:21:09,951 - root - INFO - step: 76225 loss: 16.1706 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.6924 +[titan] 2025-07-10 06:21:09,951 - root - INFO - lr: 6.6633e-05 gnorm: 0.98 [16:46:07< 5:13:48] +[titan] 2025-07-10 06:21:13,899 - root - INFO - step: 76230 loss: 16.3586 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8664 +[titan] 2025-07-10 06:21:13,899 - root - INFO - lr: 6.6618e-05 gnorm: 0.93 [16:46:11< 5:13:45] +[titan] 2025-07-10 06:21:17,819 - root - INFO - step: 76235 loss: 16.1394 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.6590 +[titan] 2025-07-10 06:21:17,819 - root - INFO - lr: 6.6603e-05 gnorm: 0.95 [16:46:15< 5:13:41] +[titan] 2025-07-10 06:21:21,728 - root - INFO - step: 76240 loss: 16.2064 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7230 +[titan] 2025-07-10 06:21:21,729 - root - INFO - lr: 6.6589e-05 gnorm: 1.07 [16:46:19< 5:13:37] +[titan] 2025-07-10 06:21:25,646 - root - INFO - step: 76245 loss: 16.3814 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 06:21:25,646 - root - INFO - lr: 6.6574e-05 gnorm: 1.05 [16:46:23< 5:13:33] +[titan] 2025-07-10 06:21:28,768 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:21:29,555 - root - INFO - step: 76250 loss: 16.3062 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.8144 +[titan] 2025-07-10 06:21:29,555 - root - INFO - lr: 6.6559e-05 gnorm: 0.98 [16:46:27< 5:13:29] +[titan] 2025-07-10 06:21:33,458 - root - INFO - step: 76255 loss: 16.1172 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.6566 +[titan] 2025-07-10 06:21:33,458 - root - INFO - lr: 6.6545e-05 gnorm: 0.98 [16:46:31< 5:13:25] +[titan] 2025-07-10 06:21:37,360 - root - INFO - step: 76260 loss: 16.4580 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5206 global_avg_mtp_loss: 13.9375 +[titan] 2025-07-10 06:21:37,361 - root - INFO - lr: 6.6530e-05 gnorm: 0.93 [16:46:34< 5:13:21] +[titan] 2025-07-10 06:21:41,283 - root - INFO - step: 76265 loss: 16.1332 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4557 global_avg_mtp_loss: 13.6775 +[titan] 2025-07-10 06:21:41,283 - root - INFO - lr: 6.6515e-05 gnorm: 0.92 [16:46:38< 5:13:17] +[titan] 2025-07-10 06:21:45,180 - root - INFO - step: 76270 loss: 16.2254 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7470 +[titan] 2025-07-10 06:21:45,180 - root - INFO - lr: 6.6501e-05 gnorm: 0.98 [16:46:42< 5:13:13] +[titan] 2025-07-10 06:21:49,100 - root - INFO - step: 76275 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8082 +[titan] 2025-07-10 06:21:49,101 - root - INFO - lr: 6.6486e-05 gnorm: 0.97 [16:46:46< 5:13:09] +[titan] 2025-07-10 06:21:53,026 - root - INFO - step: 76280 loss: 16.4640 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9398 +[titan] 2025-07-10 06:21:53,026 - root - INFO - lr: 6.6471e-05 gnorm: 0.91 [16:46:50< 5:13:05] +[titan] 2025-07-10 06:21:57,027 - root - INFO - step: 76285 loss: 16.4667 memory: 44.58GiB(31.99%) tps: 81,902 tflops: 282.66 mfu: 28.58% global_avg_ntp_loss: 2.5191 global_avg_mtp_loss: 13.9476 +[titan] 2025-07-10 06:21:57,028 - root - INFO - lr: 6.6457e-05 gnorm: 0.95 [16:46:54< 5:13:01] +[titan] 2025-07-10 06:21:59,558 - root - INFO - Dumping profiler traces at step 76288 +[titan] 2025-07-10 06:21:59,590 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:22:01,157 - root - INFO - step: 76290 loss: 16.1895 memory: 44.58GiB(31.99%) tps: 79,360 tflops: 273.89 mfu: 27.69% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.7104 +[titan] 2025-07-10 06:22:01,157 - root - INFO - lr: 6.6442e-05 gnorm: 0.94 [16:46:58< 5:12:57] +[titan] 2025-07-10 06:22:05,097 - root - INFO - step: 76295 loss: 16.3621 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8514 +[titan] 2025-07-10 06:22:05,097 - root - INFO - lr: 6.6427e-05 gnorm: 0.97 [16:47:02< 5:12:53] +[titan] 2025-07-10 06:22:08,263 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:22:09,057 - root - INFO - step: 76300 loss: 16.3647 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8655 +[titan] 2025-07-10 06:22:09,057 - root - INFO - lr: 6.6413e-05 gnorm: 0.99 [16:47:06< 5:12:49] +[titan] 2025-07-10 06:22:12,982 - root - INFO - step: 76305 loss: 16.5136 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9753 +[titan] 2025-07-10 06:22:12,982 - root - INFO - lr: 6.6398e-05 gnorm: 0.99 [16:47:10< 5:12:45] +[titan] 2025-07-10 06:22:16,917 - root - INFO - step: 76310 loss: 16.1794 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.7067 +[titan] 2025-07-10 06:22:16,917 - root - INFO - lr: 6.6383e-05 gnorm: 0.92 [16:47:14< 5:12:41] +[titan] 2025-07-10 06:22:20,860 - root - INFO - step: 76315 loss: 16.3932 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5065 global_avg_mtp_loss: 13.8867 +[titan] 2025-07-10 06:22:20,860 - root - INFO - lr: 6.6369e-05 gnorm: 0.95 [16:47:18< 5:12:37] +[titan] 2025-07-10 06:22:24,780 - root - INFO - step: 76320 loss: 16.3388 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.8470 +[titan] 2025-07-10 06:22:24,780 - root - INFO - lr: 6.6354e-05 gnorm: 0.99 [16:47:22< 5:12:33] +[titan] 2025-07-10 06:22:28,721 - root - INFO - step: 76325 loss: 16.1851 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7125 +[titan] 2025-07-10 06:22:28,721 - root - INFO - lr: 6.6340e-05 gnorm: 0.99 [16:47:26< 5:12:29] +[titan] 2025-07-10 06:22:32,626 - root - INFO - step: 76330 loss: 16.1439 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.6887 +[titan] 2025-07-10 06:22:32,626 - root - INFO - lr: 6.6325e-05 gnorm: 0.96 [16:47:30< 5:12:25] +[titan] 2025-07-10 06:22:36,532 - root - INFO - step: 76335 loss: 16.0395 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.5894 +[titan] 2025-07-10 06:22:36,533 - root - INFO - lr: 6.6310e-05 gnorm: 0.87 [16:47:34< 5:12:21] +[titan] 2025-07-10 06:22:40,431 - root - INFO - step: 76340 loss: 16.2210 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 06:22:40,432 - root - INFO - lr: 6.6296e-05 gnorm: 0.94 [16:47:38< 5:12:17] +[titan] 2025-07-10 06:22:44,336 - root - INFO - step: 76345 loss: 16.2453 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7727 +[titan] 2025-07-10 06:22:44,336 - root - INFO - lr: 6.6281e-05 gnorm: 0.93 [16:47:41< 5:12:13] +[titan] 2025-07-10 06:22:47,464 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:22:48,258 - root - INFO - step: 76350 loss: 16.0820 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6154 +[titan] 2025-07-10 06:22:48,258 - root - INFO - lr: 6.6267e-05 gnorm: 1.01 [16:47:45< 5:12:09] +[titan] 2025-07-10 06:22:52,200 - root - INFO - step: 76355 loss: 16.3485 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8446 +[titan] 2025-07-10 06:22:52,200 - root - INFO - lr: 6.6252e-05 gnorm: 0.98 [16:47:49< 5:12:05] +[titan] 2025-07-10 06:22:56,146 - root - INFO - step: 76360 loss: 16.4600 memory: 44.58GiB(31.99%) tps: 83,059 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9308 +[titan] 2025-07-10 06:22:56,146 - root - INFO - lr: 6.6237e-05 gnorm: 0.99 [16:47:53< 5:12:01] +[titan] 2025-07-10 06:23:00,050 - root - INFO - step: 76365 loss: 16.1499 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6863 +[titan] 2025-07-10 06:23:00,051 - root - INFO - lr: 6.6223e-05 gnorm: 0.94 [16:47:57< 5:11:57] +[titan] 2025-07-10 06:23:03,981 - root - INFO - step: 76370 loss: 16.1997 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7234 +[titan] 2025-07-10 06:23:03,981 - root - INFO - lr: 6.6208e-05 gnorm: 0.99 [16:48:01< 5:11:53] +[titan] 2025-07-10 06:23:07,894 - root - INFO - step: 76375 loss: 16.5612 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 14.0201 +[titan] 2025-07-10 06:23:07,894 - root - INFO - lr: 6.6194e-05 gnorm: 0.90 [16:48:05< 5:11:49] +[titan] 2025-07-10 06:23:11,842 - root - INFO - step: 76380 loss: 16.0856 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6329 +[titan] 2025-07-10 06:23:11,842 - root - INFO - lr: 6.6179e-05 gnorm: 0.95 [16:48:09< 5:11:45] +[titan] 2025-07-10 06:23:15,775 - root - INFO - step: 76385 loss: 16.3034 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.8079 +[titan] 2025-07-10 06:23:15,775 - root - INFO - lr: 6.6164e-05 gnorm: 0.95 [16:48:13< 5:11:41] +[titan] 2025-07-10 06:23:19,694 - root - INFO - step: 76390 loss: 15.9916 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.5603 +[titan] 2025-07-10 06:23:19,695 - root - INFO - lr: 6.6150e-05 gnorm: 0.92 [16:48:17< 5:11:38] +[titan] 2025-07-10 06:23:23,607 - root - INFO - step: 76395 loss: 16.2519 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 06:23:23,608 - root - INFO - lr: 6.6135e-05 gnorm: 0.93 [16:48:21< 5:11:34] +[titan] 2025-07-10 06:23:26,786 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:23:27,579 - root - INFO - step: 76400 loss: 16.3521 memory: 44.58GiB(31.99%) tps: 82,521 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.8264 +[titan] 2025-07-10 06:23:27,579 - root - INFO - lr: 6.6121e-05 gnorm: 1.11 [16:48:25< 5:11:30] +[titan] 2025-07-10 06:23:31,501 - root - INFO - step: 76405 loss: 16.3523 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8522 +[titan] 2025-07-10 06:23:31,502 - root - INFO - lr: 6.6106e-05 gnorm: 0.91 [16:48:29< 5:11:26] +[titan] 2025-07-10 06:23:35,459 - root - INFO - step: 76410 loss: 16.3614 memory: 44.58GiB(31.99%) tps: 82,808 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.5094 global_avg_mtp_loss: 13.8519 +[titan] 2025-07-10 06:23:35,459 - root - INFO - lr: 6.6091e-05 gnorm: 1.00 [16:48:33< 5:11:22] +[titan] 2025-07-10 06:23:39,417 - root - INFO - step: 76415 loss: 16.3819 memory: 44.58GiB(31.99%) tps: 82,800 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8828 +[titan] 2025-07-10 06:23:39,417 - root - INFO - lr: 6.6077e-05 gnorm: 0.98 [16:48:37< 5:11:18] +[titan] 2025-07-10 06:23:43,337 - root - INFO - step: 76420 loss: 16.2922 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.8054 +[titan] 2025-07-10 06:23:43,337 - root - INFO - lr: 6.6062e-05 gnorm: 0.97 [16:48:40< 5:11:14] +[titan] 2025-07-10 06:23:47,245 - root - INFO - step: 76425 loss: 16.0947 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6375 +[titan] 2025-07-10 06:23:47,245 - root - INFO - lr: 6.6048e-05 gnorm: 0.92 [16:48:44< 5:11:10] +[titan] 2025-07-10 06:23:51,188 - root - INFO - step: 76430 loss: 16.4050 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.8921 +[titan] 2025-07-10 06:23:51,188 - root - INFO - lr: 6.6033e-05 gnorm: 1.01 [16:48:48< 5:11:06] +[titan] 2025-07-10 06:23:55,148 - root - INFO - step: 76435 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.8133 +[titan] 2025-07-10 06:23:55,148 - root - INFO - lr: 6.6019e-05 gnorm: 0.91 [16:48:52< 5:11:02] +[titan] 2025-07-10 06:23:59,079 - root - INFO - step: 76440 loss: 16.1447 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6849 +[titan] 2025-07-10 06:23:59,079 - root - INFO - lr: 6.6004e-05 gnorm: 1.01 [16:48:56< 5:10:58] +[titan] 2025-07-10 06:24:02,982 - root - INFO - step: 76445 loss: 16.6488 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 14.1005 +[titan] 2025-07-10 06:24:02,983 - root - INFO - lr: 6.5989e-05 gnorm: 1.04 [16:49:00< 5:10:54] +[titan] 2025-07-10 06:24:06,117 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:24:06,904 - root - INFO - step: 76450 loss: 16.2593 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7774 +[titan] 2025-07-10 06:24:06,904 - root - INFO - lr: 6.5975e-05 gnorm: 0.95 [16:49:04< 5:10:50] +[titan] 2025-07-10 06:24:10,829 - root - INFO - step: 76455 loss: 16.3939 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.8741 +[titan] 2025-07-10 06:24:10,829 - root - INFO - lr: 6.5960e-05 gnorm: 0.97 [16:49:08< 5:10:46] +[titan] 2025-07-10 06:24:14,733 - root - INFO - step: 76460 loss: 16.0756 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6278 +[titan] 2025-07-10 06:24:14,734 - root - INFO - lr: 6.5946e-05 gnorm: 0.96 [16:49:12< 5:10:42] +[titan] 2025-07-10 06:24:18,691 - root - INFO - step: 76465 loss: 16.3339 memory: 44.58GiB(31.99%) tps: 82,800 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8418 +[titan] 2025-07-10 06:24:18,692 - root - INFO - lr: 6.5931e-05 gnorm: 0.98 [16:49:16< 5:10:38] +[titan] 2025-07-10 06:24:22,621 - root - INFO - step: 76470 loss: 15.9638 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4365 global_avg_mtp_loss: 13.5273 +[titan] 2025-07-10 06:24:22,621 - root - INFO - lr: 6.5917e-05 gnorm: 0.92 [16:49:20< 5:10:34] +[titan] 2025-07-10 06:24:26,528 - root - INFO - step: 76475 loss: 16.0456 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.5868 +[titan] 2025-07-10 06:24:26,528 - root - INFO - lr: 6.5902e-05 gnorm: 0.96 [16:49:24< 5:10:30] +[titan] 2025-07-10 06:24:30,454 - root - INFO - step: 76480 loss: 16.2238 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.7544 +[titan] 2025-07-10 06:24:30,454 - root - INFO - lr: 6.5888e-05 gnorm: 0.90 [16:49:28< 5:10:26] +[titan] 2025-07-10 06:24:34,382 - root - INFO - step: 76485 loss: 16.4351 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9150 +[titan] 2025-07-10 06:24:34,382 - root - INFO - lr: 6.5873e-05 gnorm: 0.95 [16:49:31< 5:10:22] +[titan] 2025-07-10 06:24:38,273 - root - INFO - step: 76490 loss: 16.4439 memory: 44.58GiB(31.99%) tps: 84,229 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9191 +[titan] 2025-07-10 06:24:38,273 - root - INFO - lr: 6.5859e-05 gnorm: 0.97 [16:49:35< 5:10:18] +[titan] 2025-07-10 06:24:42,183 - root - INFO - step: 76495 loss: 16.0881 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6307 +[titan] 2025-07-10 06:24:42,184 - root - INFO - lr: 6.5844e-05 gnorm: 0.97 [16:49:39< 5:10:14] +[titan] 2025-07-10 06:24:45,324 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:24:46,121 - root - INFO - step: 76500 loss: 16.2226 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.7286 +[titan] 2025-07-10 06:24:46,121 - root - INFO - lr: 6.5829e-05 gnorm: 1.00 [16:49:43< 5:10:10] +[titan] 2025-07-10 06:24:50,016 - root - INFO - step: 76505 loss: 16.2940 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.7869 +[titan] 2025-07-10 06:24:50,017 - root - INFO - lr: 6.5815e-05 gnorm: 0.91 [16:49:47< 5:10:06] +[titan] 2025-07-10 06:24:53,934 - root - INFO - step: 76510 loss: 16.2729 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7895 +[titan] 2025-07-10 06:24:53,934 - root - INFO - lr: 6.5800e-05 gnorm: 0.97 [16:49:51< 5:10:02] +[titan] 2025-07-10 06:24:57,858 - root - INFO - step: 76515 loss: 16.3225 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8317 +[titan] 2025-07-10 06:24:57,859 - root - INFO - lr: 6.5786e-05 gnorm: 0.98 [16:49:55< 5:09:58] +[titan] 2025-07-10 06:25:01,789 - root - INFO - step: 76520 loss: 16.0830 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.5998 +[titan] 2025-07-10 06:25:01,789 - root - INFO - lr: 6.5771e-05 gnorm: 1.06 [16:49:59< 5:09:54] +[titan] 2025-07-10 06:25:05,700 - root - INFO - step: 76525 loss: 16.2326 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7590 +[titan] 2025-07-10 06:25:05,700 - root - INFO - lr: 6.5757e-05 gnorm: 0.90 [16:50:03< 5:09:50] +[titan] 2025-07-10 06:25:09,616 - root - INFO - step: 76530 loss: 16.3734 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8746 +[titan] 2025-07-10 06:25:09,617 - root - INFO - lr: 6.5742e-05 gnorm: 0.96 [16:50:07< 5:09:46] +[titan] 2025-07-10 06:25:13,549 - root - INFO - step: 76535 loss: 16.1797 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7004 +[titan] 2025-07-10 06:25:13,549 - root - INFO - lr: 6.5728e-05 gnorm: 0.95 [16:50:11< 5:09:42] +[titan] 2025-07-10 06:25:17,447 - root - INFO - step: 76540 loss: 16.3296 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8316 +[titan] 2025-07-10 06:25:17,447 - root - INFO - lr: 6.5713e-05 gnorm: 0.96 [16:50:15< 5:09:38] +[titan] 2025-07-10 06:25:21,358 - root - INFO - step: 76545 loss: 16.3537 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 06:25:21,358 - root - INFO - lr: 6.5699e-05 gnorm: 0.90 [16:50:18< 5:09:34] +[titan] 2025-07-10 06:25:24,488 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:25:25,275 - root - INFO - step: 76550 loss: 15.8870 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4105 global_avg_mtp_loss: 13.4764 +[titan] 2025-07-10 06:25:25,276 - root - INFO - lr: 6.5684e-05 gnorm: 0.93 [16:50:22< 5:09:30] +[titan] 2025-07-10 06:25:29,191 - root - INFO - step: 76555 loss: 16.2739 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7868 +[titan] 2025-07-10 06:25:29,191 - root - INFO - lr: 6.5670e-05 gnorm: 0.96 [16:50:26< 5:09:26] +[titan] 2025-07-10 06:25:33,133 - root - INFO - step: 76560 loss: 16.0591 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6087 +[titan] 2025-07-10 06:25:33,133 - root - INFO - lr: 6.5655e-05 gnorm: 0.95 [16:50:30< 5:09:23] +[titan] 2025-07-10 06:25:37,048 - root - INFO - step: 76565 loss: 16.4915 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5385 global_avg_mtp_loss: 13.9530 +[titan] 2025-07-10 06:25:37,048 - root - INFO - lr: 6.5641e-05 gnorm: 1.02 [16:50:34< 5:09:19] +[titan] 2025-07-10 06:25:40,955 - root - INFO - step: 76570 loss: 16.3531 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.8507 +[titan] 2025-07-10 06:25:40,956 - root - INFO - lr: 6.5626e-05 gnorm: 0.99 [16:50:38< 5:09:15] +[titan] 2025-07-10 06:25:44,883 - root - INFO - step: 76575 loss: 16.3637 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8710 +[titan] 2025-07-10 06:25:44,883 - root - INFO - lr: 6.5612e-05 gnorm: 0.96 [16:50:42< 5:09:11] +[titan] 2025-07-10 06:25:48,807 - root - INFO - step: 76580 loss: 16.2481 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7645 +[titan] 2025-07-10 06:25:48,807 - root - INFO - lr: 6.5597e-05 gnorm: 0.94 [16:50:46< 5:09:07] +[titan] 2025-07-10 06:25:52,728 - root - INFO - step: 76585 loss: 15.9688 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4517 global_avg_mtp_loss: 13.5171 +[titan] 2025-07-10 06:25:52,728 - root - INFO - lr: 6.5583e-05 gnorm: 1.01 [16:50:50< 5:09:03] +[titan] 2025-07-10 06:25:56,643 - root - INFO - step: 76590 loss: 16.6134 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5494 global_avg_mtp_loss: 14.0640 +[titan] 2025-07-10 06:25:56,643 - root - INFO - lr: 6.5568e-05 gnorm: 0.95 [16:50:54< 5:08:59] +[titan] 2025-07-10 06:26:00,606 - root - INFO - step: 76595 loss: 16.3593 memory: 44.58GiB(31.99%) tps: 82,691 tflops: 285.38 mfu: 28.86% global_avg_ntp_loss: 2.5188 global_avg_mtp_loss: 13.8405 +[titan] 2025-07-10 06:26:00,607 - root - INFO - lr: 6.5554e-05 gnorm: 0.94 [16:50:58< 5:08:55] +[titan] 2025-07-10 06:26:03,743 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:26:04,528 - root - INFO - step: 76600 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7872 +[titan] 2025-07-10 06:26:04,529 - root - INFO - lr: 6.5539e-05 gnorm: 0.90 [16:51:02< 5:08:51] +[titan] 2025-07-10 06:26:08,542 - root - INFO - step: 76605 loss: 16.5221 memory: 44.58GiB(31.99%) tps: 81,656 tflops: 281.81 mfu: 28.49% global_avg_ntp_loss: 2.5341 global_avg_mtp_loss: 13.9879 +[titan] 2025-07-10 06:26:08,542 - root - INFO - lr: 6.5525e-05 gnorm: 0.92 [16:51:06< 5:08:47] +[titan] 2025-07-10 06:26:12,500 - root - INFO - step: 76610 loss: 16.3985 memory: 44.58GiB(31.99%) tps: 82,787 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.8871 +[titan] 2025-07-10 06:26:12,500 - root - INFO - lr: 6.5510e-05 gnorm: 0.94 [16:51:10< 5:08:43] +[titan] 2025-07-10 06:26:16,410 - root - INFO - step: 76615 loss: 16.3281 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8331 +[titan] 2025-07-10 06:26:16,410 - root - INFO - lr: 6.5496e-05 gnorm: 0.99 [16:51:13< 5:08:39] +[titan] 2025-07-10 06:26:20,312 - root - INFO - step: 76620 loss: 16.0957 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.6408 +[titan] 2025-07-10 06:26:20,313 - root - INFO - lr: 6.5481e-05 gnorm: 0.96 [16:51:17< 5:08:35] +[titan] 2025-07-10 06:26:24,210 - root - INFO - step: 76625 loss: 16.1976 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7172 +[titan] 2025-07-10 06:26:24,210 - root - INFO - lr: 6.5467e-05 gnorm: 0.92 [16:51:21< 5:08:31] +[titan] 2025-07-10 06:26:28,125 - root - INFO - step: 76630 loss: 16.2073 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7309 +[titan] 2025-07-10 06:26:28,126 - root - INFO - lr: 6.5452e-05 gnorm: 0.96 [16:51:25< 5:08:27] +[titan] 2025-07-10 06:26:32,048 - root - INFO - step: 76635 loss: 16.2840 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7973 +[titan] 2025-07-10 06:26:32,048 - root - INFO - lr: 6.5438e-05 gnorm: 0.94 [16:51:29< 5:08:23] +[titan] 2025-07-10 06:26:35,965 - root - INFO - step: 76640 loss: 15.9104 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4246 global_avg_mtp_loss: 13.4858 +[titan] 2025-07-10 06:26:35,965 - root - INFO - lr: 6.5423e-05 gnorm: 0.97 [16:51:33< 5:08:19] +[titan] 2025-07-10 06:26:39,888 - root - INFO - step: 76645 loss: 16.5435 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.5365 global_avg_mtp_loss: 14.0071 +[titan] 2025-07-10 06:26:39,888 - root - INFO - lr: 6.5409e-05 gnorm: 0.91 [16:51:37< 5:08:15] +[titan] 2025-07-10 06:26:42,994 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:26:43,787 - root - INFO - step: 76650 loss: 16.1597 memory: 44.58GiB(31.99%) tps: 84,043 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.7066 +[titan] 2025-07-10 06:26:43,787 - root - INFO - lr: 6.5395e-05 gnorm: 0.96 [16:51:41< 5:08:11] +[titan] 2025-07-10 06:26:47,682 - root - INFO - step: 76655 loss: 16.4603 memory: 44.58GiB(31.99%) tps: 84,144 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5209 global_avg_mtp_loss: 13.9394 +[titan] 2025-07-10 06:26:47,682 - root - INFO - lr: 6.5380e-05 gnorm: 0.93 [16:51:45< 5:08:07] +[titan] 2025-07-10 06:26:51,567 - root - INFO - step: 76660 loss: 16.2629 memory: 44.58GiB(31.99%) tps: 84,336 tflops: 291.06 mfu: 29.43% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7780 +[titan] 2025-07-10 06:26:51,568 - root - INFO - lr: 6.5366e-05 gnorm: 0.93 [16:51:49< 5:08:03] +[titan] 2025-07-10 06:26:55,504 - root - INFO - step: 76665 loss: 16.4054 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8944 +[titan] 2025-07-10 06:26:55,504 - root - INFO - lr: 6.5351e-05 gnorm: 0.92 [16:51:53< 5:07:59] +[titan] 2025-07-10 06:26:59,407 - root - INFO - step: 76670 loss: 16.0661 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4454 global_avg_mtp_loss: 13.6207 +[titan] 2025-07-10 06:26:59,407 - root - INFO - lr: 6.5337e-05 gnorm: 1.03 [16:51:56< 5:07:55] +[titan] 2025-07-10 06:27:03,307 - root - INFO - step: 76675 loss: 16.0409 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4499 global_avg_mtp_loss: 13.5911 +[titan] 2025-07-10 06:27:03,307 - root - INFO - lr: 6.5322e-05 gnorm: 0.98 [16:52:00< 5:07:51] +[titan] 2025-07-10 06:27:07,270 - root - INFO - step: 76680 loss: 16.2192 memory: 44.58GiB(31.99%) tps: 82,693 tflops: 285.39 mfu: 28.86% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 06:27:07,270 - root - INFO - lr: 6.5308e-05 gnorm: 0.91 [16:52:04< 5:07:47] +[titan] 2025-07-10 06:27:11,187 - root - INFO - step: 76685 loss: 16.2543 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7741 +[titan] 2025-07-10 06:27:11,188 - root - INFO - lr: 6.5293e-05 gnorm: 0.97 [16:52:08< 5:07:43] +[titan] 2025-07-10 06:27:15,120 - root - INFO - step: 76690 loss: 16.1962 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7237 +[titan] 2025-07-10 06:27:15,121 - root - INFO - lr: 6.5279e-05 gnorm: 0.92 [16:52:12< 5:07:39] +[titan] 2025-07-10 06:27:19,061 - root - INFO - step: 76695 loss: 16.2966 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.8121 +[titan] 2025-07-10 06:27:19,061 - root - INFO - lr: 6.5265e-05 gnorm: 0.91 [16:52:16< 5:07:35] +[titan] 2025-07-10 06:27:22,191 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:27:22,983 - root - INFO - step: 76700 loss: 16.3163 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.8250 +[titan] 2025-07-10 06:27:22,983 - root - INFO - lr: 6.5250e-05 gnorm: 0.93 [16:52:20< 5:07:31] +[titan] 2025-07-10 06:27:26,895 - root - INFO - step: 76705 loss: 16.3772 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.8722 +[titan] 2025-07-10 06:27:26,895 - root - INFO - lr: 6.5236e-05 gnorm: 0.98 [16:52:24< 5:07:27] +[titan] 2025-07-10 06:27:30,832 - root - INFO - step: 76710 loss: 16.2517 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 06:27:30,833 - root - INFO - lr: 6.5221e-05 gnorm: 0.96 [16:52:28< 5:07:23] +[titan] 2025-07-10 06:27:34,743 - root - INFO - step: 76715 loss: 16.0512 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4465 global_avg_mtp_loss: 13.6047 +[titan] 2025-07-10 06:27:34,743 - root - INFO - lr: 6.5207e-05 gnorm: 0.95 [16:52:32< 5:07:19] +[titan] 2025-07-10 06:27:38,659 - root - INFO - step: 76720 loss: 16.1168 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.6498 +[titan] 2025-07-10 06:27:38,660 - root - INFO - lr: 6.5192e-05 gnorm: 1.00 [16:52:36< 5:07:15] +[titan] 2025-07-10 06:27:42,594 - root - INFO - step: 76725 loss: 16.3382 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8404 +[titan] 2025-07-10 06:27:42,594 - root - INFO - lr: 6.5178e-05 gnorm: 1.01 [16:52:40< 5:07:11] +[titan] 2025-07-10 06:27:46,501 - root - INFO - step: 76730 loss: 16.4313 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9078 +[titan] 2025-07-10 06:27:46,501 - root - INFO - lr: 6.5164e-05 gnorm: 1.04 [16:52:44< 5:07:07] +[titan] 2025-07-10 06:27:50,411 - root - INFO - step: 76735 loss: 16.1450 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6873 +[titan] 2025-07-10 06:27:50,411 - root - INFO - lr: 6.5149e-05 gnorm: 0.94 [16:52:47< 5:07:04] +[titan] 2025-07-10 06:27:54,317 - root - INFO - step: 76740 loss: 16.2929 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.8057 +[titan] 2025-07-10 06:27:54,317 - root - INFO - lr: 6.5135e-05 gnorm: 1.03 [16:52:51< 5:07:00] +[titan] 2025-07-10 06:27:58,226 - root - INFO - step: 76745 loss: 16.3283 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8239 +[titan] 2025-07-10 06:27:58,227 - root - INFO - lr: 6.5120e-05 gnorm: 0.97 [16:52:55< 5:06:56] +[titan] 2025-07-10 06:28:01,330 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:28:02,119 - root - INFO - step: 76750 loss: 16.1635 memory: 44.58GiB(31.99%) tps: 84,187 tflops: 290.54 mfu: 29.38% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6943 +[titan] 2025-07-10 06:28:02,119 - root - INFO - lr: 6.5106e-05 gnorm: 1.03 [16:52:59< 5:06:52] +[titan] 2025-07-10 06:28:06,045 - root - INFO - step: 76755 loss: 15.9936 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5539 +[titan] 2025-07-10 06:28:06,045 - root - INFO - lr: 6.5091e-05 gnorm: 0.97 [16:53:03< 5:06:48] +[titan] 2025-07-10 06:28:09,957 - root - INFO - step: 76760 loss: 16.0890 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6324 +[titan] 2025-07-10 06:28:09,958 - root - INFO - lr: 6.5077e-05 gnorm: 0.91 [16:53:07< 5:06:44] +[titan] 2025-07-10 06:28:13,861 - root - INFO - step: 76765 loss: 16.2130 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7249 +[titan] 2025-07-10 06:28:13,861 - root - INFO - lr: 6.5063e-05 gnorm: 1.01 [16:53:11< 5:06:40] +[titan] 2025-07-10 06:28:17,776 - root - INFO - step: 76770 loss: 16.1821 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.7151 +[titan] 2025-07-10 06:28:17,776 - root - INFO - lr: 6.5048e-05 gnorm: 0.91 [16:53:15< 5:06:36] +[titan] 2025-07-10 06:28:21,689 - root - INFO - step: 76775 loss: 16.5156 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9819 +[titan] 2025-07-10 06:28:21,690 - root - INFO - lr: 6.5034e-05 gnorm: 0.93 [16:53:19< 5:06:32] +[titan] 2025-07-10 06:28:25,593 - root - INFO - step: 76780 loss: 16.2051 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7239 +[titan] 2025-07-10 06:28:25,594 - root - INFO - lr: 6.5019e-05 gnorm: 0.90 [16:53:23< 5:06:28] +[titan] 2025-07-10 06:28:29,499 - root - INFO - step: 76785 loss: 15.9419 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4246 global_avg_mtp_loss: 13.5173 +[titan] 2025-07-10 06:28:29,499 - root - INFO - lr: 6.5005e-05 gnorm: 0.95 [16:53:27< 5:06:24] +[titan] 2025-07-10 06:28:33,439 - root - INFO - step: 76790 loss: 15.9701 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4291 global_avg_mtp_loss: 13.5410 +[titan] 2025-07-10 06:28:33,439 - root - INFO - lr: 6.4991e-05 gnorm: 0.89 [16:53:31< 5:06:20] +[titan] 2025-07-10 06:28:37,384 - root - INFO - step: 76795 loss: 16.2916 memory: 44.58GiB(31.99%) tps: 83,063 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8017 +[titan] 2025-07-10 06:28:37,384 - root - INFO - lr: 6.4976e-05 gnorm: 0.97 [16:53:34< 5:06:16] +[titan] 2025-07-10 06:28:40,593 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:28:41,389 - root - INFO - step: 76800 loss: 16.1099 memory: 44.58GiB(31.99%) tps: 81,829 tflops: 282.40 mfu: 28.55% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6510 +[titan] 2025-07-10 06:28:41,389 - root - INFO - lr: 6.4962e-05 gnorm: 1.00 [16:53:38< 5:06:12] +[titan] 2025-07-10 06:28:41,543 - root - INFO - Dumping profiler traces at step 76800 +[titan] 2025-07-10 06:28:41,575 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:28:45,499 - root - INFO - step: 76805 loss: 16.3863 memory: 44.58GiB(31.99%) tps: 79,740 tflops: 275.20 mfu: 27.83% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.8632 +[titan] 2025-07-10 06:28:45,499 - root - INFO - lr: 6.4948e-05 gnorm: 1.01 [16:53:43< 5:06:08] +[titan] 2025-07-10 06:28:49,423 - root - INFO - step: 76810 loss: 16.4496 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9265 +[titan] 2025-07-10 06:28:49,423 - root - INFO - lr: 6.4933e-05 gnorm: 0.96 [16:53:46< 5:06:04] +[titan] 2025-07-10 06:28:53,342 - root - INFO - step: 76815 loss: 16.0325 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.5798 +[titan] 2025-07-10 06:28:53,343 - root - INFO - lr: 6.4919e-05 gnorm: 1.08 [16:53:50< 5:06:00] +[titan] 2025-07-10 06:28:57,276 - root - INFO - step: 76820 loss: 16.1683 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.6845 +[titan] 2025-07-10 06:28:57,276 - root - INFO - lr: 6.4904e-05 gnorm: 0.96 [16:53:54< 5:05:56] +[titan] 2025-07-10 06:29:01,192 - root - INFO - step: 76825 loss: 16.5127 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9803 +[titan] 2025-07-10 06:29:01,193 - root - INFO - lr: 6.4890e-05 gnorm: 0.94 [16:53:58< 5:05:52] +[titan] 2025-07-10 06:29:05,160 - root - INFO - step: 76830 loss: 16.1657 memory: 44.58GiB(31.99%) tps: 82,591 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.6851 +[titan] 2025-07-10 06:29:05,161 - root - INFO - lr: 6.4876e-05 gnorm: 0.92 [16:54:02< 5:05:48] +[titan] 2025-07-10 06:29:09,100 - root - INFO - step: 76835 loss: 16.4317 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9076 +[titan] 2025-07-10 06:29:09,100 - root - INFO - lr: 6.4861e-05 gnorm: 0.94 [16:54:06< 5:05:44] +[titan] 2025-07-10 06:29:13,015 - root - INFO - step: 76840 loss: 16.2792 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7806 +[titan] 2025-07-10 06:29:13,016 - root - INFO - lr: 6.4847e-05 gnorm: 1.02 [16:54:10< 5:05:40] +[titan] 2025-07-10 06:29:16,960 - root - INFO - step: 76845 loss: 15.9863 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.5534 +[titan] 2025-07-10 06:29:16,960 - root - INFO - lr: 6.4833e-05 gnorm: 1.11 [16:54:14< 5:05:36] +[titan] 2025-07-10 06:29:20,085 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:29:20,881 - root - INFO - step: 76850 loss: 15.9125 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.17% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.4787 +[titan] 2025-07-10 06:29:20,881 - root - INFO - lr: 6.4818e-05 gnorm: 0.97 [16:54:18< 5:05:32] +[titan] 2025-07-10 06:29:24,797 - root - INFO - step: 76855 loss: 16.1025 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.6535 +[titan] 2025-07-10 06:29:24,797 - root - INFO - lr: 6.4804e-05 gnorm: 0.98 [16:54:22< 5:05:28] +[titan] 2025-07-10 06:29:28,739 - root - INFO - step: 76860 loss: 16.2596 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7785 +[titan] 2025-07-10 06:29:28,740 - root - INFO - lr: 6.4790e-05 gnorm: 0.98 [16:54:26< 5:05:24] +[titan] 2025-07-10 06:29:32,659 - root - INFO - step: 76865 loss: 16.4844 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.5351 global_avg_mtp_loss: 13.9493 +[titan] 2025-07-10 06:29:32,659 - root - INFO - lr: 6.4775e-05 gnorm: 0.97 [16:54:30< 5:05:20] +[titan] 2025-07-10 06:29:36,561 - root - INFO - step: 76870 loss: 16.5632 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5711 global_avg_mtp_loss: 13.9922 +[titan] 2025-07-10 06:29:36,561 - root - INFO - lr: 6.4761e-05 gnorm: 1.02 [16:54:34< 5:05:16] +[titan] 2025-07-10 06:29:40,488 - root - INFO - step: 76875 loss: 16.3535 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-10 06:29:40,488 - root - INFO - lr: 6.4746e-05 gnorm: 0.95 [16:54:38< 5:05:12] +[titan] 2025-07-10 06:29:44,403 - root - INFO - step: 76880 loss: 16.0982 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.6521 +[titan] 2025-07-10 06:29:44,403 - root - INFO - lr: 6.4732e-05 gnorm: 0.95 [16:54:41< 5:05:08] +[titan] 2025-07-10 06:29:48,323 - root - INFO - step: 76885 loss: 16.1501 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6926 +[titan] 2025-07-10 06:29:48,323 - root - INFO - lr: 6.4718e-05 gnorm: 0.96 [16:54:45< 5:05:04] +[titan] 2025-07-10 06:29:52,251 - root - INFO - step: 76890 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.8234 +[titan] 2025-07-10 06:29:52,251 - root - INFO - lr: 6.4703e-05 gnorm: 0.92 [16:54:49< 5:05:00] +[titan] 2025-07-10 06:29:56,182 - root - INFO - step: 76895 loss: 16.4878 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.9646 +[titan] 2025-07-10 06:29:56,183 - root - INFO - lr: 6.4689e-05 gnorm: 0.92 [16:54:53< 5:04:57] +[titan] 2025-07-10 06:29:59,310 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:30:00,110 - root - INFO - step: 76900 loss: 16.3534 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8452 +[titan] 2025-07-10 06:30:00,110 - root - INFO - lr: 6.4675e-05 gnorm: 0.94 [16:54:57< 5:04:53] +[titan] 2025-07-10 06:30:04,030 - root - INFO - step: 76905 loss: 16.2103 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7340 +[titan] 2025-07-10 06:30:04,030 - root - INFO - lr: 6.4660e-05 gnorm: 0.95 [16:55:01< 5:04:49] +[titan] 2025-07-10 06:30:07,956 - root - INFO - step: 76910 loss: 16.4210 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.9062 +[titan] 2025-07-10 06:30:07,956 - root - INFO - lr: 6.4646e-05 gnorm: 0.92 [16:55:05< 5:04:45] +[titan] 2025-07-10 06:30:11,932 - root - INFO - step: 76915 loss: 16.3382 memory: 44.58GiB(31.99%) tps: 82,418 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8428 +[titan] 2025-07-10 06:30:11,933 - root - INFO - lr: 6.4632e-05 gnorm: 0.94 [16:55:09< 5:04:41] +[titan] 2025-07-10 06:30:15,871 - root - INFO - step: 76920 loss: 16.5781 memory: 44.58GiB(31.99%) tps: 83,199 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5412 global_avg_mtp_loss: 14.0369 +[titan] 2025-07-10 06:30:15,871 - root - INFO - lr: 6.4617e-05 gnorm: 0.93 [16:55:13< 5:04:37] +[titan] 2025-07-10 06:30:19,819 - root - INFO - step: 76925 loss: 15.6210 memory: 44.58GiB(31.99%) tps: 83,010 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.3911 global_avg_mtp_loss: 13.2300 +[titan] 2025-07-10 06:30:19,819 - root - INFO - lr: 6.4603e-05 gnorm: 0.98 [16:55:17< 5:04:33] +[titan] 2025-07-10 06:30:23,720 - root - INFO - step: 76930 loss: 16.3889 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8720 +[titan] 2025-07-10 06:30:23,721 - root - INFO - lr: 6.4589e-05 gnorm: 0.94 [16:55:21< 5:04:29] +[titan] 2025-07-10 06:30:27,636 - root - INFO - step: 76935 loss: 16.3897 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8724 +[titan] 2025-07-10 06:30:27,637 - root - INFO - lr: 6.4575e-05 gnorm: 0.91 [16:55:25< 5:04:25] +[titan] 2025-07-10 06:30:31,552 - root - INFO - step: 76940 loss: 16.1438 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6762 +[titan] 2025-07-10 06:30:31,553 - root - INFO - lr: 6.4560e-05 gnorm: 0.97 [16:55:29< 5:04:21] +[titan] 2025-07-10 06:30:35,509 - root - INFO - step: 76945 loss: 16.0665 memory: 44.58GiB(31.99%) tps: 82,835 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6135 +[titan] 2025-07-10 06:30:35,509 - root - INFO - lr: 6.4546e-05 gnorm: 0.98 [16:55:33< 5:04:17] +[titan] 2025-07-10 06:30:38,641 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:30:39,441 - root - INFO - step: 76950 loss: 16.4581 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5241 global_avg_mtp_loss: 13.9340 +[titan] 2025-07-10 06:30:39,441 - root - INFO - lr: 6.4532e-05 gnorm: 0.96 [16:55:36< 5:04:13] +[titan] 2025-07-10 06:30:43,365 - root - INFO - step: 76955 loss: 16.3663 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-10 06:30:43,366 - root - INFO - lr: 6.4517e-05 gnorm: 0.96 [16:55:40< 5:04:09] +[titan] 2025-07-10 06:30:47,310 - root - INFO - step: 76960 loss: 16.5825 memory: 44.58GiB(31.99%) tps: 83,082 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5454 global_avg_mtp_loss: 14.0371 +[titan] 2025-07-10 06:30:47,310 - root - INFO - lr: 6.4503e-05 gnorm: 0.93 [16:55:44< 5:04:05] +[titan] 2025-07-10 06:30:51,216 - root - INFO - step: 76965 loss: 16.2070 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.7461 +[titan] 2025-07-10 06:30:51,216 - root - INFO - lr: 6.4489e-05 gnorm: 1.04 [16:55:48< 5:04:01] +[titan] 2025-07-10 06:30:55,123 - root - INFO - step: 76970 loss: 16.2788 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7958 +[titan] 2025-07-10 06:30:55,123 - root - INFO - lr: 6.4474e-05 gnorm: 0.99 [16:55:52< 5:03:57] +[titan] 2025-07-10 06:30:59,034 - root - INFO - step: 76975 loss: 16.4609 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5174 global_avg_mtp_loss: 13.9434 +[titan] 2025-07-10 06:30:59,034 - root - INFO - lr: 6.4460e-05 gnorm: 0.89 [16:55:56< 5:03:53] +[titan] 2025-07-10 06:31:02,944 - root - INFO - step: 76980 loss: 16.1966 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4663 global_avg_mtp_loss: 13.7302 +[titan] 2025-07-10 06:31:02,944 - root - INFO - lr: 6.4446e-05 gnorm: 0.91 [16:56:00< 5:03:49] +[titan] 2025-07-10 06:31:06,884 - root - INFO - step: 76985 loss: 16.2831 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.7988 +[titan] 2025-07-10 06:31:06,884 - root - INFO - lr: 6.4431e-05 gnorm: 0.92 [16:56:04< 5:03:45] +[titan] 2025-07-10 06:31:10,817 - root - INFO - step: 76990 loss: 16.1353 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6664 +[titan] 2025-07-10 06:31:10,817 - root - INFO - lr: 6.4417e-05 gnorm: 0.97 [16:56:08< 5:03:41] +[titan] 2025-07-10 06:31:14,755 - root - INFO - step: 76995 loss: 16.0807 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.6346 +[titan] 2025-07-10 06:31:14,755 - root - INFO - lr: 6.4403e-05 gnorm: 0.97 [16:56:12< 5:03:37] +[titan] 2025-07-10 06:31:17,918 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:31:18,715 - root - INFO - step: 77000 loss: 16.2007 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7188 +[titan] 2025-07-10 06:31:18,715 - root - INFO - lr: 6.4389e-05 gnorm: 0.95 [16:56:16< 5:03:33] +[titan] 2025-07-10 06:31:22,664 - root - INFO - step: 77005 loss: 16.1244 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6613 +[titan] 2025-07-10 06:31:22,664 - root - INFO - lr: 6.4374e-05 gnorm: 0.99 [16:56:20< 5:03:29] +[titan] 2025-07-10 06:31:26,605 - root - INFO - step: 77010 loss: 15.8903 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4311 global_avg_mtp_loss: 13.4592 +[titan] 2025-07-10 06:31:26,605 - root - INFO - lr: 6.4360e-05 gnorm: 1.00 [16:56:24< 5:03:25] +[titan] 2025-07-10 06:31:30,522 - root - INFO - step: 77015 loss: 16.3253 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.8347 +[titan] 2025-07-10 06:31:30,523 - root - INFO - lr: 6.4346e-05 gnorm: 0.94 [16:56:28< 5:03:21] +[titan] 2025-07-10 06:31:34,457 - root - INFO - step: 77020 loss: 16.2096 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7355 +[titan] 2025-07-10 06:31:34,458 - root - INFO - lr: 6.4332e-05 gnorm: 0.96 [16:56:32< 5:03:17] +[titan] 2025-07-10 06:31:38,382 - root - INFO - step: 77025 loss: 16.3700 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8572 +[titan] 2025-07-10 06:31:38,383 - root - INFO - lr: 6.4317e-05 gnorm: 0.94 [16:56:35< 5:03:13] +[titan] 2025-07-10 06:31:42,329 - root - INFO - step: 77030 loss: 15.8770 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4154 global_avg_mtp_loss: 13.4617 +[titan] 2025-07-10 06:31:42,329 - root - INFO - lr: 6.4303e-05 gnorm: 0.97 [16:56:39< 5:03:09] +[titan] 2025-07-10 06:31:46,309 - root - INFO - step: 77035 loss: 16.4407 memory: 44.58GiB(31.99%) tps: 82,332 tflops: 284.14 mfu: 28.73% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.9306 +[titan] 2025-07-10 06:31:46,310 - root - INFO - lr: 6.4289e-05 gnorm: 0.94 [16:56:43< 5:03:05] +[titan] 2025-07-10 06:31:50,221 - root - INFO - step: 77040 loss: 16.2708 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 06:31:50,222 - root - INFO - lr: 6.4274e-05 gnorm: 0.98 [16:56:47< 5:03:01] +[titan] 2025-07-10 06:31:54,112 - root - INFO - step: 77045 loss: 16.1857 memory: 44.58GiB(31.99%) tps: 84,235 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7133 +[titan] 2025-07-10 06:31:54,112 - root - INFO - lr: 6.4260e-05 gnorm: 0.94 [16:56:51< 5:02:57] +[titan] 2025-07-10 06:31:57,217 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:31:58,007 - root - INFO - step: 77050 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.8535 +[titan] 2025-07-10 06:31:58,008 - root - INFO - lr: 6.4246e-05 gnorm: 0.97 [16:56:55< 5:02:54] +[titan] 2025-07-10 06:32:01,974 - root - INFO - step: 77055 loss: 16.1957 memory: 44.58GiB(31.99%) tps: 82,612 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7112 +[titan] 2025-07-10 06:32:01,975 - root - INFO - lr: 6.4232e-05 gnorm: 0.98 [16:56:59< 5:02:50] +[titan] 2025-07-10 06:32:05,883 - root - INFO - step: 77060 loss: 15.9777 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4423 global_avg_mtp_loss: 13.5355 +[titan] 2025-07-10 06:32:05,884 - root - INFO - lr: 6.4217e-05 gnorm: 0.99 [16:57:03< 5:02:46] +[titan] 2025-07-10 06:32:09,794 - root - INFO - step: 77065 loss: 16.0558 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4409 global_avg_mtp_loss: 13.6148 +[titan] 2025-07-10 06:32:09,794 - root - INFO - lr: 6.4203e-05 gnorm: 0.97 [16:57:07< 5:02:42] +[titan] 2025-07-10 06:32:13,710 - root - INFO - step: 77070 loss: 16.3355 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8438 +[titan] 2025-07-10 06:32:13,710 - root - INFO - lr: 6.4189e-05 gnorm: 0.93 [16:57:11< 5:02:38] +[titan] 2025-07-10 06:32:17,617 - root - INFO - step: 77075 loss: 16.4128 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.9033 +[titan] 2025-07-10 06:32:17,617 - root - INFO - lr: 6.4175e-05 gnorm: 1.00 [16:57:15< 5:02:34] +[titan] 2025-07-10 06:32:21,528 - root - INFO - step: 77080 loss: 16.1945 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.7250 +[titan] 2025-07-10 06:32:21,528 - root - INFO - lr: 6.4160e-05 gnorm: 0.94 [16:57:19< 5:02:30] +[titan] 2025-07-10 06:32:25,440 - root - INFO - step: 77085 loss: 16.6008 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5725 global_avg_mtp_loss: 14.0283 +[titan] 2025-07-10 06:32:25,440 - root - INFO - lr: 6.4146e-05 gnorm: 0.93 [16:57:22< 5:02:26] +[titan] 2025-07-10 06:32:29,359 - root - INFO - step: 77090 loss: 16.4540 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9202 +[titan] 2025-07-10 06:32:29,359 - root - INFO - lr: 6.4132e-05 gnorm: 0.92 [16:57:26< 5:02:22] +[titan] 2025-07-10 06:32:33,254 - root - INFO - step: 77095 loss: 16.2760 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.7784 +[titan] 2025-07-10 06:32:33,254 - root - INFO - lr: 6.4118e-05 gnorm: 0.97 [16:57:30< 5:02:18] +[titan] 2025-07-10 06:32:36,393 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:32:37,185 - root - INFO - step: 77100 loss: 16.2147 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7457 +[titan] 2025-07-10 06:32:37,185 - root - INFO - lr: 6.4104e-05 gnorm: 0.99 [16:57:34< 5:02:14] +[titan] 2025-07-10 06:32:41,099 - root - INFO - step: 77105 loss: 16.3550 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.8647 +[titan] 2025-07-10 06:32:41,099 - root - INFO - lr: 6.4089e-05 gnorm: 0.99 [16:57:38< 5:02:10] +[titan] 2025-07-10 06:32:45,042 - root - INFO - step: 77110 loss: 16.2720 memory: 44.58GiB(31.99%) tps: 83,109 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.7887 +[titan] 2025-07-10 06:32:45,043 - root - INFO - lr: 6.4075e-05 gnorm: 1.00 [16:57:42< 5:02:06] +[titan] 2025-07-10 06:32:48,961 - root - INFO - step: 77115 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8096 +[titan] 2025-07-10 06:32:48,962 - root - INFO - lr: 6.4061e-05 gnorm: 0.92 [16:57:46< 5:02:02] +[titan] 2025-07-10 06:32:52,885 - root - INFO - step: 77120 loss: 16.4326 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.5092 global_avg_mtp_loss: 13.9234 +[titan] 2025-07-10 06:32:52,885 - root - INFO - lr: 6.4047e-05 gnorm: 0.91 [16:57:50< 5:01:58] +[titan] 2025-07-10 06:32:56,794 - root - INFO - step: 77125 loss: 16.0246 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4349 global_avg_mtp_loss: 13.5898 +[titan] 2025-07-10 06:32:56,794 - root - INFO - lr: 6.4032e-05 gnorm: 0.93 [16:57:54< 5:01:54] +[titan] 2025-07-10 06:33:00,737 - root - INFO - step: 77130 loss: 16.2976 memory: 44.58GiB(31.99%) tps: 83,126 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.8040 +[titan] 2025-07-10 06:33:00,737 - root - INFO - lr: 6.4018e-05 gnorm: 0.96 [16:57:58< 5:01:50] +[titan] 2025-07-10 06:33:04,641 - root - INFO - step: 77135 loss: 16.4113 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.8997 +[titan] 2025-07-10 06:33:04,641 - root - INFO - lr: 6.4004e-05 gnorm: 0.95 [16:58:02< 5:01:46] +[titan] 2025-07-10 06:33:08,570 - root - INFO - step: 77140 loss: 16.3843 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8801 +[titan] 2025-07-10 06:33:08,570 - root - INFO - lr: 6.3990e-05 gnorm: 0.94 [16:58:06< 5:01:42] +[titan] 2025-07-10 06:33:12,483 - root - INFO - step: 77145 loss: 16.4403 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9126 +[titan] 2025-07-10 06:33:12,484 - root - INFO - lr: 6.3976e-05 gnorm: 0.97 [16:58:10< 5:01:38] +[titan] 2025-07-10 06:33:15,612 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:33:16,408 - root - INFO - step: 77150 loss: 16.3689 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.8562 +[titan] 2025-07-10 06:33:16,409 - root - INFO - lr: 6.3961e-05 gnorm: 0.95 [16:58:13< 5:01:34] +[titan] 2025-07-10 06:33:20,336 - root - INFO - step: 77155 loss: 16.4499 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9304 +[titan] 2025-07-10 06:33:20,336 - root - INFO - lr: 6.3947e-05 gnorm: 0.94 [16:58:17< 5:01:30] +[titan] 2025-07-10 06:33:24,235 - root - INFO - step: 77160 loss: 16.1018 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6446 +[titan] 2025-07-10 06:33:24,236 - root - INFO - lr: 6.3933e-05 gnorm: 0.94 [16:58:21< 5:01:26] +[titan] 2025-07-10 06:33:28,162 - root - INFO - step: 77165 loss: 16.1890 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7135 +[titan] 2025-07-10 06:33:28,163 - root - INFO - lr: 6.3919e-05 gnorm: 0.93 [16:58:25< 5:01:22] +[titan] 2025-07-10 06:33:32,075 - root - INFO - step: 77170 loss: 16.1278 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.6624 +[titan] 2025-07-10 06:33:32,075 - root - INFO - lr: 6.3905e-05 gnorm: 0.99 [16:58:29< 5:01:18] +[titan] 2025-07-10 06:33:35,984 - root - INFO - step: 77175 loss: 16.1565 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.7035 +[titan] 2025-07-10 06:33:35,985 - root - INFO - lr: 6.3890e-05 gnorm: 0.95 [16:58:33< 5:01:14] +[titan] 2025-07-10 06:33:39,915 - root - INFO - step: 77180 loss: 16.1775 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.7134 +[titan] 2025-07-10 06:33:39,915 - root - INFO - lr: 6.3876e-05 gnorm: 0.93 [16:58:37< 5:01:10] +[titan] 2025-07-10 06:33:43,840 - root - INFO - step: 77185 loss: 16.1455 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 06:33:43,841 - root - INFO - lr: 6.3862e-05 gnorm: 0.90 [16:58:41< 5:01:06] +[titan] 2025-07-10 06:33:47,759 - root - INFO - step: 77190 loss: 16.2740 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4959 global_avg_mtp_loss: 13.7781 +[titan] 2025-07-10 06:33:47,759 - root - INFO - lr: 6.3848e-05 gnorm: 0.93 [16:58:45< 5:01:02] +[titan] 2025-07-10 06:33:51,652 - root - INFO - step: 77195 loss: 16.4774 memory: 44.58GiB(31.99%) tps: 84,179 tflops: 290.52 mfu: 29.37% global_avg_ntp_loss: 2.5319 global_avg_mtp_loss: 13.9454 +[titan] 2025-07-10 06:33:51,652 - root - INFO - lr: 6.3834e-05 gnorm: 0.98 [16:58:49< 5:00:58] +[titan] 2025-07-10 06:33:54,804 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:33:55,593 - root - INFO - step: 77200 loss: 16.5306 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 14.0108 +[titan] 2025-07-10 06:33:55,593 - root - INFO - lr: 6.3819e-05 gnorm: 0.91 [16:58:53< 5:00:54] +[titan] 2025-07-10 06:33:59,492 - root - INFO - step: 77205 loss: 15.6933 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.3939 global_avg_mtp_loss: 13.2994 +[titan] 2025-07-10 06:33:59,492 - root - INFO - lr: 6.3805e-05 gnorm: 0.96 [16:58:57< 5:00:50] +[titan] 2025-07-10 06:34:03,370 - root - INFO - step: 77210 loss: 16.2098 memory: 44.58GiB(31.99%) tps: 84,502 tflops: 291.63 mfu: 29.49% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7290 +[titan] 2025-07-10 06:34:03,370 - root - INFO - lr: 6.3791e-05 gnorm: 0.98 [16:59:00< 5:00:46] +[titan] 2025-07-10 06:34:07,277 - root - INFO - step: 77215 loss: 16.3226 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8237 +[titan] 2025-07-10 06:34:07,278 - root - INFO - lr: 6.3777e-05 gnorm: 0.93 [16:59:04< 5:00:42] +[titan] 2025-07-10 06:34:11,175 - root - INFO - step: 77220 loss: 16.4294 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.9172 +[titan] 2025-07-10 06:34:11,175 - root - INFO - lr: 6.3763e-05 gnorm: 0.99 [16:59:08< 5:00:38] +[titan] 2025-07-10 06:34:15,103 - root - INFO - step: 77225 loss: 16.2247 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7462 +[titan] 2025-07-10 06:34:15,103 - root - INFO - lr: 6.3749e-05 gnorm: 0.98 [16:59:12< 5:00:34] +[titan] 2025-07-10 06:34:19,038 - root - INFO - step: 77230 loss: 16.4733 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.06% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9393 +[titan] 2025-07-10 06:34:19,039 - root - INFO - lr: 6.3734e-05 gnorm: 0.99 [16:59:16< 5:00:31] +[titan] 2025-07-10 06:34:22,950 - root - INFO - step: 77235 loss: 16.2968 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.7925 +[titan] 2025-07-10 06:34:22,950 - root - INFO - lr: 6.3720e-05 gnorm: 0.94 [16:59:20< 5:00:27] +[titan] 2025-07-10 06:34:26,892 - root - INFO - step: 77240 loss: 16.2508 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7744 +[titan] 2025-07-10 06:34:26,892 - root - INFO - lr: 6.3706e-05 gnorm: 0.95 [16:59:24< 5:00:23] +[titan] 2025-07-10 06:34:30,815 - root - INFO - step: 77245 loss: 16.2591 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7691 +[titan] 2025-07-10 06:34:30,816 - root - INFO - lr: 6.3692e-05 gnorm: 0.94 [16:59:28< 5:00:19] +[titan] 2025-07-10 06:34:33,951 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:34:34,751 - root - INFO - step: 77250 loss: 16.6499 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.5624 global_avg_mtp_loss: 14.0875 +[titan] 2025-07-10 06:34:34,752 - root - INFO - lr: 6.3678e-05 gnorm: 0.93 [16:59:32< 5:00:15] +[titan] 2025-07-10 06:34:38,681 - root - INFO - step: 77255 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.8776 +[titan] 2025-07-10 06:34:38,681 - root - INFO - lr: 6.3664e-05 gnorm: 0.96 [16:59:36< 5:00:11] +[titan] 2025-07-10 06:34:42,610 - root - INFO - step: 77260 loss: 16.3384 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8333 +[titan] 2025-07-10 06:34:42,610 - root - INFO - lr: 6.3649e-05 gnorm: 0.97 [16:59:40< 5:00:07] +[titan] 2025-07-10 06:34:46,520 - root - INFO - step: 77265 loss: 16.5432 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 14.0028 +[titan] 2025-07-10 06:34:46,521 - root - INFO - lr: 6.3635e-05 gnorm: 0.92 [16:59:44< 5:00:03] +[titan] 2025-07-10 06:34:50,409 - root - INFO - step: 77270 loss: 16.7319 memory: 44.58GiB(31.99%) tps: 84,267 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.5754 global_avg_mtp_loss: 14.1564 +[titan] 2025-07-10 06:34:50,410 - root - INFO - lr: 6.3621e-05 gnorm: 0.92 [16:59:47< 4:59:59] +[titan] 2025-07-10 06:34:54,332 - root - INFO - step: 77275 loss: 16.4571 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9368 +[titan] 2025-07-10 06:34:54,332 - root - INFO - lr: 6.3607e-05 gnorm: 0.96 [16:59:51< 4:59:55] +[titan] 2025-07-10 06:34:58,242 - root - INFO - step: 77280 loss: 16.2267 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7547 +[titan] 2025-07-10 06:34:58,242 - root - INFO - lr: 6.3593e-05 gnorm: 1.00 [16:59:55< 4:59:51] +[titan] 2025-07-10 06:35:02,181 - root - INFO - step: 77285 loss: 15.9121 memory: 44.58GiB(31.99%) tps: 83,190 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.4239 global_avg_mtp_loss: 13.4881 +[titan] 2025-07-10 06:35:02,181 - root - INFO - lr: 6.3579e-05 gnorm: 0.96 [16:59:59< 4:59:47] +[titan] 2025-07-10 06:35:06,098 - root - INFO - step: 77290 loss: 16.1960 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.7286 +[titan] 2025-07-10 06:35:06,098 - root - INFO - lr: 6.3565e-05 gnorm: 0.94 [17:00:03< 4:59:43] +[titan] 2025-07-10 06:35:10,005 - root - INFO - step: 77295 loss: 16.2663 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.7727 +[titan] 2025-07-10 06:35:10,005 - root - INFO - lr: 6.3550e-05 gnorm: 0.97 [17:00:07< 4:59:39] +[titan] 2025-07-10 06:35:13,125 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:35:13,910 - root - INFO - step: 77300 loss: 16.4346 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9101 +[titan] 2025-07-10 06:35:13,910 - root - INFO - lr: 6.3536e-05 gnorm: 1.01 [17:00:11< 4:59:35] +[titan] 2025-07-10 06:35:17,817 - root - INFO - step: 77305 loss: 16.3095 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.7994 +[titan] 2025-07-10 06:35:17,817 - root - INFO - lr: 6.3522e-05 gnorm: 0.93 [17:00:15< 4:59:31] +[titan] 2025-07-10 06:35:21,792 - root - INFO - step: 77310 loss: 16.0221 memory: 44.58GiB(31.99%) tps: 82,438 tflops: 284.51 mfu: 28.77% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.5702 +[titan] 2025-07-10 06:35:21,792 - root - INFO - lr: 6.3508e-05 gnorm: 0.96 [17:00:19< 4:59:27] +[titan] 2025-07-10 06:35:23,501 - root - INFO - Dumping profiler traces at step 77312 +[titan] 2025-07-10 06:35:23,534 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:35:25,898 - root - INFO - step: 77315 loss: 16.2221 memory: 44.58GiB(31.99%) tps: 79,822 tflops: 275.48 mfu: 27.85% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7412 +[titan] 2025-07-10 06:35:25,898 - root - INFO - lr: 6.3494e-05 gnorm: 0.98 [17:00:23< 4:59:23] +[titan] 2025-07-10 06:35:29,828 - root - INFO - step: 77320 loss: 16.3132 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8179 +[titan] 2025-07-10 06:35:29,828 - root - INFO - lr: 6.3480e-05 gnorm: 0.94 [17:00:27< 4:59:19] +[titan] 2025-07-10 06:35:33,743 - root - INFO - step: 77325 loss: 16.4746 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.9519 +[titan] 2025-07-10 06:35:33,743 - root - INFO - lr: 6.3466e-05 gnorm: 0.99 [17:00:31< 4:59:15] +[titan] 2025-07-10 06:35:37,654 - root - INFO - step: 77330 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7982 +[titan] 2025-07-10 06:35:37,655 - root - INFO - lr: 6.3452e-05 gnorm: 1.01 [17:00:35< 4:59:11] +[titan] 2025-07-10 06:35:41,571 - root - INFO - step: 77335 loss: 16.3680 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8698 +[titan] 2025-07-10 06:35:41,571 - root - INFO - lr: 6.3437e-05 gnorm: 0.97 [17:00:39< 4:59:07] +[titan] 2025-07-10 06:35:45,542 - root - INFO - step: 77340 loss: 16.2542 memory: 44.58GiB(31.99%) tps: 82,521 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.7608 +[titan] 2025-07-10 06:35:45,542 - root - INFO - lr: 6.3423e-05 gnorm: 1.05 [17:00:43< 4:59:03] +[titan] 2025-07-10 06:35:49,457 - root - INFO - step: 77345 loss: 16.1531 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4574 global_avg_mtp_loss: 13.6957 +[titan] 2025-07-10 06:35:49,457 - root - INFO - lr: 6.3409e-05 gnorm: 0.99 [17:00:46< 4:58:59] +[titan] 2025-07-10 06:35:52,588 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:35:53,379 - root - INFO - step: 77350 loss: 16.3073 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.7945 +[titan] 2025-07-10 06:35:53,379 - root - INFO - lr: 6.3395e-05 gnorm: 0.94 [17:00:50< 4:58:55] +[titan] 2025-07-10 06:35:57,295 - root - INFO - step: 77355 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 06:35:57,296 - root - INFO - lr: 6.3381e-05 gnorm: 0.94 [17:00:54< 4:58:51] +[titan] 2025-07-10 06:36:01,207 - root - INFO - step: 77360 loss: 16.1147 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6489 +[titan] 2025-07-10 06:36:01,207 - root - INFO - lr: 6.3367e-05 gnorm: 0.96 [17:00:58< 4:58:47] +[titan] 2025-07-10 06:36:05,153 - root - INFO - step: 77365 loss: 16.5089 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9824 +[titan] 2025-07-10 06:36:05,153 - root - INFO - lr: 6.3353e-05 gnorm: 0.94 [17:01:02< 4:58:43] +[titan] 2025-07-10 06:36:09,069 - root - INFO - step: 77370 loss: 16.0666 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.6055 +[titan] 2025-07-10 06:36:09,069 - root - INFO - lr: 6.3339e-05 gnorm: 0.99 [17:01:06< 4:58:39] +[titan] 2025-07-10 06:36:12,978 - root - INFO - step: 77375 loss: 16.2414 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7617 +[titan] 2025-07-10 06:36:12,978 - root - INFO - lr: 6.3325e-05 gnorm: 0.95 [17:01:10< 4:58:35] +[titan] 2025-07-10 06:36:16,880 - root - INFO - step: 77380 loss: 16.3236 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 06:36:16,881 - root - INFO - lr: 6.3311e-05 gnorm: 0.98 [17:01:14< 4:58:31] +[titan] 2025-07-10 06:36:20,791 - root - INFO - step: 77385 loss: 16.2737 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7964 +[titan] 2025-07-10 06:36:20,792 - root - INFO - lr: 6.3296e-05 gnorm: 0.90 [17:01:18< 4:58:27] +[titan] 2025-07-10 06:36:24,692 - root - INFO - step: 77390 loss: 15.7730 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4043 global_avg_mtp_loss: 13.3687 +[titan] 2025-07-10 06:36:24,692 - root - INFO - lr: 6.3282e-05 gnorm: 0.96 [17:01:22< 4:58:24] +[titan] 2025-07-10 06:36:28,614 - root - INFO - step: 77395 loss: 16.1444 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.6674 +[titan] 2025-07-10 06:36:28,614 - root - INFO - lr: 6.3268e-05 gnorm: 0.95 [17:01:26< 4:58:20] +[titan] 2025-07-10 06:36:31,732 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:36:32,527 - root - INFO - step: 77400 loss: 16.1117 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4517 global_avg_mtp_loss: 13.6599 +[titan] 2025-07-10 06:36:32,527 - root - INFO - lr: 6.3254e-05 gnorm: 0.97 [17:01:30< 4:58:16] +[titan] 2025-07-10 06:36:36,463 - root - INFO - step: 77405 loss: 16.2933 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-10 06:36:36,463 - root - INFO - lr: 6.3240e-05 gnorm: 0.95 [17:01:33< 4:58:12] +[titan] 2025-07-10 06:36:40,394 - root - INFO - step: 77410 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.8159 +[titan] 2025-07-10 06:36:40,395 - root - INFO - lr: 6.3226e-05 gnorm: 0.98 [17:01:37< 4:58:08] +[titan] 2025-07-10 06:36:44,300 - root - INFO - step: 77415 loss: 16.5597 memory: 44.58GiB(31.99%) tps: 83,908 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5318 global_avg_mtp_loss: 14.0280 +[titan] 2025-07-10 06:36:44,300 - root - INFO - lr: 6.3212e-05 gnorm: 0.94 [17:01:41< 4:58:04] +[titan] 2025-07-10 06:36:48,193 - root - INFO - step: 77420 loss: 16.1951 memory: 44.58GiB(31.99%) tps: 84,174 tflops: 290.50 mfu: 29.37% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7183 +[titan] 2025-07-10 06:36:48,193 - root - INFO - lr: 6.3198e-05 gnorm: 0.93 [17:01:45< 4:58:00] +[titan] 2025-07-10 06:36:52,088 - root - INFO - step: 77425 loss: 16.3977 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8868 +[titan] 2025-07-10 06:36:52,088 - root - INFO - lr: 6.3184e-05 gnorm: 0.95 [17:01:49< 4:57:56] +[titan] 2025-07-10 06:36:55,979 - root - INFO - step: 77430 loss: 16.1231 memory: 44.58GiB(31.99%) tps: 84,213 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.6552 +[titan] 2025-07-10 06:36:55,980 - root - INFO - lr: 6.3170e-05 gnorm: 0.94 [17:01:53< 4:57:52] +[titan] 2025-07-10 06:36:59,931 - root - INFO - step: 77435 loss: 16.1613 memory: 44.58GiB(31.99%) tps: 82,919 tflops: 286.17 mfu: 28.93% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6973 +[titan] 2025-07-10 06:36:59,932 - root - INFO - lr: 6.3156e-05 gnorm: 0.92 [17:01:57< 4:57:48] +[titan] 2025-07-10 06:37:03,819 - root - INFO - step: 77440 loss: 16.2390 memory: 44.58GiB(31.99%) tps: 84,296 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7567 +[titan] 2025-07-10 06:37:03,819 - root - INFO - lr: 6.3142e-05 gnorm: 0.98 [17:02:01< 4:57:44] +[titan] 2025-07-10 06:37:07,751 - root - INFO - step: 77445 loss: 16.2352 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7494 +[titan] 2025-07-10 06:37:07,751 - root - INFO - lr: 6.3128e-05 gnorm: 0.93 [17:02:05< 4:57:40] +[titan] 2025-07-10 06:37:10,856 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:37:11,645 - root - INFO - step: 77450 loss: 16.2162 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.7488 +[titan] 2025-07-10 06:37:11,645 - root - INFO - lr: 6.3114e-05 gnorm: 0.99 [17:02:09< 4:57:36] +[titan] 2025-07-10 06:37:15,595 - root - INFO - step: 77455 loss: 16.3825 memory: 44.58GiB(31.99%) tps: 82,961 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8774 +[titan] 2025-07-10 06:37:15,595 - root - INFO - lr: 6.3100e-05 gnorm: 0.98 [17:02:13< 4:57:32] +[titan] 2025-07-10 06:37:19,490 - root - INFO - step: 77460 loss: 16.2154 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7381 +[titan] 2025-07-10 06:37:19,490 - root - INFO - lr: 6.3085e-05 gnorm: 0.93 [17:02:16< 4:57:28] +[titan] 2025-07-10 06:37:23,388 - root - INFO - step: 77465 loss: 16.0782 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4607 global_avg_mtp_loss: 13.6175 +[titan] 2025-07-10 06:37:23,388 - root - INFO - lr: 6.3071e-05 gnorm: 0.99 [17:02:20< 4:57:24] +[titan] 2025-07-10 06:37:27,331 - root - INFO - step: 77470 loss: 16.2255 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7301 +[titan] 2025-07-10 06:37:27,331 - root - INFO - lr: 6.3057e-05 gnorm: 1.00 [17:02:24< 4:57:20] +[titan] 2025-07-10 06:37:31,255 - root - INFO - step: 77475 loss: 16.1045 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4522 global_avg_mtp_loss: 13.6523 +[titan] 2025-07-10 06:37:31,255 - root - INFO - lr: 6.3043e-05 gnorm: 0.97 [17:02:28< 4:57:16] +[titan] 2025-07-10 06:37:35,158 - root - INFO - step: 77480 loss: 16.3609 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8707 +[titan] 2025-07-10 06:37:35,159 - root - INFO - lr: 6.3029e-05 gnorm: 0.95 [17:02:32< 4:57:12] +[titan] 2025-07-10 06:37:39,091 - root - INFO - step: 77485 loss: 16.2520 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7721 +[titan] 2025-07-10 06:37:39,091 - root - INFO - lr: 6.3015e-05 gnorm: 0.93 [17:02:36< 4:57:08] +[titan] 2025-07-10 06:37:43,007 - root - INFO - step: 77490 loss: 16.1333 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 06:37:43,007 - root - INFO - lr: 6.3001e-05 gnorm: 0.96 [17:02:40< 4:57:04] +[titan] 2025-07-10 06:37:46,931 - root - INFO - step: 77495 loss: 16.3103 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8154 +[titan] 2025-07-10 06:37:46,931 - root - INFO - lr: 6.2987e-05 gnorm: 0.94 [17:02:44< 4:57:00] +[titan] 2025-07-10 06:37:50,112 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:37:50,900 - root - INFO - step: 77500 loss: 15.8724 memory: 44.58GiB(31.99%) tps: 82,561 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.4291 global_avg_mtp_loss: 13.4433 +[titan] 2025-07-10 06:37:50,900 - root - INFO - lr: 6.2973e-05 gnorm: 0.94 [17:02:48< 4:56:56] +[titan] 2025-07-10 06:37:54,841 - root - INFO - step: 77505 loss: 16.0375 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5883 +[titan] 2025-07-10 06:37:54,841 - root - INFO - lr: 6.2959e-05 gnorm: 0.95 [17:02:52< 4:56:52] +[titan] 2025-07-10 06:37:58,753 - root - INFO - step: 77510 loss: 16.1782 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.7168 +[titan] 2025-07-10 06:37:58,754 - root - INFO - lr: 6.2945e-05 gnorm: 0.93 [17:02:56< 4:56:48] +[titan] 2025-07-10 06:38:02,692 - root - INFO - step: 77515 loss: 16.0343 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4384 global_avg_mtp_loss: 13.5958 +[titan] 2025-07-10 06:38:02,692 - root - INFO - lr: 6.2931e-05 gnorm: 0.97 [17:03:00< 4:56:44] +[titan] 2025-07-10 06:38:06,634 - root - INFO - step: 77520 loss: 16.3833 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8823 +[titan] 2025-07-10 06:38:06,635 - root - INFO - lr: 6.2917e-05 gnorm: 0.97 [17:03:04< 4:56:40] +[titan] 2025-07-10 06:38:10,570 - root - INFO - step: 77525 loss: 16.3097 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.8255 +[titan] 2025-07-10 06:38:10,571 - root - INFO - lr: 6.2903e-05 gnorm: 0.92 [17:03:08< 4:56:36] +[titan] 2025-07-10 06:38:14,530 - root - INFO - step: 77530 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 82,762 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8187 +[titan] 2025-07-10 06:38:14,530 - root - INFO - lr: 6.2889e-05 gnorm: 0.90 [17:03:12< 4:56:32] +[titan] 2025-07-10 06:38:18,446 - root - INFO - step: 77535 loss: 16.1483 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6773 +[titan] 2025-07-10 06:38:18,446 - root - INFO - lr: 6.2875e-05 gnorm: 1.01 [17:03:15< 4:56:28] +[titan] 2025-07-10 06:38:22,384 - root - INFO - step: 77540 loss: 16.2167 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7354 +[titan] 2025-07-10 06:38:22,384 - root - INFO - lr: 6.2861e-05 gnorm: 0.97 [17:03:19< 4:56:24] +[titan] 2025-07-10 06:38:26,312 - root - INFO - step: 77545 loss: 16.5378 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 14.0048 +[titan] 2025-07-10 06:38:26,312 - root - INFO - lr: 6.2847e-05 gnorm: 0.95 [17:03:23< 4:56:20] +[titan] 2025-07-10 06:38:29,461 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:38:30,247 - root - INFO - step: 77550 loss: 16.2355 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7544 +[titan] 2025-07-10 06:38:30,247 - root - INFO - lr: 6.2833e-05 gnorm: 0.94 [17:03:27< 4:56:16] +[titan] 2025-07-10 06:38:34,162 - root - INFO - step: 77555 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8354 +[titan] 2025-07-10 06:38:34,163 - root - INFO - lr: 6.2819e-05 gnorm: 0.95 [17:03:31< 4:56:12] +[titan] 2025-07-10 06:38:38,108 - root - INFO - step: 77560 loss: 16.2328 memory: 44.58GiB(31.99%) tps: 83,055 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7538 +[titan] 2025-07-10 06:38:38,108 - root - INFO - lr: 6.2805e-05 gnorm: 1.00 [17:03:35< 4:56:09] +[titan] 2025-07-10 06:38:42,026 - root - INFO - step: 77565 loss: 16.3701 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.8488 +[titan] 2025-07-10 06:38:42,026 - root - INFO - lr: 6.2791e-05 gnorm: 1.07 [17:03:39< 4:56:05] +[titan] 2025-07-10 06:38:45,959 - root - INFO - step: 77570 loss: 16.1655 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.7125 +[titan] 2025-07-10 06:38:45,959 - root - INFO - lr: 6.2777e-05 gnorm: 0.92 [17:03:43< 4:56:01] +[titan] 2025-07-10 06:38:49,878 - root - INFO - step: 77575 loss: 16.1390 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6749 +[titan] 2025-07-10 06:38:49,878 - root - INFO - lr: 6.2763e-05 gnorm: 0.94 [17:03:47< 4:55:57] +[titan] 2025-07-10 06:38:53,787 - root - INFO - step: 77580 loss: 16.1092 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6520 +[titan] 2025-07-10 06:38:53,788 - root - INFO - lr: 6.2749e-05 gnorm: 0.96 [17:03:51< 4:55:53] +[titan] 2025-07-10 06:38:57,697 - root - INFO - step: 77585 loss: 16.2105 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7368 +[titan] 2025-07-10 06:38:57,697 - root - INFO - lr: 6.2735e-05 gnorm: 0.98 [17:03:55< 4:55:49] +[titan] 2025-07-10 06:39:01,612 - root - INFO - step: 77590 loss: 16.0350 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4446 global_avg_mtp_loss: 13.5903 +[titan] 2025-07-10 06:39:01,612 - root - INFO - lr: 6.2721e-05 gnorm: 0.99 [17:03:59< 4:55:45] +[titan] 2025-07-10 06:39:05,513 - root - INFO - step: 77595 loss: 16.1950 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.7208 +[titan] 2025-07-10 06:39:05,514 - root - INFO - lr: 6.2707e-05 gnorm: 0.99 [17:04:03< 4:55:41] +[titan] 2025-07-10 06:39:08,665 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:39:09,454 - root - INFO - step: 77600 loss: 16.3027 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 06:39:09,454 - root - INFO - lr: 6.2693e-05 gnorm: 0.94 [17:04:06< 4:55:37] +[titan] 2025-07-10 06:39:13,371 - root - INFO - step: 77605 loss: 16.1846 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7054 +[titan] 2025-07-10 06:39:13,372 - root - INFO - lr: 6.2679e-05 gnorm: 0.99 [17:04:10< 4:55:33] +[titan] 2025-07-10 06:39:17,281 - root - INFO - step: 77610 loss: 16.4675 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5274 global_avg_mtp_loss: 13.9400 +[titan] 2025-07-10 06:39:17,282 - root - INFO - lr: 6.2665e-05 gnorm: 0.92 [17:04:14< 4:55:29] +[titan] 2025-07-10 06:39:21,198 - root - INFO - step: 77615 loss: 16.1999 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4703 global_avg_mtp_loss: 13.7296 +[titan] 2025-07-10 06:39:21,198 - root - INFO - lr: 6.2651e-05 gnorm: 0.90 [17:04:18< 4:55:25] +[titan] 2025-07-10 06:39:25,114 - root - INFO - step: 77620 loss: 16.2496 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7760 +[titan] 2025-07-10 06:39:25,115 - root - INFO - lr: 6.2637e-05 gnorm: 0.95 [17:04:22< 4:55:21] +[titan] 2025-07-10 06:39:29,042 - root - INFO - step: 77625 loss: 16.1073 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.6263 +[titan] 2025-07-10 06:39:29,042 - root - INFO - lr: 6.2623e-05 gnorm: 0.98 [17:04:26< 4:55:17] +[titan] 2025-07-10 06:39:32,955 - root - INFO - step: 77630 loss: 16.1195 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6546 +[titan] 2025-07-10 06:39:32,955 - root - INFO - lr: 6.2609e-05 gnorm: 1.02 [17:04:30< 4:55:13] +[titan] 2025-07-10 06:39:36,870 - root - INFO - step: 77635 loss: 16.1625 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.6978 +[titan] 2025-07-10 06:39:36,870 - root - INFO - lr: 6.2595e-05 gnorm: 0.93 [17:04:34< 4:55:09] +[titan] 2025-07-10 06:39:40,825 - root - INFO - step: 77640 loss: 16.2306 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7547 +[titan] 2025-07-10 06:39:40,826 - root - INFO - lr: 6.2581e-05 gnorm: 0.95 [17:04:38< 4:55:05] +[titan] 2025-07-10 06:39:44,770 - root - INFO - step: 77645 loss: 16.2694 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7895 +[titan] 2025-07-10 06:39:44,771 - root - INFO - lr: 6.2567e-05 gnorm: 0.97 [17:04:42< 4:55:01] +[titan] 2025-07-10 06:39:47,901 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:39:48,683 - root - INFO - step: 77650 loss: 16.4500 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.9311 +[titan] 2025-07-10 06:39:48,684 - root - INFO - lr: 6.2554e-05 gnorm: 0.95 [17:04:46< 4:54:57] +[titan] 2025-07-10 06:39:52,587 - root - INFO - step: 77655 loss: 16.1098 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6481 +[titan] 2025-07-10 06:39:52,588 - root - INFO - lr: 6.2540e-05 gnorm: 0.99 [17:04:50< 4:54:53] +[titan] 2025-07-10 06:39:56,492 - root - INFO - step: 77660 loss: 16.3534 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8506 +[titan] 2025-07-10 06:39:56,492 - root - INFO - lr: 6.2526e-05 gnorm: 0.98 [17:04:53< 4:54:49] +[titan] 2025-07-10 06:40:00,407 - root - INFO - step: 77665 loss: 16.1307 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6684 +[titan] 2025-07-10 06:40:00,407 - root - INFO - lr: 6.2512e-05 gnorm: 0.94 [17:04:57< 4:54:45] +[titan] 2025-07-10 06:40:04,309 - root - INFO - step: 77670 loss: 16.0593 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.6118 +[titan] 2025-07-10 06:40:04,309 - root - INFO - lr: 6.2498e-05 gnorm: 0.99 [17:05:01< 4:54:41] +[titan] 2025-07-10 06:40:08,217 - root - INFO - step: 77675 loss: 16.2068 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7299 +[titan] 2025-07-10 06:40:08,218 - root - INFO - lr: 6.2484e-05 gnorm: 0.98 [17:05:05< 4:54:37] +[titan] 2025-07-10 06:40:12,142 - root - INFO - step: 77680 loss: 16.4353 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.9187 +[titan] 2025-07-10 06:40:12,142 - root - INFO - lr: 6.2470e-05 gnorm: 1.03 [17:05:09< 4:54:33] +[titan] 2025-07-10 06:40:16,071 - root - INFO - step: 77685 loss: 16.1453 memory: 44.58GiB(31.99%) tps: 83,410 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.6776 +[titan] 2025-07-10 06:40:16,071 - root - INFO - lr: 6.2456e-05 gnorm: 0.96 [17:05:13< 4:54:29] +[titan] 2025-07-10 06:40:20,020 - root - INFO - step: 77690 loss: 16.1232 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4625 global_avg_mtp_loss: 13.6607 +[titan] 2025-07-10 06:40:20,021 - root - INFO - lr: 6.2442e-05 gnorm: 0.96 [17:05:17< 4:54:25] +[titan] 2025-07-10 06:40:23,923 - root - INFO - step: 77695 loss: 16.1706 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.7163 +[titan] 2025-07-10 06:40:23,923 - root - INFO - lr: 6.2428e-05 gnorm: 0.96 [17:05:21< 4:54:21] +[titan] 2025-07-10 06:40:27,050 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:40:27,837 - root - INFO - step: 77700 loss: 16.2675 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 06:40:27,838 - root - INFO - lr: 6.2414e-05 gnorm: 0.95 [17:05:25< 4:54:17] +[titan] 2025-07-10 06:40:31,725 - root - INFO - step: 77705 loss: 16.2372 memory: 44.58GiB(31.99%) tps: 84,298 tflops: 290.92 mfu: 29.42% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.7412 +[titan] 2025-07-10 06:40:31,725 - root - INFO - lr: 6.2400e-05 gnorm: 1.07 [17:05:29< 4:54:13] +[titan] 2025-07-10 06:40:35,644 - root - INFO - step: 77710 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7130 +[titan] 2025-07-10 06:40:35,644 - root - INFO - lr: 6.2386e-05 gnorm: 0.99 [17:05:33< 4:54:09] +[titan] 2025-07-10 06:40:39,547 - root - INFO - step: 77715 loss: 16.3632 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8628 +[titan] 2025-07-10 06:40:39,547 - root - INFO - lr: 6.2372e-05 gnorm: 0.95 [17:05:37< 4:54:05] +[titan] 2025-07-10 06:40:43,439 - root - INFO - step: 77720 loss: 15.7418 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.3905 global_avg_mtp_loss: 13.3514 +[titan] 2025-07-10 06:40:43,439 - root - INFO - lr: 6.2358e-05 gnorm: 0.96 [17:05:40< 4:54:01] +[titan] 2025-07-10 06:40:47,386 - root - INFO - step: 77725 loss: 16.2943 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8002 +[titan] 2025-07-10 06:40:47,386 - root - INFO - lr: 6.2345e-05 gnorm: 0.97 [17:05:44< 4:53:57] +[titan] 2025-07-10 06:40:51,316 - root - INFO - step: 77730 loss: 16.4353 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.9208 +[titan] 2025-07-10 06:40:51,316 - root - INFO - lr: 6.2331e-05 gnorm: 0.95 [17:05:48< 4:53:54] +[titan] 2025-07-10 06:40:55,229 - root - INFO - step: 77735 loss: 16.0948 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4546 global_avg_mtp_loss: 13.6402 +[titan] 2025-07-10 06:40:55,230 - root - INFO - lr: 6.2317e-05 gnorm: 0.98 [17:05:52< 4:53:50] +[titan] 2025-07-10 06:40:59,132 - root - INFO - step: 77740 loss: 16.3090 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8001 +[titan] 2025-07-10 06:40:59,133 - root - INFO - lr: 6.2303e-05 gnorm: 0.98 [17:05:56< 4:53:46] +[titan] 2025-07-10 06:41:03,050 - root - INFO - step: 77745 loss: 16.4159 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5220 global_avg_mtp_loss: 13.8940 +[titan] 2025-07-10 06:41:03,050 - root - INFO - lr: 6.2289e-05 gnorm: 0.94 [17:06:00< 4:53:42] +[titan] 2025-07-10 06:41:06,169 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:41:06,960 - root - INFO - step: 77750 loss: 16.5668 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5504 global_avg_mtp_loss: 14.0165 +[titan] 2025-07-10 06:41:06,961 - root - INFO - lr: 6.2275e-05 gnorm: 1.00 [17:06:04< 4:53:38] +[titan] 2025-07-10 06:41:10,855 - root - INFO - step: 77755 loss: 16.2682 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7791 +[titan] 2025-07-10 06:41:10,855 - root - INFO - lr: 6.2261e-05 gnorm: 0.92 [17:06:08< 4:53:34] +[titan] 2025-07-10 06:41:14,780 - root - INFO - step: 77760 loss: 16.4800 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.5254 global_avg_mtp_loss: 13.9546 +[titan] 2025-07-10 06:41:14,780 - root - INFO - lr: 6.2247e-05 gnorm: 0.94 [17:06:12< 4:53:30] +[titan] 2025-07-10 06:41:18,703 - root - INFO - step: 77765 loss: 16.3630 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8681 +[titan] 2025-07-10 06:41:18,703 - root - INFO - lr: 6.2233e-05 gnorm: 0.90 [17:06:16< 4:53:26] +[titan] 2025-07-10 06:41:22,628 - root - INFO - step: 77770 loss: 16.3065 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8112 +[titan] 2025-07-10 06:41:22,628 - root - INFO - lr: 6.2219e-05 gnorm: 0.94 [17:06:20< 4:53:22] +[titan] 2025-07-10 06:41:26,577 - root - INFO - step: 77775 loss: 16.4260 memory: 44.58GiB(31.99%) tps: 82,975 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.8987 +[titan] 2025-07-10 06:41:26,578 - root - INFO - lr: 6.2206e-05 gnorm: 0.96 [17:06:24< 4:53:18] +[titan] 2025-07-10 06:41:30,542 - root - INFO - step: 77780 loss: 16.2223 memory: 44.58GiB(31.99%) tps: 82,651 tflops: 285.24 mfu: 28.84% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7387 +[titan] 2025-07-10 06:41:30,543 - root - INFO - lr: 6.2192e-05 gnorm: 0.92 [17:06:28< 4:53:14] +[titan] 2025-07-10 06:41:34,464 - root - INFO - step: 77785 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.8151 +[titan] 2025-07-10 06:41:34,464 - root - INFO - lr: 6.2178e-05 gnorm: 0.91 [17:06:31< 4:53:10] +[titan] 2025-07-10 06:41:38,410 - root - INFO - step: 77790 loss: 16.1558 memory: 44.58GiB(31.99%) tps: 83,063 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6918 +[titan] 2025-07-10 06:41:38,410 - root - INFO - lr: 6.2164e-05 gnorm: 0.98 [17:06:35< 4:53:06] +[titan] 2025-07-10 06:41:42,342 - root - INFO - step: 77795 loss: 16.2266 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7478 +[titan] 2025-07-10 06:41:42,343 - root - INFO - lr: 6.2150e-05 gnorm: 0.97 [17:06:39< 4:53:02] +[titan] 2025-07-10 06:41:45,490 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:41:46,333 - root - INFO - step: 77800 loss: 16.3163 memory: 44.58GiB(31.99%) tps: 82,126 tflops: 283.43 mfu: 28.66% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8247 +[titan] 2025-07-10 06:41:46,333 - root - INFO - lr: 6.2136e-05 gnorm: 0.92 [17:06:43< 4:52:58] +[titan] 2025-07-10 06:41:50,328 - root - INFO - step: 77805 loss: 16.2627 memory: 44.58GiB(31.99%) tps: 82,030 tflops: 283.10 mfu: 28.62% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7720 +[titan] 2025-07-10 06:41:50,328 - root - INFO - lr: 6.2122e-05 gnorm: 0.95 [17:06:47< 4:52:54] +[titan] 2025-07-10 06:41:54,256 - root - INFO - step: 77810 loss: 16.4347 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.9134 +[titan] 2025-07-10 06:41:54,257 - root - INFO - lr: 6.2108e-05 gnorm: 0.94 [17:06:51< 4:52:50] +[titan] 2025-07-10 06:41:58,173 - root - INFO - step: 77815 loss: 16.1427 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.6760 +[titan] 2025-07-10 06:41:58,174 - root - INFO - lr: 6.2095e-05 gnorm: 1.02 [17:06:55< 4:52:46] +[titan] 2025-07-10 06:42:02,097 - root - INFO - step: 77820 loss: 16.0924 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.6164 +[titan] 2025-07-10 06:42:02,097 - root - INFO - lr: 6.2081e-05 gnorm: 0.99 [17:06:59< 4:52:42] +[titan] 2025-07-10 06:42:05,481 - root - INFO - Dumping profiler traces at step 77824 +[titan] 2025-07-10 06:42:05,513 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:42:06,306 - root - INFO - step: 77825 loss: 16.3929 memory: 44.58GiB(31.99%) tps: 77,847 tflops: 268.66 mfu: 27.17% global_avg_ntp_loss: 2.5152 global_avg_mtp_loss: 13.8777 +[titan] 2025-07-10 06:42:06,306 - root - INFO - lr: 6.2067e-05 gnorm: 1.01 [17:07:03< 4:52:38] +[titan] 2025-07-10 06:42:10,234 - root - INFO - step: 77830 loss: 16.6178 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5667 global_avg_mtp_loss: 14.0511 +[titan] 2025-07-10 06:42:10,234 - root - INFO - lr: 6.2053e-05 gnorm: 0.99 [17:07:07< 4:52:34] +[titan] 2025-07-10 06:42:14,181 - root - INFO - step: 77835 loss: 16.3452 memory: 44.58GiB(31.99%) tps: 83,016 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8495 +[titan] 2025-07-10 06:42:14,182 - root - INFO - lr: 6.2039e-05 gnorm: 1.02 [17:07:11< 4:52:30] +[titan] 2025-07-10 06:42:18,110 - root - INFO - step: 77840 loss: 16.3483 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.8588 +[titan] 2025-07-10 06:42:18,110 - root - INFO - lr: 6.2025e-05 gnorm: 0.96 [17:07:15< 4:52:26] +[titan] 2025-07-10 06:42:22,048 - root - INFO - step: 77845 loss: 16.4270 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.9210 +[titan] 2025-07-10 06:42:22,048 - root - INFO - lr: 6.2011e-05 gnorm: 0.94 [17:07:19< 4:52:22] +[titan] 2025-07-10 06:42:25,183 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:42:25,977 - root - INFO - step: 77850 loss: 16.2382 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.7445 +[titan] 2025-07-10 06:42:25,977 - root - INFO - lr: 6.1998e-05 gnorm: 0.92 [17:07:23< 4:52:18] +[titan] 2025-07-10 06:42:29,908 - root - INFO - step: 77855 loss: 16.1473 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.6871 +[titan] 2025-07-10 06:42:29,908 - root - INFO - lr: 6.1984e-05 gnorm: 0.97 [17:07:27< 4:52:14] +[titan] 2025-07-10 06:42:33,812 - root - INFO - step: 77860 loss: 16.1777 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.6954 +[titan] 2025-07-10 06:42:33,812 - root - INFO - lr: 6.1970e-05 gnorm: 0.99 [17:07:31< 4:52:10] +[titan] 2025-07-10 06:42:37,763 - root - INFO - step: 77865 loss: 16.2975 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.7949 +[titan] 2025-07-10 06:42:37,763 - root - INFO - lr: 6.1956e-05 gnorm: 1.01 [17:07:35< 4:52:06] +[titan] 2025-07-10 06:42:41,692 - root - INFO - step: 77870 loss: 16.5296 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9927 +[titan] 2025-07-10 06:42:41,693 - root - INFO - lr: 6.1942e-05 gnorm: 0.91 [17:07:39< 4:52:03] +[titan] 2025-07-10 06:42:45,663 - root - INFO - step: 77875 loss: 16.1985 memory: 44.58GiB(31.99%) tps: 82,537 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7218 +[titan] 2025-07-10 06:42:45,663 - root - INFO - lr: 6.1928e-05 gnorm: 1.01 [17:07:43< 4:51:59] +[titan] 2025-07-10 06:42:49,591 - root - INFO - step: 77880 loss: 16.1409 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6728 +[titan] 2025-07-10 06:42:49,591 - root - INFO - lr: 6.1915e-05 gnorm: 0.91 [17:07:47< 4:51:55] +[titan] 2025-07-10 06:42:53,514 - root - INFO - step: 77885 loss: 16.3636 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8624 +[titan] 2025-07-10 06:42:53,514 - root - INFO - lr: 6.1901e-05 gnorm: 0.93 [17:07:50< 4:51:51] +[titan] 2025-07-10 06:42:57,430 - root - INFO - step: 77890 loss: 16.5302 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9923 +[titan] 2025-07-10 06:42:57,430 - root - INFO - lr: 6.1887e-05 gnorm: 0.90 [17:07:54< 4:51:47] +[titan] 2025-07-10 06:43:01,349 - root - INFO - step: 77895 loss: 16.4387 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.9225 +[titan] 2025-07-10 06:43:01,349 - root - INFO - lr: 6.1873e-05 gnorm: 0.97 [17:07:58< 4:51:43] +[titan] 2025-07-10 06:43:04,496 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:43:05,291 - root - INFO - step: 77900 loss: 16.3639 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 06:43:05,291 - root - INFO - lr: 6.1859e-05 gnorm: 0.97 [17:08:02< 4:51:39] +[titan] 2025-07-10 06:43:09,215 - root - INFO - step: 77905 loss: 16.2534 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 06:43:09,215 - root - INFO - lr: 6.1845e-05 gnorm: 0.90 [17:08:06< 4:51:35] +[titan] 2025-07-10 06:43:13,135 - root - INFO - step: 77910 loss: 16.3586 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8633 +[titan] 2025-07-10 06:43:13,135 - root - INFO - lr: 6.1832e-05 gnorm: 0.90 [17:08:10< 4:51:31] +[titan] 2025-07-10 06:43:17,071 - root - INFO - step: 77915 loss: 16.3804 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5178 global_avg_mtp_loss: 13.8626 +[titan] 2025-07-10 06:43:17,071 - root - INFO - lr: 6.1818e-05 gnorm: 0.94 [17:08:14< 4:51:27] +[titan] 2025-07-10 06:43:20,982 - root - INFO - step: 77920 loss: 16.2025 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7211 +[titan] 2025-07-10 06:43:20,982 - root - INFO - lr: 6.1804e-05 gnorm: 0.97 [17:08:18< 4:51:23] +[titan] 2025-07-10 06:43:24,929 - root - INFO - step: 77925 loss: 16.1783 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7028 +[titan] 2025-07-10 06:43:24,929 - root - INFO - lr: 6.1790e-05 gnorm: 0.97 [17:08:22< 4:51:19] +[titan] 2025-07-10 06:43:28,858 - root - INFO - step: 77930 loss: 16.1877 memory: 44.58GiB(31.99%) tps: 83,415 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7212 +[titan] 2025-07-10 06:43:28,858 - root - INFO - lr: 6.1776e-05 gnorm: 0.97 [17:08:26< 4:51:15] +[titan] 2025-07-10 06:43:32,773 - root - INFO - step: 77935 loss: 16.2928 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.8022 +[titan] 2025-07-10 06:43:32,773 - root - INFO - lr: 6.1763e-05 gnorm: 0.96 [17:08:30< 4:51:11] +[titan] 2025-07-10 06:43:36,713 - root - INFO - step: 77940 loss: 16.5815 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 14.0441 +[titan] 2025-07-10 06:43:36,714 - root - INFO - lr: 6.1749e-05 gnorm: 1.02 [17:08:34< 4:51:07] +[titan] 2025-07-10 06:43:40,647 - root - INFO - step: 77945 loss: 16.3609 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8568 +[titan] 2025-07-10 06:43:40,647 - root - INFO - lr: 6.1735e-05 gnorm: 1.02 [17:08:38< 4:51:03] +[titan] 2025-07-10 06:43:43,780 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:43:44,570 - root - INFO - step: 77950 loss: 16.4668 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.9524 +[titan] 2025-07-10 06:43:44,570 - root - INFO - lr: 6.1721e-05 gnorm: 0.94 [17:08:42< 4:50:59] +[titan] 2025-07-10 06:43:48,501 - root - INFO - step: 77955 loss: 16.4507 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.5281 global_avg_mtp_loss: 13.9225 +[titan] 2025-07-10 06:43:48,501 - root - INFO - lr: 6.1707e-05 gnorm: 0.95 [17:08:45< 4:50:55] +[titan] 2025-07-10 06:43:52,422 - root - INFO - step: 77960 loss: 16.2690 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7912 +[titan] 2025-07-10 06:43:52,422 - root - INFO - lr: 6.1694e-05 gnorm: 0.94 [17:08:49< 4:50:51] +[titan] 2025-07-10 06:43:56,342 - root - INFO - step: 77965 loss: 16.3192 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 06:43:56,343 - root - INFO - lr: 6.1680e-05 gnorm: 0.95 [17:08:53< 4:50:47] +[titan] 2025-07-10 06:44:00,303 - root - INFO - step: 77970 loss: 16.3811 memory: 44.58GiB(31.99%) tps: 82,738 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8704 +[titan] 2025-07-10 06:44:00,303 - root - INFO - lr: 6.1666e-05 gnorm: 0.97 [17:08:57< 4:50:43] +[titan] 2025-07-10 06:44:04,231 - root - INFO - step: 77975 loss: 16.1501 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.6777 +[titan] 2025-07-10 06:44:04,231 - root - INFO - lr: 6.1652e-05 gnorm: 0.94 [17:09:01< 4:50:39] +[titan] 2025-07-10 06:44:08,158 - root - INFO - step: 77980 loss: 16.2463 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7647 +[titan] 2025-07-10 06:44:08,158 - root - INFO - lr: 6.1638e-05 gnorm: 0.93 [17:09:05< 4:50:35] +[titan] 2025-07-10 06:44:12,093 - root - INFO - step: 77985 loss: 16.3901 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8773 +[titan] 2025-07-10 06:44:12,094 - root - INFO - lr: 6.1625e-05 gnorm: 0.91 [17:09:09< 4:50:31] +[titan] 2025-07-10 06:44:16,026 - root - INFO - step: 77990 loss: 16.1896 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7178 +[titan] 2025-07-10 06:44:16,026 - root - INFO - lr: 6.1611e-05 gnorm: 0.96 [17:09:13< 4:50:27] +[titan] 2025-07-10 06:44:19,959 - root - INFO - step: 77995 loss: 16.0650 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4551 global_avg_mtp_loss: 13.6098 +[titan] 2025-07-10 06:44:19,959 - root - INFO - lr: 6.1597e-05 gnorm: 1.02 [17:09:17< 4:50:23] +[titan] 2025-07-10 06:44:23,087 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:44:23,884 - root - INFO - step: 78000 loss: 16.1227 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4565 global_avg_mtp_loss: 13.6662 +[titan] 2025-07-10 06:44:23,885 - root - INFO - lr: 6.1583e-05 gnorm: 0.96 [17:09:21< 4:50:19] +[titan] 2025-07-10 06:44:27,830 - root - INFO - step: 78005 loss: 16.2984 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.5015 global_avg_mtp_loss: 13.7969 +[titan] 2025-07-10 06:44:27,830 - root - INFO - lr: 6.1570e-05 gnorm: 0.92 [17:09:25< 4:50:15] +[titan] 2025-07-10 06:44:31,747 - root - INFO - step: 78010 loss: 16.1093 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6427 +[titan] 2025-07-10 06:44:31,748 - root - INFO - lr: 6.1556e-05 gnorm: 0.93 [17:09:29< 4:50:11] +[titan] 2025-07-10 06:44:35,676 - root - INFO - step: 78015 loss: 16.1955 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7249 +[titan] 2025-07-10 06:44:35,676 - root - INFO - lr: 6.1542e-05 gnorm: 0.92 [17:09:33< 4:50:07] +[titan] 2025-07-10 06:44:39,581 - root - INFO - step: 78020 loss: 16.2530 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 06:44:39,581 - root - INFO - lr: 6.1528e-05 gnorm: 0.93 [17:09:37< 4:50:03] +[titan] 2025-07-10 06:44:43,502 - root - INFO - step: 78025 loss: 16.2447 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7740 +[titan] 2025-07-10 06:44:43,503 - root - INFO - lr: 6.1515e-05 gnorm: 0.94 [17:09:40< 4:50:00] +[titan] 2025-07-10 06:44:47,457 - root - INFO - step: 78030 loss: 16.0701 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6093 +[titan] 2025-07-10 06:44:47,457 - root - INFO - lr: 6.1501e-05 gnorm: 0.96 [17:09:44< 4:49:56] +[titan] 2025-07-10 06:44:51,395 - root - INFO - step: 78035 loss: 16.6668 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5626 global_avg_mtp_loss: 14.1042 +[titan] 2025-07-10 06:44:51,395 - root - INFO - lr: 6.1487e-05 gnorm: 0.93 [17:09:48< 4:49:52] +[titan] 2025-07-10 06:44:55,335 - root - INFO - step: 78040 loss: 16.2948 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.7971 +[titan] 2025-07-10 06:44:55,335 - root - INFO - lr: 6.1473e-05 gnorm: 1.03 [17:09:52< 4:49:48] +[titan] 2025-07-10 06:44:59,229 - root - INFO - step: 78045 loss: 16.3752 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8747 +[titan] 2025-07-10 06:44:59,229 - root - INFO - lr: 6.1460e-05 gnorm: 0.94 [17:09:56< 4:49:44] +[titan] 2025-07-10 06:45:02,343 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:45:03,147 - root - INFO - step: 78050 loss: 16.1643 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.6941 +[titan] 2025-07-10 06:45:03,148 - root - INFO - lr: 6.1446e-05 gnorm: 0.97 [17:10:00< 4:49:40] +[titan] 2025-07-10 06:45:07,081 - root - INFO - step: 78055 loss: 16.1598 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6962 +[titan] 2025-07-10 06:45:07,081 - root - INFO - lr: 6.1432e-05 gnorm: 0.98 [17:10:04< 4:49:36] +[titan] 2025-07-10 06:45:11,013 - root - INFO - step: 78060 loss: 15.9908 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4197 global_avg_mtp_loss: 13.5712 +[titan] 2025-07-10 06:45:11,013 - root - INFO - lr: 6.1418e-05 gnorm: 0.99 [17:10:08< 4:49:32] +[titan] 2025-07-10 06:45:14,927 - root - INFO - step: 78065 loss: 16.3188 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8226 +[titan] 2025-07-10 06:45:14,928 - root - INFO - lr: 6.1405e-05 gnorm: 0.91 [17:10:12< 4:49:28] +[titan] 2025-07-10 06:45:18,823 - root - INFO - step: 78070 loss: 16.3358 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.8543 +[titan] 2025-07-10 06:45:18,823 - root - INFO - lr: 6.1391e-05 gnorm: 0.95 [17:10:16< 4:49:24] +[titan] 2025-07-10 06:45:22,743 - root - INFO - step: 78075 loss: 16.1354 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6750 +[titan] 2025-07-10 06:45:22,743 - root - INFO - lr: 6.1377e-05 gnorm: 0.94 [17:10:20< 4:49:20] +[titan] 2025-07-10 06:45:26,651 - root - INFO - step: 78080 loss: 16.2012 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7229 +[titan] 2025-07-10 06:45:26,652 - root - INFO - lr: 6.1363e-05 gnorm: 0.95 [17:10:24< 4:49:16] +[titan] 2025-07-10 06:45:30,567 - root - INFO - step: 78085 loss: 16.2657 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.7710 +[titan] 2025-07-10 06:45:30,567 - root - INFO - lr: 6.1350e-05 gnorm: 0.97 [17:10:28< 4:49:12] +[titan] 2025-07-10 06:45:34,511 - root - INFO - step: 78090 loss: 16.2174 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7509 +[titan] 2025-07-10 06:45:34,511 - root - INFO - lr: 6.1336e-05 gnorm: 0.95 [17:10:31< 4:49:08] +[titan] 2025-07-10 06:45:38,426 - root - INFO - step: 78095 loss: 16.3823 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8767 +[titan] 2025-07-10 06:45:38,426 - root - INFO - lr: 6.1322e-05 gnorm: 0.96 [17:10:35< 4:49:04] +[titan] 2025-07-10 06:45:41,571 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:45:42,369 - root - INFO - step: 78100 loss: 16.2990 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 06:45:42,369 - root - INFO - lr: 6.1309e-05 gnorm: 0.95 [17:10:39< 4:49:00] +[titan] 2025-07-10 06:45:46,314 - root - INFO - step: 78105 loss: 16.1528 memory: 44.58GiB(31.99%) tps: 83,071 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 06:45:46,314 - root - INFO - lr: 6.1295e-05 gnorm: 0.93 [17:10:43< 4:48:56] +[titan] 2025-07-10 06:45:50,253 - root - INFO - step: 78110 loss: 16.4127 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.9029 +[titan] 2025-07-10 06:45:50,253 - root - INFO - lr: 6.1281e-05 gnorm: 0.93 [17:10:47< 4:48:52] +[titan] 2025-07-10 06:45:54,147 - root - INFO - step: 78115 loss: 16.1214 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6727 +[titan] 2025-07-10 06:45:54,147 - root - INFO - lr: 6.1267e-05 gnorm: 0.96 [17:10:51< 4:48:48] +[titan] 2025-07-10 06:45:58,068 - root - INFO - step: 78120 loss: 16.6027 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5532 global_avg_mtp_loss: 14.0495 +[titan] 2025-07-10 06:45:58,068 - root - INFO - lr: 6.1254e-05 gnorm: 1.02 [17:10:55< 4:48:44] +[titan] 2025-07-10 06:46:01,972 - root - INFO - step: 78125 loss: 16.3832 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 06:46:01,973 - root - INFO - lr: 6.1240e-05 gnorm: 0.93 [17:10:59< 4:48:40] +[titan] 2025-07-10 06:46:05,879 - root - INFO - step: 78130 loss: 16.3507 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.8300 +[titan] 2025-07-10 06:46:05,879 - root - INFO - lr: 6.1226e-05 gnorm: 0.99 [17:11:03< 4:48:36] +[titan] 2025-07-10 06:46:09,783 - root - INFO - step: 78135 loss: 16.1136 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4579 global_avg_mtp_loss: 13.6558 +[titan] 2025-07-10 06:46:09,784 - root - INFO - lr: 6.1213e-05 gnorm: 0.97 [17:11:07< 4:48:32] +[titan] 2025-07-10 06:46:13,719 - root - INFO - step: 78140 loss: 16.0947 memory: 44.58GiB(31.99%) tps: 83,256 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.6467 +[titan] 2025-07-10 06:46:13,720 - root - INFO - lr: 6.1199e-05 gnorm: 0.94 [17:11:11< 4:48:28] +[titan] 2025-07-10 06:46:17,623 - root - INFO - step: 78145 loss: 16.2898 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.8087 +[titan] 2025-07-10 06:46:17,624 - root - INFO - lr: 6.1185e-05 gnorm: 0.93 [17:11:15< 4:48:24] +[titan] 2025-07-10 06:46:20,738 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:46:21,530 - root - INFO - step: 78150 loss: 16.1342 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.6671 +[titan] 2025-07-10 06:46:21,530 - root - INFO - lr: 6.1172e-05 gnorm: 0.95 [17:11:18< 4:48:20] +[titan] 2025-07-10 06:46:25,467 - root - INFO - step: 78155 loss: 16.6882 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.5698 global_avg_mtp_loss: 14.1184 +[titan] 2025-07-10 06:46:25,468 - root - INFO - lr: 6.1158e-05 gnorm: 0.95 [17:11:22< 4:48:16] +[titan] 2025-07-10 06:46:29,374 - root - INFO - step: 78160 loss: 16.2468 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7721 +[titan] 2025-07-10 06:46:29,374 - root - INFO - lr: 6.1144e-05 gnorm: 0.93 [17:11:26< 4:48:12] +[titan] 2025-07-10 06:46:33,285 - root - INFO - step: 78165 loss: 16.3764 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8801 +[titan] 2025-07-10 06:46:33,285 - root - INFO - lr: 6.1130e-05 gnorm: 1.06 [17:11:30< 4:48:08] +[titan] 2025-07-10 06:46:37,213 - root - INFO - step: 78170 loss: 16.4575 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5292 global_avg_mtp_loss: 13.9283 +[titan] 2025-07-10 06:46:37,213 - root - INFO - lr: 6.1117e-05 gnorm: 0.94 [17:11:34< 4:48:04] +[titan] 2025-07-10 06:46:41,158 - root - INFO - step: 78175 loss: 16.2596 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.7744 +[titan] 2025-07-10 06:46:41,158 - root - INFO - lr: 6.1103e-05 gnorm: 0.95 [17:11:38< 4:48:00] +[titan] 2025-07-10 06:46:45,104 - root - INFO - step: 78180 loss: 16.4942 memory: 44.58GiB(31.99%) tps: 83,051 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.9667 +[titan] 2025-07-10 06:46:45,104 - root - INFO - lr: 6.1089e-05 gnorm: 1.00 [17:11:42< 4:47:56] +[titan] 2025-07-10 06:46:49,044 - root - INFO - step: 78185 loss: 16.4306 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.9205 +[titan] 2025-07-10 06:46:49,045 - root - INFO - lr: 6.1076e-05 gnorm: 0.96 [17:11:46< 4:47:53] +[titan] 2025-07-10 06:46:52,998 - root - INFO - step: 78190 loss: 16.3555 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 06:46:52,999 - root - INFO - lr: 6.1062e-05 gnorm: 0.95 [17:11:50< 4:47:49] +[titan] 2025-07-10 06:46:56,921 - root - INFO - step: 78195 loss: 16.3820 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8738 +[titan] 2025-07-10 06:46:56,921 - root - INFO - lr: 6.1048e-05 gnorm: 0.90 [17:11:54< 4:47:45] +[titan] 2025-07-10 06:47:00,044 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:47:00,831 - root - INFO - step: 78200 loss: 16.1713 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.7017 +[titan] 2025-07-10 06:47:00,832 - root - INFO - lr: 6.1035e-05 gnorm: 0.99 [17:11:58< 4:47:41] +[titan] 2025-07-10 06:47:04,768 - root - INFO - step: 78205 loss: 16.2232 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7434 +[titan] 2025-07-10 06:47:04,768 - root - INFO - lr: 6.1021e-05 gnorm: 1.02 [17:12:02< 4:47:37] +[titan] 2025-07-10 06:47:08,668 - root - INFO - step: 78210 loss: 16.2001 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7223 +[titan] 2025-07-10 06:47:08,669 - root - INFO - lr: 6.1007e-05 gnorm: 1.00 [17:12:06< 4:47:33] +[titan] 2025-07-10 06:47:12,603 - root - INFO - step: 78215 loss: 16.2834 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.8044 +[titan] 2025-07-10 06:47:12,603 - root - INFO - lr: 6.0994e-05 gnorm: 0.97 [17:12:10< 4:47:29] +[titan] 2025-07-10 06:47:16,541 - root - INFO - step: 78220 loss: 16.3282 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8286 +[titan] 2025-07-10 06:47:16,541 - root - INFO - lr: 6.0980e-05 gnorm: 1.11 [17:12:13< 4:47:25] +[titan] 2025-07-10 06:47:20,476 - root - INFO - step: 78225 loss: 16.4267 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-10 06:47:20,477 - root - INFO - lr: 6.0966e-05 gnorm: 0.96 [17:12:17< 4:47:21] +[titan] 2025-07-10 06:47:24,398 - root - INFO - step: 78230 loss: 15.9928 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4267 global_avg_mtp_loss: 13.5662 +[titan] 2025-07-10 06:47:24,399 - root - INFO - lr: 6.0953e-05 gnorm: 0.93 [17:12:21< 4:47:17] +[titan] 2025-07-10 06:47:28,315 - root - INFO - step: 78235 loss: 16.2010 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7162 +[titan] 2025-07-10 06:47:28,315 - root - INFO - lr: 6.0939e-05 gnorm: 0.96 [17:12:25< 4:47:13] +[titan] 2025-07-10 06:47:32,272 - root - INFO - step: 78240 loss: 16.4852 memory: 44.58GiB(31.99%) tps: 82,815 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9538 +[titan] 2025-07-10 06:47:32,272 - root - INFO - lr: 6.0926e-05 gnorm: 0.96 [17:12:29< 4:47:09] +[titan] 2025-07-10 06:47:36,217 - root - INFO - step: 78245 loss: 16.4142 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-10 06:47:36,217 - root - INFO - lr: 6.0912e-05 gnorm: 0.92 [17:12:33< 4:47:05] +[titan] 2025-07-10 06:47:39,342 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:47:40,154 - root - INFO - step: 78250 loss: 16.1806 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7044 +[titan] 2025-07-10 06:47:40,155 - root - INFO - lr: 6.0898e-05 gnorm: 0.98 [17:12:37< 4:47:01] +[titan] 2025-07-10 06:47:44,082 - root - INFO - step: 78255 loss: 16.3631 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.8531 +[titan] 2025-07-10 06:47:44,083 - root - INFO - lr: 6.0885e-05 gnorm: 0.93 [17:12:41< 4:46:57] +[titan] 2025-07-10 06:47:47,996 - root - INFO - step: 78260 loss: 16.4561 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.9384 +[titan] 2025-07-10 06:47:47,996 - root - INFO - lr: 6.0871e-05 gnorm: 0.98 [17:12:45< 4:46:53] +[titan] 2025-07-10 06:47:51,914 - root - INFO - step: 78265 loss: 16.0866 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4571 global_avg_mtp_loss: 13.6295 +[titan] 2025-07-10 06:47:51,914 - root - INFO - lr: 6.0857e-05 gnorm: 0.92 [17:12:49< 4:46:49] +[titan] 2025-07-10 06:47:55,848 - root - INFO - step: 78270 loss: 16.2044 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7236 +[titan] 2025-07-10 06:47:55,848 - root - INFO - lr: 6.0844e-05 gnorm: 0.94 [17:12:53< 4:46:45] +[titan] 2025-07-10 06:47:59,771 - root - INFO - step: 78275 loss: 16.2549 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7696 +[titan] 2025-07-10 06:47:59,771 - root - INFO - lr: 6.0830e-05 gnorm: 0.93 [17:12:57< 4:46:41] +[titan] 2025-07-10 06:48:03,677 - root - INFO - step: 78280 loss: 16.4725 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.9511 +[titan] 2025-07-10 06:48:03,678 - root - INFO - lr: 6.0816e-05 gnorm: 0.92 [17:13:01< 4:46:37] +[titan] 2025-07-10 06:48:07,574 - root - INFO - step: 78285 loss: 15.9442 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.4226 global_avg_mtp_loss: 13.5216 +[titan] 2025-07-10 06:48:07,574 - root - INFO - lr: 6.0803e-05 gnorm: 0.98 [17:13:05< 4:46:33] +[titan] 2025-07-10 06:48:11,471 - root - INFO - step: 78290 loss: 16.1167 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.6532 +[titan] 2025-07-10 06:48:11,471 - root - INFO - lr: 6.0789e-05 gnorm: 0.98 [17:13:08< 4:46:29] +[titan] 2025-07-10 06:48:15,383 - root - INFO - step: 78295 loss: 16.2251 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7483 +[titan] 2025-07-10 06:48:15,383 - root - INFO - lr: 6.0776e-05 gnorm: 0.97 [17:13:12< 4:46:25] +[titan] 2025-07-10 06:48:18,491 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:48:19,285 - root - INFO - step: 78300 loss: 16.3973 memory: 44.58GiB(31.99%) tps: 83,990 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 06:48:19,285 - root - INFO - lr: 6.0762e-05 gnorm: 1.00 [17:13:16< 4:46:21] +[titan] 2025-07-10 06:48:23,191 - root - INFO - step: 78305 loss: 16.3129 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8192 +[titan] 2025-07-10 06:48:23,191 - root - INFO - lr: 6.0748e-05 gnorm: 1.00 [17:13:20< 4:46:17] +[titan] 2025-07-10 06:48:27,106 - root - INFO - step: 78310 loss: 16.2417 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7707 +[titan] 2025-07-10 06:48:27,106 - root - INFO - lr: 6.0735e-05 gnorm: 0.96 [17:13:24< 4:46:13] +[titan] 2025-07-10 06:48:31,057 - root - INFO - step: 78315 loss: 16.1434 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4576 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 06:48:31,057 - root - INFO - lr: 6.0721e-05 gnorm: 0.94 [17:13:28< 4:46:09] +[titan] 2025-07-10 06:48:34,965 - root - INFO - step: 78320 loss: 16.2106 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7217 +[titan] 2025-07-10 06:48:34,965 - root - INFO - lr: 6.0708e-05 gnorm: 0.98 [17:13:32< 4:46:05] +[titan] 2025-07-10 06:48:38,892 - root - INFO - step: 78325 loss: 16.1507 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.6807 +[titan] 2025-07-10 06:48:38,892 - root - INFO - lr: 6.0694e-05 gnorm: 0.93 [17:13:36< 4:46:01] +[titan] 2025-07-10 06:48:42,833 - root - INFO - step: 78330 loss: 16.2322 memory: 44.58GiB(31.99%) tps: 83,141 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7396 +[titan] 2025-07-10 06:48:42,834 - root - INFO - lr: 6.0680e-05 gnorm: 0.94 [17:13:40< 4:45:57] +[titan] 2025-07-10 06:48:46,839 - root - INFO - step: 78335 loss: 15.9276 memory: 44.58GiB(31.99%) tps: 81,802 tflops: 282.31 mfu: 28.55% global_avg_ntp_loss: 2.4213 global_avg_mtp_loss: 13.5062 +[titan] 2025-07-10 06:48:46,840 - root - INFO - lr: 6.0667e-05 gnorm: 0.97 [17:13:44< 4:45:53] +[titan] 2025-07-10 06:48:47,783 - root - INFO - Dumping profiler traces at step 78336 +[titan] 2025-07-10 06:48:47,817 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:48:50,993 - root - INFO - step: 78340 loss: 16.1773 memory: 44.58GiB(31.99%) tps: 78,898 tflops: 272.29 mfu: 27.53% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.7084 +[titan] 2025-07-10 06:48:50,993 - root - INFO - lr: 6.0653e-05 gnorm: 0.97 [17:13:48< 4:45:50] +[titan] 2025-07-10 06:48:54,901 - root - INFO - step: 78345 loss: 16.2989 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 06:48:54,902 - root - INFO - lr: 6.0640e-05 gnorm: 0.94 [17:13:52< 4:45:46] +[titan] 2025-07-10 06:48:58,014 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:48:58,800 - root - INFO - step: 78350 loss: 16.0710 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6012 +[titan] 2025-07-10 06:48:58,800 - root - INFO - lr: 6.0626e-05 gnorm: 0.95 [17:13:56< 4:45:42] +[titan] 2025-07-10 06:49:02,737 - root - INFO - step: 78355 loss: 16.2563 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.7553 +[titan] 2025-07-10 06:49:02,737 - root - INFO - lr: 6.0612e-05 gnorm: 0.94 [17:14:00< 4:45:38] +[titan] 2025-07-10 06:49:06,669 - root - INFO - step: 78360 loss: 16.1006 memory: 44.58GiB(31.99%) tps: 83,333 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4540 global_avg_mtp_loss: 13.6465 +[titan] 2025-07-10 06:49:06,670 - root - INFO - lr: 6.0599e-05 gnorm: 0.99 [17:14:04< 4:45:34] +[titan] 2025-07-10 06:49:10,601 - root - INFO - step: 78365 loss: 16.1460 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6794 +[titan] 2025-07-10 06:49:10,601 - root - INFO - lr: 6.0585e-05 gnorm: 0.95 [17:14:08< 4:45:30] +[titan] 2025-07-10 06:49:14,520 - root - INFO - step: 78370 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.9184 +[titan] 2025-07-10 06:49:14,520 - root - INFO - lr: 6.0572e-05 gnorm: 0.94 [17:14:11< 4:45:26] +[titan] 2025-07-10 06:49:18,421 - root - INFO - step: 78375 loss: 16.5065 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 13.9761 +[titan] 2025-07-10 06:49:18,421 - root - INFO - lr: 6.0558e-05 gnorm: 0.93 [17:14:15< 4:45:22] +[titan] 2025-07-10 06:49:22,350 - root - INFO - step: 78380 loss: 16.1945 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7094 +[titan] 2025-07-10 06:49:22,350 - root - INFO - lr: 6.0545e-05 gnorm: 0.93 [17:14:19< 4:45:18] +[titan] 2025-07-10 06:49:26,252 - root - INFO - step: 78385 loss: 16.2705 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7778 +[titan] 2025-07-10 06:49:26,252 - root - INFO - lr: 6.0531e-05 gnorm: 1.00 [17:14:23< 4:45:14] +[titan] 2025-07-10 06:49:30,187 - root - INFO - step: 78390 loss: 16.2740 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 06:49:30,187 - root - INFO - lr: 6.0517e-05 gnorm: 0.95 [17:14:27< 4:45:10] +[titan] 2025-07-10 06:49:34,091 - root - INFO - step: 78395 loss: 16.1659 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.6952 +[titan] 2025-07-10 06:49:34,091 - root - INFO - lr: 6.0504e-05 gnorm: 0.96 [17:14:31< 4:45:06] +[titan] 2025-07-10 06:49:37,228 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:49:38,019 - root - INFO - step: 78400 loss: 16.1121 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.6346 +[titan] 2025-07-10 06:49:38,020 - root - INFO - lr: 6.0490e-05 gnorm: 0.99 [17:14:35< 4:45:02] +[titan] 2025-07-10 06:49:41,937 - root - INFO - step: 78405 loss: 16.4660 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5218 global_avg_mtp_loss: 13.9442 +[titan] 2025-07-10 06:49:41,937 - root - INFO - lr: 6.0477e-05 gnorm: 0.95 [17:14:39< 4:44:58] +[titan] 2025-07-10 06:49:45,841 - root - INFO - step: 78410 loss: 16.3468 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8475 +[titan] 2025-07-10 06:49:45,841 - root - INFO - lr: 6.0463e-05 gnorm: 0.99 [17:14:43< 4:44:54] +[titan] 2025-07-10 06:49:49,760 - root - INFO - step: 78415 loss: 16.1617 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.6877 +[titan] 2025-07-10 06:49:49,760 - root - INFO - lr: 6.0450e-05 gnorm: 0.93 [17:14:47< 4:44:50] +[titan] 2025-07-10 06:49:53,697 - root - INFO - step: 78420 loss: 16.0564 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4474 global_avg_mtp_loss: 13.6090 +[titan] 2025-07-10 06:49:53,697 - root - INFO - lr: 6.0436e-05 gnorm: 1.02 [17:14:51< 4:44:46] +[titan] 2025-07-10 06:49:57,608 - root - INFO - step: 78425 loss: 16.3834 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5149 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 06:49:57,608 - root - INFO - lr: 6.0423e-05 gnorm: 0.94 [17:14:55< 4:44:42] +[titan] 2025-07-10 06:50:01,509 - root - INFO - step: 78430 loss: 16.1165 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6455 +[titan] 2025-07-10 06:50:01,509 - root - INFO - lr: 6.0409e-05 gnorm: 0.96 [17:14:58< 4:44:38] +[titan] 2025-07-10 06:50:05,410 - root - INFO - step: 78435 loss: 16.3946 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8824 +[titan] 2025-07-10 06:50:05,410 - root - INFO - lr: 6.0395e-05 gnorm: 0.98 [17:15:02< 4:44:34] +[titan] 2025-07-10 06:50:09,295 - root - INFO - step: 78440 loss: 16.1270 memory: 44.58GiB(31.99%) tps: 84,358 tflops: 291.13 mfu: 29.44% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.6513 +[titan] 2025-07-10 06:50:09,295 - root - INFO - lr: 6.0382e-05 gnorm: 0.92 [17:15:06< 4:44:30] +[titan] 2025-07-10 06:50:13,208 - root - INFO - step: 78445 loss: 16.2195 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.7370 +[titan] 2025-07-10 06:50:13,208 - root - INFO - lr: 6.0368e-05 gnorm: 1.01 [17:15:10< 4:44:26] +[titan] 2025-07-10 06:50:16,337 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:50:17,125 - root - INFO - step: 78450 loss: 16.3037 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8093 +[titan] 2025-07-10 06:50:17,125 - root - INFO - lr: 6.0355e-05 gnorm: 0.99 [17:15:14< 4:44:22] +[titan] 2025-07-10 06:50:21,041 - root - INFO - step: 78455 loss: 16.1363 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6745 +[titan] 2025-07-10 06:50:21,042 - root - INFO - lr: 6.0341e-05 gnorm: 1.02 [17:15:18< 4:44:18] +[titan] 2025-07-10 06:50:24,958 - root - INFO - step: 78460 loss: 16.2507 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7654 +[titan] 2025-07-10 06:50:24,958 - root - INFO - lr: 6.0328e-05 gnorm: 0.95 [17:15:22< 4:44:14] +[titan] 2025-07-10 06:50:28,845 - root - INFO - step: 78465 loss: 16.1552 memory: 44.58GiB(31.99%) tps: 84,308 tflops: 290.96 mfu: 29.42% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6934 +[titan] 2025-07-10 06:50:28,845 - root - INFO - lr: 6.0314e-05 gnorm: 1.00 [17:15:26< 4:44:10] +[titan] 2025-07-10 06:50:32,758 - root - INFO - step: 78470 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9187 +[titan] 2025-07-10 06:50:32,758 - root - INFO - lr: 6.0301e-05 gnorm: 0.93 [17:15:30< 4:44:06] +[titan] 2025-07-10 06:50:36,660 - root - INFO - step: 78475 loss: 16.2763 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7954 +[titan] 2025-07-10 06:50:36,660 - root - INFO - lr: 6.0287e-05 gnorm: 0.99 [17:15:34< 4:44:02] +[titan] 2025-07-10 06:50:40,581 - root - INFO - step: 78480 loss: 16.3342 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8380 +[titan] 2025-07-10 06:50:40,582 - root - INFO - lr: 6.0274e-05 gnorm: 0.93 [17:15:37< 4:43:58] +[titan] 2025-07-10 06:50:44,498 - root - INFO - step: 78485 loss: 16.3874 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8829 +[titan] 2025-07-10 06:50:44,498 - root - INFO - lr: 6.0260e-05 gnorm: 1.00 [17:15:41< 4:43:54] +[titan] 2025-07-10 06:50:48,436 - root - INFO - step: 78490 loss: 16.1154 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.6397 +[titan] 2025-07-10 06:50:48,436 - root - INFO - lr: 6.0247e-05 gnorm: 0.95 [17:15:45< 4:43:50] +[titan] 2025-07-10 06:50:52,344 - root - INFO - step: 78495 loss: 16.5400 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.5435 global_avg_mtp_loss: 13.9965 +[titan] 2025-07-10 06:50:52,344 - root - INFO - lr: 6.0233e-05 gnorm: 0.93 [17:15:49< 4:43:46] +[titan] 2025-07-10 06:50:55,457 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:50:56,269 - root - INFO - step: 78500 loss: 16.1618 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.6807 +[titan] 2025-07-10 06:50:56,269 - root - INFO - lr: 6.0220e-05 gnorm: 1.08 [17:15:53< 4:43:42] +[titan] 2025-07-10 06:51:00,164 - root - INFO - step: 78505 loss: 16.4432 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.9344 +[titan] 2025-07-10 06:51:00,164 - root - INFO - lr: 6.0206e-05 gnorm: 0.94 [17:15:57< 4:43:39] +[titan] 2025-07-10 06:51:04,080 - root - INFO - step: 78510 loss: 16.2882 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7980 +[titan] 2025-07-10 06:51:04,080 - root - INFO - lr: 6.0193e-05 gnorm: 0.93 [17:16:01< 4:43:35] +[titan] 2025-07-10 06:51:08,009 - root - INFO - step: 78515 loss: 16.5477 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 14.0100 +[titan] 2025-07-10 06:51:08,009 - root - INFO - lr: 6.0179e-05 gnorm: 0.95 [17:16:05< 4:43:31] +[titan] 2025-07-10 06:51:11,944 - root - INFO - step: 78520 loss: 16.3936 memory: 44.58GiB(31.99%) tps: 83,289 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8782 +[titan] 2025-07-10 06:51:11,944 - root - INFO - lr: 6.0166e-05 gnorm: 0.95 [17:16:09< 4:43:27] +[titan] 2025-07-10 06:51:15,852 - root - INFO - step: 78525 loss: 16.2547 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7650 +[titan] 2025-07-10 06:51:15,852 - root - INFO - lr: 6.0152e-05 gnorm: 0.94 [17:16:13< 4:43:23] +[titan] 2025-07-10 06:51:19,779 - root - INFO - step: 78530 loss: 16.0597 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.6178 +[titan] 2025-07-10 06:51:19,779 - root - INFO - lr: 6.0139e-05 gnorm: 0.97 [17:16:17< 4:43:19] +[titan] 2025-07-10 06:51:23,679 - root - INFO - step: 78535 loss: 16.3529 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8499 +[titan] 2025-07-10 06:51:23,679 - root - INFO - lr: 6.0125e-05 gnorm: 0.97 [17:16:21< 4:43:15] +[titan] 2025-07-10 06:51:27,595 - root - INFO - step: 78540 loss: 16.5357 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 13.9865 +[titan] 2025-07-10 06:51:27,595 - root - INFO - lr: 6.0112e-05 gnorm: 1.00 [17:16:25< 4:43:11] +[titan] 2025-07-10 06:51:31,499 - root - INFO - step: 78545 loss: 16.3417 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8478 +[titan] 2025-07-10 06:51:31,499 - root - INFO - lr: 6.0098e-05 gnorm: 0.94 [17:16:28< 4:43:07] +[titan] 2025-07-10 06:51:34,626 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:51:35,412 - root - INFO - step: 78550 loss: 16.3906 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.8834 +[titan] 2025-07-10 06:51:35,412 - root - INFO - lr: 6.0085e-05 gnorm: 0.94 [17:16:32< 4:43:03] +[titan] 2025-07-10 06:51:39,330 - root - INFO - step: 78555 loss: 16.0130 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4495 global_avg_mtp_loss: 13.5635 +[titan] 2025-07-10 06:51:39,331 - root - INFO - lr: 6.0071e-05 gnorm: 0.97 [17:16:36< 4:42:59] +[titan] 2025-07-10 06:51:43,237 - root - INFO - step: 78560 loss: 16.0545 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.5995 +[titan] 2025-07-10 06:51:43,237 - root - INFO - lr: 6.0058e-05 gnorm: 0.98 [17:16:40< 4:42:55] +[titan] 2025-07-10 06:51:47,137 - root - INFO - step: 78565 loss: 16.1753 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.7019 +[titan] 2025-07-10 06:51:47,137 - root - INFO - lr: 6.0044e-05 gnorm: 0.92 [17:16:44< 4:42:51] +[titan] 2025-07-10 06:51:51,070 - root - INFO - step: 78570 loss: 15.9173 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4225 global_avg_mtp_loss: 13.4948 +[titan] 2025-07-10 06:51:51,070 - root - INFO - lr: 6.0031e-05 gnorm: 0.96 [17:16:48< 4:42:47] +[titan] 2025-07-10 06:51:54,992 - root - INFO - step: 78575 loss: 16.2295 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7484 +[titan] 2025-07-10 06:51:54,992 - root - INFO - lr: 6.0017e-05 gnorm: 0.92 [17:16:52< 4:42:43] +[titan] 2025-07-10 06:51:58,932 - root - INFO - step: 78580 loss: 16.7132 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5772 global_avg_mtp_loss: 14.1360 +[titan] 2025-07-10 06:51:58,932 - root - INFO - lr: 6.0004e-05 gnorm: 0.95 [17:16:56< 4:42:39] +[titan] 2025-07-10 06:52:02,879 - root - INFO - step: 78585 loss: 16.3723 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.8527 +[titan] 2025-07-10 06:52:02,880 - root - INFO - lr: 5.9990e-05 gnorm: 0.94 [17:17:00< 4:42:35] +[titan] 2025-07-10 06:52:06,803 - root - INFO - step: 78590 loss: 16.0139 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4460 global_avg_mtp_loss: 13.5679 +[titan] 2025-07-10 06:52:06,804 - root - INFO - lr: 5.9977e-05 gnorm: 1.04 [17:17:04< 4:42:31] +[titan] 2025-07-10 06:52:10,746 - root - INFO - step: 78595 loss: 16.4594 memory: 44.58GiB(31.99%) tps: 83,120 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.9354 +[titan] 2025-07-10 06:52:10,746 - root - INFO - lr: 5.9964e-05 gnorm: 0.98 [17:17:08< 4:42:27] +[titan] 2025-07-10 06:52:13,862 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:52:14,652 - root - INFO - step: 78600 loss: 16.0922 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.6390 +[titan] 2025-07-10 06:52:14,652 - root - INFO - lr: 5.9950e-05 gnorm: 1.03 [17:17:12< 4:42:23] +[titan] 2025-07-10 06:52:18,573 - root - INFO - step: 78605 loss: 16.3270 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8194 +[titan] 2025-07-10 06:52:18,574 - root - INFO - lr: 5.9937e-05 gnorm: 0.94 [17:17:15< 4:42:19] +[titan] 2025-07-10 06:52:22,498 - root - INFO - step: 78610 loss: 16.2976 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8027 +[titan] 2025-07-10 06:52:22,498 - root - INFO - lr: 5.9923e-05 gnorm: 0.94 [17:17:19< 4:42:15] +[titan] 2025-07-10 06:52:26,422 - root - INFO - step: 78615 loss: 16.3551 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8402 +[titan] 2025-07-10 06:52:26,423 - root - INFO - lr: 5.9910e-05 gnorm: 0.98 [17:17:23< 4:42:11] +[titan] 2025-07-10 06:52:30,375 - root - INFO - step: 78620 loss: 16.0146 memory: 44.58GiB(31.99%) tps: 82,900 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.4355 global_avg_mtp_loss: 13.5791 +[titan] 2025-07-10 06:52:30,376 - root - INFO - lr: 5.9896e-05 gnorm: 0.96 [17:17:27< 4:42:07] +[titan] 2025-07-10 06:52:34,285 - root - INFO - step: 78625 loss: 16.8328 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.6126 global_avg_mtp_loss: 14.2201 +[titan] 2025-07-10 06:52:34,286 - root - INFO - lr: 5.9883e-05 gnorm: 1.03 [17:17:31< 4:42:03] +[titan] 2025-07-10 06:52:38,230 - root - INFO - step: 78630 loss: 16.2776 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7853 +[titan] 2025-07-10 06:52:38,231 - root - INFO - lr: 5.9869e-05 gnorm: 1.00 [17:17:35< 4:41:59] +[titan] 2025-07-10 06:52:42,164 - root - INFO - step: 78635 loss: 16.5198 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5314 global_avg_mtp_loss: 13.9884 +[titan] 2025-07-10 06:52:42,165 - root - INFO - lr: 5.9856e-05 gnorm: 1.00 [17:17:39< 4:41:55] +[titan] 2025-07-10 06:52:46,068 - root - INFO - step: 78640 loss: 16.0139 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4445 global_avg_mtp_loss: 13.5694 +[titan] 2025-07-10 06:52:46,068 - root - INFO - lr: 5.9843e-05 gnorm: 0.99 [17:17:43< 4:41:51] +[titan] 2025-07-10 06:52:49,968 - root - INFO - step: 78645 loss: 16.3323 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.8449 +[titan] 2025-07-10 06:52:49,969 - root - INFO - lr: 5.9829e-05 gnorm: 0.95 [17:17:47< 4:41:47] +[titan] 2025-07-10 06:52:53,080 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:52:53,869 - root - INFO - step: 78650 loss: 16.5668 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 14.0236 +[titan] 2025-07-10 06:52:53,869 - root - INFO - lr: 5.9816e-05 gnorm: 0.96 [17:17:51< 4:41:43] +[titan] 2025-07-10 06:52:57,785 - root - INFO - step: 78655 loss: 16.3164 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.8286 +[titan] 2025-07-10 06:52:57,785 - root - INFO - lr: 5.9802e-05 gnorm: 0.98 [17:17:55< 4:41:39] +[titan] 2025-07-10 06:53:01,694 - root - INFO - step: 78660 loss: 16.2278 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 06:53:01,694 - root - INFO - lr: 5.9789e-05 gnorm: 1.01 [17:17:59< 4:41:35] +[titan] 2025-07-10 06:53:05,607 - root - INFO - step: 78665 loss: 16.3710 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.8745 +[titan] 2025-07-10 06:53:05,607 - root - INFO - lr: 5.9775e-05 gnorm: 0.96 [17:18:03< 4:41:31] +[titan] 2025-07-10 06:53:09,557 - root - INFO - step: 78670 loss: 16.1622 memory: 44.58GiB(31.99%) tps: 82,975 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.6917 +[titan] 2025-07-10 06:53:09,557 - root - INFO - lr: 5.9762e-05 gnorm: 0.98 [17:18:06< 4:41:28] +[titan] 2025-07-10 06:53:13,467 - root - INFO - step: 78675 loss: 16.3267 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.8350 +[titan] 2025-07-10 06:53:13,467 - root - INFO - lr: 5.9749e-05 gnorm: 0.94 [17:18:10< 4:41:24] +[titan] 2025-07-10 06:53:17,381 - root - INFO - step: 78680 loss: 16.1998 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7267 +[titan] 2025-07-10 06:53:17,382 - root - INFO - lr: 5.9735e-05 gnorm: 0.94 [17:18:14< 4:41:20] +[titan] 2025-07-10 06:53:21,306 - root - INFO - step: 78685 loss: 15.9797 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4375 global_avg_mtp_loss: 13.5422 +[titan] 2025-07-10 06:53:21,307 - root - INFO - lr: 5.9722e-05 gnorm: 0.94 [17:18:18< 4:41:16] +[titan] 2025-07-10 06:53:25,247 - root - INFO - step: 78690 loss: 16.4533 memory: 44.58GiB(31.99%) tps: 83,165 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.9430 +[titan] 2025-07-10 06:53:25,247 - root - INFO - lr: 5.9708e-05 gnorm: 0.95 [17:18:22< 4:41:12] +[titan] 2025-07-10 06:53:29,151 - root - INFO - step: 78695 loss: 16.5281 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.9843 +[titan] 2025-07-10 06:53:29,151 - root - INFO - lr: 5.9695e-05 gnorm: 0.98 [17:18:26< 4:41:08] +[titan] 2025-07-10 06:53:32,252 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:53:33,036 - root - INFO - step: 78700 loss: 16.2565 memory: 44.58GiB(31.99%) tps: 84,344 tflops: 291.09 mfu: 29.43% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.7604 +[titan] 2025-07-10 06:53:33,036 - root - INFO - lr: 5.9682e-05 gnorm: 0.96 [17:18:30< 4:41:04] +[titan] 2025-07-10 06:53:36,946 - root - INFO - step: 78705 loss: 16.2031 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7273 +[titan] 2025-07-10 06:53:36,946 - root - INFO - lr: 5.9668e-05 gnorm: 0.92 [17:18:34< 4:41:00] +[titan] 2025-07-10 06:53:40,844 - root - INFO - step: 78710 loss: 16.2902 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.8035 +[titan] 2025-07-10 06:53:40,844 - root - INFO - lr: 5.9655e-05 gnorm: 1.00 [17:18:38< 4:40:56] +[titan] 2025-07-10 06:53:44,761 - root - INFO - step: 78715 loss: 16.5380 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5397 global_avg_mtp_loss: 13.9983 +[titan] 2025-07-10 06:53:44,761 - root - INFO - lr: 5.9641e-05 gnorm: 1.02 [17:18:42< 4:40:52] +[titan] 2025-07-10 06:53:48,698 - root - INFO - step: 78720 loss: 16.3219 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8180 +[titan] 2025-07-10 06:53:48,698 - root - INFO - lr: 5.9628e-05 gnorm: 0.95 [17:18:46< 4:40:48] +[titan] 2025-07-10 06:53:52,605 - root - INFO - step: 78725 loss: 15.8076 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4136 global_avg_mtp_loss: 13.3939 +[titan] 2025-07-10 06:53:52,605 - root - INFO - lr: 5.9615e-05 gnorm: 0.99 [17:18:49< 4:40:44] +[titan] 2025-07-10 06:53:56,511 - root - INFO - step: 78730 loss: 16.1522 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4555 global_avg_mtp_loss: 13.6967 +[titan] 2025-07-10 06:53:56,511 - root - INFO - lr: 5.9601e-05 gnorm: 0.95 [17:18:53< 4:40:40] +[titan] 2025-07-10 06:54:00,423 - root - INFO - step: 78735 loss: 16.2160 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7300 +[titan] 2025-07-10 06:54:00,423 - root - INFO - lr: 5.9588e-05 gnorm: 1.00 [17:18:57< 4:40:36] +[titan] 2025-07-10 06:54:04,385 - root - INFO - step: 78740 loss: 16.3500 memory: 44.58GiB(31.99%) tps: 82,706 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 06:54:04,386 - root - INFO - lr: 5.9574e-05 gnorm: 0.96 [17:19:01< 4:40:32] +[titan] 2025-07-10 06:54:08,289 - root - INFO - step: 78745 loss: 16.1699 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7015 +[titan] 2025-07-10 06:54:08,289 - root - INFO - lr: 5.9561e-05 gnorm: 0.99 [17:19:05< 4:40:28] +[titan] 2025-07-10 06:54:11,432 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:54:12,225 - root - INFO - step: 78750 loss: 15.9339 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4251 global_avg_mtp_loss: 13.5088 +[titan] 2025-07-10 06:54:12,225 - root - INFO - lr: 5.9548e-05 gnorm: 0.95 [17:19:09< 4:40:24] +[titan] 2025-07-10 06:54:16,126 - root - INFO - step: 78755 loss: 16.5206 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5377 global_avg_mtp_loss: 13.9829 +[titan] 2025-07-10 06:54:16,127 - root - INFO - lr: 5.9534e-05 gnorm: 0.96 [17:19:13< 4:40:20] +[titan] 2025-07-10 06:54:20,040 - root - INFO - step: 78760 loss: 16.0834 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4593 global_avg_mtp_loss: 13.6241 +[titan] 2025-07-10 06:54:20,040 - root - INFO - lr: 5.9521e-05 gnorm: 0.93 [17:19:17< 4:40:16] +[titan] 2025-07-10 06:54:23,994 - root - INFO - step: 78765 loss: 16.4690 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5283 global_avg_mtp_loss: 13.9407 +[titan] 2025-07-10 06:54:23,994 - root - INFO - lr: 5.9508e-05 gnorm: 0.92 [17:19:21< 4:40:12] +[titan] 2025-07-10 06:54:27,894 - root - INFO - step: 78770 loss: 16.1982 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.6984 +[titan] 2025-07-10 06:54:27,894 - root - INFO - lr: 5.9494e-05 gnorm: 1.02 [17:19:25< 4:40:08] +[titan] 2025-07-10 06:54:31,794 - root - INFO - step: 78775 loss: 16.2862 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.7922 +[titan] 2025-07-10 06:54:31,794 - root - INFO - lr: 5.9481e-05 gnorm: 0.95 [17:19:29< 4:40:04] +[titan] 2025-07-10 06:54:35,749 - root - INFO - step: 78780 loss: 16.2099 memory: 44.58GiB(31.99%) tps: 82,843 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 06:54:35,750 - root - INFO - lr: 5.9467e-05 gnorm: 1.00 [17:19:33< 4:40:00] +[titan] 2025-07-10 06:54:39,681 - root - INFO - step: 78785 loss: 16.2095 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7451 +[titan] 2025-07-10 06:54:39,681 - root - INFO - lr: 5.9454e-05 gnorm: 0.95 [17:19:37< 4:39:56] +[titan] 2025-07-10 06:54:43,577 - root - INFO - step: 78790 loss: 16.2382 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.7404 +[titan] 2025-07-10 06:54:43,577 - root - INFO - lr: 5.9441e-05 gnorm: 0.94 [17:19:40< 4:39:52] +[titan] 2025-07-10 06:54:47,487 - root - INFO - step: 78795 loss: 16.2352 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 06:54:47,487 - root - INFO - lr: 5.9427e-05 gnorm: 0.94 [17:19:44< 4:39:48] +[titan] 2025-07-10 06:54:50,623 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:54:51,409 - root - INFO - step: 78800 loss: 16.1816 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.7184 +[titan] 2025-07-10 06:54:51,409 - root - INFO - lr: 5.9414e-05 gnorm: 0.95 [17:19:48< 4:39:44] +[titan] 2025-07-10 06:54:55,301 - root - INFO - step: 78805 loss: 16.3077 memory: 44.58GiB(31.99%) tps: 84,192 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.7970 +[titan] 2025-07-10 06:54:55,302 - root - INFO - lr: 5.9401e-05 gnorm: 0.93 [17:19:52< 4:39:40] +[titan] 2025-07-10 06:54:59,244 - root - INFO - step: 78810 loss: 16.5573 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.5480 global_avg_mtp_loss: 14.0093 +[titan] 2025-07-10 06:54:59,244 - root - INFO - lr: 5.9387e-05 gnorm: 0.98 [17:19:56< 4:39:36] +[titan] 2025-07-10 06:55:03,133 - root - INFO - step: 78815 loss: 16.1838 memory: 44.58GiB(31.99%) tps: 84,266 tflops: 290.81 mfu: 29.40% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7151 +[titan] 2025-07-10 06:55:03,133 - root - INFO - lr: 5.9374e-05 gnorm: 0.94 [17:20:00< 4:39:32] +[titan] 2025-07-10 06:55:07,041 - root - INFO - step: 78820 loss: 16.4928 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9608 +[titan] 2025-07-10 06:55:07,041 - root - INFO - lr: 5.9361e-05 gnorm: 0.97 [17:20:04< 4:39:28] +[titan] 2025-07-10 06:55:10,956 - root - INFO - step: 78825 loss: 16.3940 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8865 +[titan] 2025-07-10 06:55:10,957 - root - INFO - lr: 5.9347e-05 gnorm: 0.93 [17:20:08< 4:39:24] +[titan] 2025-07-10 06:55:14,873 - root - INFO - step: 78830 loss: 16.3681 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8586 +[titan] 2025-07-10 06:55:14,874 - root - INFO - lr: 5.9334e-05 gnorm: 0.95 [17:20:12< 4:39:20] +[titan] 2025-07-10 06:55:18,799 - root - INFO - step: 78835 loss: 16.1316 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6632 +[titan] 2025-07-10 06:55:18,799 - root - INFO - lr: 5.9321e-05 gnorm: 0.93 [17:20:16< 4:39:17] +[titan] 2025-07-10 06:55:22,693 - root - INFO - step: 78840 loss: 16.3704 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8758 +[titan] 2025-07-10 06:55:22,694 - root - INFO - lr: 5.9307e-05 gnorm: 0.90 [17:20:20< 4:39:13] +[titan] 2025-07-10 06:55:26,733 - root - INFO - step: 78845 loss: 16.0538 memory: 44.58GiB(31.99%) tps: 81,125 tflops: 279.98 mfu: 28.31% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6006 +[titan] 2025-07-10 06:55:26,733 - root - INFO - lr: 5.9294e-05 gnorm: 0.97 [17:20:24< 4:39:09] +[titan] 2025-07-10 06:55:29,240 - root - INFO - Dumping profiler traces at step 78848 +[titan] 2025-07-10 06:55:29,274 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 06:55:30,066 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:55:30,865 - root - INFO - step: 78850 loss: 16.2916 memory: 44.58GiB(31.99%) tps: 79,317 tflops: 273.74 mfu: 27.68% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.8030 +[titan] 2025-07-10 06:55:30,865 - root - INFO - lr: 5.9281e-05 gnorm: 0.95 [17:20:28< 4:39:05] +[titan] 2025-07-10 06:55:34,816 - root - INFO - step: 78855 loss: 16.0847 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4469 global_avg_mtp_loss: 13.6378 +[titan] 2025-07-10 06:55:34,816 - root - INFO - lr: 5.9267e-05 gnorm: 0.94 [17:20:32< 4:39:01] +[titan] 2025-07-10 06:55:38,745 - root - INFO - step: 78860 loss: 16.4218 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.9011 +[titan] 2025-07-10 06:55:38,745 - root - INFO - lr: 5.9254e-05 gnorm: 0.93 [17:20:36< 4:38:57] +[titan] 2025-07-10 06:55:42,691 - root - INFO - step: 78865 loss: 16.3606 memory: 44.58GiB(31.99%) tps: 83,047 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8586 +[titan] 2025-07-10 06:55:42,691 - root - INFO - lr: 5.9241e-05 gnorm: 0.96 [17:20:40< 4:38:53] +[titan] 2025-07-10 06:55:46,601 - root - INFO - step: 78870 loss: 16.3680 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8684 +[titan] 2025-07-10 06:55:46,602 - root - INFO - lr: 5.9227e-05 gnorm: 0.95 [17:20:43< 4:38:49] +[titan] 2025-07-10 06:55:50,522 - root - INFO - step: 78875 loss: 16.2666 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5010 global_avg_mtp_loss: 13.7656 +[titan] 2025-07-10 06:55:50,522 - root - INFO - lr: 5.9214e-05 gnorm: 0.99 [17:20:47< 4:38:45] +[titan] 2025-07-10 06:55:54,444 - root - INFO - step: 78880 loss: 16.2658 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7757 +[titan] 2025-07-10 06:55:54,444 - root - INFO - lr: 5.9201e-05 gnorm: 0.97 [17:20:51< 4:38:41] +[titan] 2025-07-10 06:55:58,364 - root - INFO - step: 78885 loss: 16.1184 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.6481 +[titan] 2025-07-10 06:55:58,364 - root - INFO - lr: 5.9188e-05 gnorm: 1.01 [17:20:55< 4:38:37] +[titan] 2025-07-10 06:56:02,293 - root - INFO - step: 78890 loss: 16.4902 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9606 +[titan] 2025-07-10 06:56:02,293 - root - INFO - lr: 5.9174e-05 gnorm: 0.96 [17:20:59< 4:38:33] +[titan] 2025-07-10 06:56:06,220 - root - INFO - step: 78895 loss: 16.2775 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7937 +[titan] 2025-07-10 06:56:06,220 - root - INFO - lr: 5.9161e-05 gnorm: 0.96 [17:21:03< 4:38:29] +[titan] 2025-07-10 06:56:09,366 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:56:10,164 - root - INFO - step: 78900 loss: 16.2793 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7919 +[titan] 2025-07-10 06:56:10,164 - root - INFO - lr: 5.9148e-05 gnorm: 0.96 [17:21:07< 4:38:25] +[titan] 2025-07-10 06:56:14,170 - root - INFO - step: 78905 loss: 16.3069 memory: 44.58GiB(31.99%) tps: 81,795 tflops: 282.29 mfu: 28.54% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8159 +[titan] 2025-07-10 06:56:14,171 - root - INFO - lr: 5.9134e-05 gnorm: 0.97 [17:21:11< 4:38:21] +[titan] 2025-07-10 06:56:18,100 - root - INFO - step: 78910 loss: 16.1502 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 06:56:18,100 - root - INFO - lr: 5.9121e-05 gnorm: 0.93 [17:21:15< 4:38:17] +[titan] 2025-07-10 06:56:22,038 - root - INFO - step: 78915 loss: 16.2909 memory: 44.58GiB(31.99%) tps: 83,210 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.8012 +[titan] 2025-07-10 06:56:22,038 - root - INFO - lr: 5.9108e-05 gnorm: 0.96 [17:21:19< 4:38:13] +[titan] 2025-07-10 06:56:25,977 - root - INFO - step: 78920 loss: 16.3598 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8600 +[titan] 2025-07-10 06:56:25,978 - root - INFO - lr: 5.9094e-05 gnorm: 0.93 [17:21:23< 4:38:09] +[titan] 2025-07-10 06:56:29,898 - root - INFO - step: 78925 loss: 16.2244 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 06:56:29,898 - root - INFO - lr: 5.9081e-05 gnorm: 0.97 [17:21:27< 4:38:05] +[titan] 2025-07-10 06:56:33,816 - root - INFO - step: 78930 loss: 16.0549 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4476 global_avg_mtp_loss: 13.6073 +[titan] 2025-07-10 06:56:33,816 - root - INFO - lr: 5.9068e-05 gnorm: 0.96 [17:21:31< 4:38:01] +[titan] 2025-07-10 06:56:37,747 - root - INFO - step: 78935 loss: 16.3700 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8543 +[titan] 2025-07-10 06:56:37,747 - root - INFO - lr: 5.9055e-05 gnorm: 0.97 [17:21:35< 4:37:57] +[titan] 2025-07-10 06:56:41,651 - root - INFO - step: 78940 loss: 16.2710 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.7866 +[titan] 2025-07-10 06:56:41,652 - root - INFO - lr: 5.9041e-05 gnorm: 0.95 [17:21:39< 4:37:53] +[titan] 2025-07-10 06:56:45,574 - root - INFO - step: 78945 loss: 16.2299 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7506 +[titan] 2025-07-10 06:56:45,575 - root - INFO - lr: 5.9028e-05 gnorm: 0.94 [17:21:42< 4:37:49] +[titan] 2025-07-10 06:56:48,719 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:56:49,509 - root - INFO - step: 78950 loss: 16.1746 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.7140 +[titan] 2025-07-10 06:56:49,509 - root - INFO - lr: 5.9015e-05 gnorm: 1.03 [17:21:46< 4:37:45] +[titan] 2025-07-10 06:56:53,425 - root - INFO - step: 78955 loss: 16.2847 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.7853 +[titan] 2025-07-10 06:56:53,425 - root - INFO - lr: 5.9002e-05 gnorm: 0.98 [17:21:50< 4:37:41] +[titan] 2025-07-10 06:56:57,351 - root - INFO - step: 78960 loss: 16.0344 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4445 global_avg_mtp_loss: 13.5899 +[titan] 2025-07-10 06:56:57,351 - root - INFO - lr: 5.8988e-05 gnorm: 0.97 [17:21:54< 4:37:37] +[titan] 2025-07-10 06:57:01,309 - root - INFO - step: 78965 loss: 15.8849 memory: 44.58GiB(31.99%) tps: 82,789 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.4194 global_avg_mtp_loss: 13.4655 +[titan] 2025-07-10 06:57:01,309 - root - INFO - lr: 5.8975e-05 gnorm: 0.94 [17:21:58< 4:37:33] +[titan] 2025-07-10 06:57:05,248 - root - INFO - step: 78970 loss: 16.1383 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6691 +[titan] 2025-07-10 06:57:05,249 - root - INFO - lr: 5.8962e-05 gnorm: 0.95 [17:22:02< 4:37:30] +[titan] 2025-07-10 06:57:09,157 - root - INFO - step: 78975 loss: 16.5138 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5336 global_avg_mtp_loss: 13.9802 +[titan] 2025-07-10 06:57:09,157 - root - INFO - lr: 5.8949e-05 gnorm: 0.96 [17:22:06< 4:37:26] +[titan] 2025-07-10 06:57:13,073 - root - INFO - step: 78980 loss: 16.1960 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.7117 +[titan] 2025-07-10 06:57:13,073 - root - INFO - lr: 5.8935e-05 gnorm: 1.72 [17:22:10< 4:37:22] +[titan] 2025-07-10 06:57:17,031 - root - INFO - step: 78985 loss: 16.0011 memory: 44.58GiB(31.99%) tps: 82,797 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.4309 global_avg_mtp_loss: 13.5702 +[titan] 2025-07-10 06:57:17,031 - root - INFO - lr: 5.8922e-05 gnorm: 0.95 [17:22:14< 4:37:18] +[titan] 2025-07-10 06:57:20,949 - root - INFO - step: 78990 loss: 16.2002 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.7290 +[titan] 2025-07-10 06:57:20,949 - root - INFO - lr: 5.8909e-05 gnorm: 0.92 [17:22:18< 4:37:14] +[titan] 2025-07-10 06:57:24,873 - root - INFO - step: 78995 loss: 16.2464 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7691 +[titan] 2025-07-10 06:57:24,873 - root - INFO - lr: 5.8896e-05 gnorm: 0.93 [17:22:22< 4:37:10] +[titan] 2025-07-10 06:57:28,034 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:57:28,853 - root - INFO - step: 79000 loss: 16.3898 memory: 44.58GiB(31.99%) tps: 82,342 tflops: 284.17 mfu: 28.73% global_avg_ntp_loss: 2.5161 global_avg_mtp_loss: 13.8737 +[titan] 2025-07-10 06:57:28,853 - root - INFO - lr: 5.8882e-05 gnorm: 0.98 [17:22:26< 4:37:06] +[titan] 2025-07-10 06:57:32,760 - root - INFO - step: 79005 loss: 15.9224 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4240 global_avg_mtp_loss: 13.4984 +[titan] 2025-07-10 06:57:32,761 - root - INFO - lr: 5.8869e-05 gnorm: 1.01 [17:22:30< 4:37:02] +[titan] 2025-07-10 06:57:36,674 - root - INFO - step: 79010 loss: 16.1521 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6824 +[titan] 2025-07-10 06:57:36,674 - root - INFO - lr: 5.8856e-05 gnorm: 0.97 [17:22:34< 4:36:58] +[titan] 2025-07-10 06:57:40,581 - root - INFO - step: 79015 loss: 16.1658 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4525 global_avg_mtp_loss: 13.7133 +[titan] 2025-07-10 06:57:40,581 - root - INFO - lr: 5.8843e-05 gnorm: 0.94 [17:22:37< 4:36:54] +[titan] 2025-07-10 06:57:44,482 - root - INFO - step: 79020 loss: 16.2799 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7881 +[titan] 2025-07-10 06:57:44,482 - root - INFO - lr: 5.8829e-05 gnorm: 0.93 [17:22:41< 4:36:50] +[titan] 2025-07-10 06:57:48,401 - root - INFO - step: 79025 loss: 16.4198 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.9112 +[titan] 2025-07-10 06:57:48,401 - root - INFO - lr: 5.8816e-05 gnorm: 0.98 [17:22:45< 4:36:46] +[titan] 2025-07-10 06:57:52,291 - root - INFO - step: 79030 loss: 16.2003 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7230 +[titan] 2025-07-10 06:57:52,291 - root - INFO - lr: 5.8803e-05 gnorm: 0.97 [17:22:49< 4:36:42] +[titan] 2025-07-10 06:57:56,189 - root - INFO - step: 79035 loss: 16.2705 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.7816 +[titan] 2025-07-10 06:57:56,189 - root - INFO - lr: 5.8790e-05 gnorm: 1.00 [17:22:53< 4:36:38] +[titan] 2025-07-10 06:58:00,088 - root - INFO - step: 79040 loss: 16.3976 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.9062 +[titan] 2025-07-10 06:58:00,088 - root - INFO - lr: 5.8776e-05 gnorm: 0.99 [17:22:57< 4:36:34] +[titan] 2025-07-10 06:58:03,979 - root - INFO - step: 79045 loss: 16.1874 memory: 44.58GiB(31.99%) tps: 84,224 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7187 +[titan] 2025-07-10 06:58:03,979 - root - INFO - lr: 5.8763e-05 gnorm: 0.93 [17:23:01< 4:36:30] +[titan] 2025-07-10 06:58:07,137 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:58:07,925 - root - INFO - step: 79050 loss: 16.2764 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7874 +[titan] 2025-07-10 06:58:07,925 - root - INFO - lr: 5.8750e-05 gnorm: 0.98 [17:23:05< 4:36:26] +[titan] 2025-07-10 06:58:11,867 - root - INFO - step: 79055 loss: 16.3131 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8194 +[titan] 2025-07-10 06:58:11,867 - root - INFO - lr: 5.8737e-05 gnorm: 0.94 [17:23:09< 4:36:22] +[titan] 2025-07-10 06:58:15,783 - root - INFO - step: 79060 loss: 16.3400 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8331 +[titan] 2025-07-10 06:58:15,783 - root - INFO - lr: 5.8724e-05 gnorm: 0.99 [17:23:13< 4:36:18] +[titan] 2025-07-10 06:58:19,725 - root - INFO - step: 79065 loss: 16.0893 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6329 +[titan] 2025-07-10 06:58:19,725 - root - INFO - lr: 5.8710e-05 gnorm: 0.96 [17:23:17< 4:36:14] +[titan] 2025-07-10 06:58:23,620 - root - INFO - step: 79070 loss: 16.2605 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7770 +[titan] 2025-07-10 06:58:23,620 - root - INFO - lr: 5.8697e-05 gnorm: 0.98 [17:23:20< 4:36:10] +[titan] 2025-07-10 06:58:27,519 - root - INFO - step: 79075 loss: 16.6473 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5614 global_avg_mtp_loss: 14.0859 +[titan] 2025-07-10 06:58:27,520 - root - INFO - lr: 5.8684e-05 gnorm: 0.95 [17:23:24< 4:36:06] +[titan] 2025-07-10 06:58:31,413 - root - INFO - step: 79080 loss: 16.4241 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.9164 +[titan] 2025-07-10 06:58:31,413 - root - INFO - lr: 5.8671e-05 gnorm: 0.97 [17:23:28< 4:36:02] +[titan] 2025-07-10 06:58:35,348 - root - INFO - step: 79085 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 83,286 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7247 +[titan] 2025-07-10 06:58:35,348 - root - INFO - lr: 5.8658e-05 gnorm: 1.04 [17:23:32< 4:35:58] +[titan] 2025-07-10 06:58:39,259 - root - INFO - step: 79090 loss: 15.9377 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4198 global_avg_mtp_loss: 13.5179 +[titan] 2025-07-10 06:58:39,259 - root - INFO - lr: 5.8644e-05 gnorm: 0.95 [17:23:36< 4:35:54] +[titan] 2025-07-10 06:58:43,163 - root - INFO - step: 79095 loss: 16.1987 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7235 +[titan] 2025-07-10 06:58:43,163 - root - INFO - lr: 5.8631e-05 gnorm: 0.96 [17:23:40< 4:35:50] +[titan] 2025-07-10 06:58:46,276 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:58:47,062 - root - INFO - step: 79100 loss: 16.5332 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5436 global_avg_mtp_loss: 13.9896 +[titan] 2025-07-10 06:58:47,063 - root - INFO - lr: 5.8618e-05 gnorm: 1.00 [17:23:44< 4:35:46] +[titan] 2025-07-10 06:58:50,967 - root - INFO - step: 79105 loss: 16.1716 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.7150 +[titan] 2025-07-10 06:58:50,968 - root - INFO - lr: 5.8605e-05 gnorm: 0.97 [17:23:48< 4:35:42] +[titan] 2025-07-10 06:58:54,889 - root - INFO - step: 79110 loss: 16.4846 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5527 global_avg_mtp_loss: 13.9319 +[titan] 2025-07-10 06:58:54,889 - root - INFO - lr: 5.8592e-05 gnorm: 1.04 [17:23:52< 4:35:38] +[titan] 2025-07-10 06:58:58,789 - root - INFO - step: 79115 loss: 16.1141 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.6593 +[titan] 2025-07-10 06:58:58,790 - root - INFO - lr: 5.8578e-05 gnorm: 0.95 [17:23:56< 4:35:34] +[titan] 2025-07-10 06:59:02,725 - root - INFO - step: 79120 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.8055 +[titan] 2025-07-10 06:59:02,725 - root - INFO - lr: 5.8565e-05 gnorm: 1.01 [17:24:00< 4:35:30] +[titan] 2025-07-10 06:59:06,643 - root - INFO - step: 79125 loss: 16.3278 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4920 global_avg_mtp_loss: 13.8358 +[titan] 2025-07-10 06:59:06,643 - root - INFO - lr: 5.8552e-05 gnorm: 0.95 [17:24:03< 4:35:26] +[titan] 2025-07-10 06:59:10,540 - root - INFO - step: 79130 loss: 16.3110 memory: 44.58GiB(31.99%) tps: 84,101 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8195 +[titan] 2025-07-10 06:59:10,540 - root - INFO - lr: 5.8539e-05 gnorm: 1.02 [17:24:07< 4:35:22] +[titan] 2025-07-10 06:59:14,464 - root - INFO - step: 79135 loss: 16.1956 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.7260 +[titan] 2025-07-10 06:59:14,464 - root - INFO - lr: 5.8526e-05 gnorm: 0.93 [17:24:11< 4:35:18] +[titan] 2025-07-10 06:59:18,392 - root - INFO - step: 79140 loss: 16.3000 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.7944 +[titan] 2025-07-10 06:59:18,392 - root - INFO - lr: 5.8513e-05 gnorm: 0.95 [17:24:15< 4:35:15] +[titan] 2025-07-10 06:59:22,286 - root - INFO - step: 79145 loss: 16.2167 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.7506 +[titan] 2025-07-10 06:59:22,286 - root - INFO - lr: 5.8499e-05 gnorm: 0.93 [17:24:19< 4:35:11] +[titan] 2025-07-10 06:59:25,393 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 06:59:26,183 - root - INFO - step: 79150 loss: 16.3960 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8887 +[titan] 2025-07-10 06:59:26,183 - root - INFO - lr: 5.8486e-05 gnorm: 0.93 [17:24:23< 4:35:07] +[titan] 2025-07-10 06:59:30,091 - root - INFO - step: 79155 loss: 16.1472 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.6763 +[titan] 2025-07-10 06:59:30,091 - root - INFO - lr: 5.8473e-05 gnorm: 0.96 [17:24:27< 4:35:03] +[titan] 2025-07-10 06:59:34,019 - root - INFO - step: 79160 loss: 16.2003 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7216 +[titan] 2025-07-10 06:59:34,019 - root - INFO - lr: 5.8460e-05 gnorm: 0.96 [17:24:31< 4:34:59] +[titan] 2025-07-10 06:59:37,921 - root - INFO - step: 79165 loss: 16.3568 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.8332 +[titan] 2025-07-10 06:59:37,922 - root - INFO - lr: 5.8447e-05 gnorm: 0.99 [17:24:35< 4:34:55] +[titan] 2025-07-10 06:59:41,829 - root - INFO - step: 79170 loss: 16.3367 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8354 +[titan] 2025-07-10 06:59:41,829 - root - INFO - lr: 5.8434e-05 gnorm: 0.96 [17:24:39< 4:34:51] +[titan] 2025-07-10 06:59:45,768 - root - INFO - step: 79175 loss: 16.3442 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8525 +[titan] 2025-07-10 06:59:45,769 - root - INFO - lr: 5.8420e-05 gnorm: 0.99 [17:24:43< 4:34:47] +[titan] 2025-07-10 06:59:49,687 - root - INFO - step: 79180 loss: 15.9552 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4337 global_avg_mtp_loss: 13.5215 +[titan] 2025-07-10 06:59:49,687 - root - INFO - lr: 5.8407e-05 gnorm: 1.06 [17:24:47< 4:34:43] +[titan] 2025-07-10 06:59:53,587 - root - INFO - step: 79185 loss: 16.3596 memory: 44.58GiB(31.99%) tps: 84,027 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8585 +[titan] 2025-07-10 06:59:53,587 - root - INFO - lr: 5.8394e-05 gnorm: 0.96 [17:24:50< 4:34:39] +[titan] 2025-07-10 06:59:57,501 - root - INFO - step: 79190 loss: 16.0114 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.5713 +[titan] 2025-07-10 06:59:57,502 - root - INFO - lr: 5.8381e-05 gnorm: 0.95 [17:24:54< 4:34:35] +[titan] 2025-07-10 07:00:01,405 - root - INFO - step: 79195 loss: 16.1649 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.7006 +[titan] 2025-07-10 07:00:01,405 - root - INFO - lr: 5.8368e-05 gnorm: 0.96 [17:24:58< 4:34:31] +[titan] 2025-07-10 07:00:04,511 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:00:05,297 - root - INFO - step: 79200 loss: 16.5091 memory: 44.58GiB(31.99%) tps: 84,196 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9723 +[titan] 2025-07-10 07:00:05,298 - root - INFO - lr: 5.8355e-05 gnorm: 0.95 [17:25:02< 4:34:27] +[titan] 2025-07-10 07:00:09,206 - root - INFO - step: 79205 loss: 16.1447 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4621 global_avg_mtp_loss: 13.6827 +[titan] 2025-07-10 07:00:09,207 - root - INFO - lr: 5.8342e-05 gnorm: 0.97 [17:25:06< 4:34:23] +[titan] 2025-07-10 07:00:13,126 - root - INFO - step: 79210 loss: 16.1787 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7105 +[titan] 2025-07-10 07:00:13,126 - root - INFO - lr: 5.8329e-05 gnorm: 0.96 [17:25:10< 4:34:19] +[titan] 2025-07-10 07:00:17,058 - root - INFO - step: 79215 loss: 16.0316 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4353 global_avg_mtp_loss: 13.5962 +[titan] 2025-07-10 07:00:17,058 - root - INFO - lr: 5.8315e-05 gnorm: 0.98 [17:25:14< 4:34:15] +[titan] 2025-07-10 07:00:21,003 - root - INFO - step: 79220 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.8331 +[titan] 2025-07-10 07:00:21,003 - root - INFO - lr: 5.8302e-05 gnorm: 1.03 [17:25:18< 4:34:11] +[titan] 2025-07-10 07:00:24,898 - root - INFO - step: 79225 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.7992 +[titan] 2025-07-10 07:00:24,898 - root - INFO - lr: 5.8289e-05 gnorm: 0.95 [17:25:22< 4:34:07] +[titan] 2025-07-10 07:00:28,815 - root - INFO - step: 79230 loss: 16.2377 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7502 +[titan] 2025-07-10 07:00:28,815 - root - INFO - lr: 5.8276e-05 gnorm: 0.96 [17:25:26< 4:34:03] +[titan] 2025-07-10 07:00:32,718 - root - INFO - step: 79235 loss: 16.4271 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5263 global_avg_mtp_loss: 13.9008 +[titan] 2025-07-10 07:00:32,718 - root - INFO - lr: 5.8263e-05 gnorm: 0.94 [17:25:30< 4:33:59] +[titan] 2025-07-10 07:00:36,622 - root - INFO - step: 79240 loss: 16.2076 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7371 +[titan] 2025-07-10 07:00:36,623 - root - INFO - lr: 5.8250e-05 gnorm: 0.95 [17:25:33< 4:33:55] +[titan] 2025-07-10 07:00:40,534 - root - INFO - step: 79245 loss: 16.0256 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.5794 +[titan] 2025-07-10 07:00:40,534 - root - INFO - lr: 5.8237e-05 gnorm: 1.10 [17:25:37< 4:33:51] +[titan] 2025-07-10 07:00:43,664 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:00:44,450 - root - INFO - step: 79250 loss: 16.2053 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7342 +[titan] 2025-07-10 07:00:44,450 - root - INFO - lr: 5.8224e-05 gnorm: 0.96 [17:25:41< 4:33:47] +[titan] 2025-07-10 07:00:48,395 - root - INFO - step: 79255 loss: 16.3223 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 07:00:48,395 - root - INFO - lr: 5.8210e-05 gnorm: 0.95 [17:25:45< 4:33:43] +[titan] 2025-07-10 07:00:52,332 - root - INFO - step: 79260 loss: 16.3963 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8809 +[titan] 2025-07-10 07:00:52,333 - root - INFO - lr: 5.8197e-05 gnorm: 0.95 [17:25:49< 4:33:39] +[titan] 2025-07-10 07:00:56,281 - root - INFO - step: 79265 loss: 16.3669 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.8650 +[titan] 2025-07-10 07:00:56,281 - root - INFO - lr: 5.8184e-05 gnorm: 0.96 [17:25:53< 4:33:35] +[titan] 2025-07-10 07:01:00,272 - root - INFO - step: 79270 loss: 16.4312 memory: 44.58GiB(31.99%) tps: 82,097 tflops: 283.33 mfu: 28.65% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.9168 +[titan] 2025-07-10 07:01:00,273 - root - INFO - lr: 5.8171e-05 gnorm: 0.94 [17:25:57< 4:33:31] +[titan] 2025-07-10 07:01:04,176 - root - INFO - step: 79275 loss: 16.2192 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7425 +[titan] 2025-07-10 07:01:04,176 - root - INFO - lr: 5.8158e-05 gnorm: 0.92 [17:26:01< 4:33:27] +[titan] 2025-07-10 07:01:08,084 - root - INFO - step: 79280 loss: 16.1533 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6916 +[titan] 2025-07-10 07:01:08,084 - root - INFO - lr: 5.8145e-05 gnorm: 0.94 [17:26:05< 4:33:23] +[titan] 2025-07-10 07:01:12,022 - root - INFO - step: 79285 loss: 16.4176 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.5133 global_avg_mtp_loss: 13.9043 +[titan] 2025-07-10 07:01:12,022 - root - INFO - lr: 5.8132e-05 gnorm: 0.96 [17:26:09< 4:33:19] +[titan] 2025-07-10 07:01:15,957 - root - INFO - step: 79290 loss: 16.5258 memory: 44.58GiB(31.99%) tps: 83,288 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.5498 global_avg_mtp_loss: 13.9760 +[titan] 2025-07-10 07:01:15,957 - root - INFO - lr: 5.8119e-05 gnorm: 1.01 [17:26:13< 4:33:15] +[titan] 2025-07-10 07:01:19,865 - root - INFO - step: 79295 loss: 15.9530 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4484 global_avg_mtp_loss: 13.5047 +[titan] 2025-07-10 07:01:19,865 - root - INFO - lr: 5.8106e-05 gnorm: 0.98 [17:26:17< 4:33:11] +[titan] 2025-07-10 07:01:23,028 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:01:23,820 - root - INFO - step: 79300 loss: 16.1990 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.7337 +[titan] 2025-07-10 07:01:23,820 - root - INFO - lr: 5.8093e-05 gnorm: 0.99 [17:26:21< 4:33:08] +[titan] 2025-07-10 07:01:27,746 - root - INFO - step: 79305 loss: 16.0154 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.5825 +[titan] 2025-07-10 07:01:27,747 - root - INFO - lr: 5.8080e-05 gnorm: 0.99 [17:26:25< 4:33:04] +[titan] 2025-07-10 07:01:31,672 - root - INFO - step: 79310 loss: 16.2230 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7474 +[titan] 2025-07-10 07:01:31,673 - root - INFO - lr: 5.8066e-05 gnorm: 1.00 [17:26:29< 4:33:00] +[titan] 2025-07-10 07:01:35,593 - root - INFO - step: 79315 loss: 16.3458 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 07:01:35,593 - root - INFO - lr: 5.8053e-05 gnorm: 0.93 [17:26:32< 4:32:56] +[titan] 2025-07-10 07:01:39,509 - root - INFO - step: 79320 loss: 15.9975 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4357 global_avg_mtp_loss: 13.5618 +[titan] 2025-07-10 07:01:39,509 - root - INFO - lr: 5.8040e-05 gnorm: 1.00 [17:26:36< 4:32:52] +[titan] 2025-07-10 07:01:43,431 - root - INFO - step: 79325 loss: 16.4966 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 13.9686 +[titan] 2025-07-10 07:01:43,431 - root - INFO - lr: 5.8027e-05 gnorm: 0.94 [17:26:40< 4:32:48] +[titan] 2025-07-10 07:01:47,413 - root - INFO - step: 79330 loss: 16.3376 memory: 44.58GiB(31.99%) tps: 82,291 tflops: 284.00 mfu: 28.72% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8428 +[titan] 2025-07-10 07:01:47,413 - root - INFO - lr: 5.8014e-05 gnorm: 0.98 [17:26:44< 4:32:44] +[titan] 2025-07-10 07:01:51,325 - root - INFO - step: 79335 loss: 16.2261 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.7552 +[titan] 2025-07-10 07:01:51,326 - root - INFO - lr: 5.8001e-05 gnorm: 1.07 [17:26:48< 4:32:40] +[titan] 2025-07-10 07:01:55,231 - root - INFO - step: 79340 loss: 16.6507 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5643 global_avg_mtp_loss: 14.0864 +[titan] 2025-07-10 07:01:55,231 - root - INFO - lr: 5.7988e-05 gnorm: 0.97 [17:26:52< 4:32:36] +[titan] 2025-07-10 07:01:59,151 - root - INFO - step: 79345 loss: 16.0865 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4510 global_avg_mtp_loss: 13.6355 +[titan] 2025-07-10 07:01:59,151 - root - INFO - lr: 5.7975e-05 gnorm: 0.98 [17:26:56< 4:32:32] +[titan] 2025-07-10 07:02:02,267 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:02:03,057 - root - INFO - step: 79350 loss: 16.3301 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8369 +[titan] 2025-07-10 07:02:03,057 - root - INFO - lr: 5.7962e-05 gnorm: 1.01 [17:27:00< 4:32:28] +[titan] 2025-07-10 07:02:07,008 - root - INFO - step: 79355 loss: 16.5286 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.5467 global_avg_mtp_loss: 13.9819 +[titan] 2025-07-10 07:02:07,008 - root - INFO - lr: 5.7949e-05 gnorm: 1.01 [17:27:04< 4:32:24] +[titan] 2025-07-10 07:02:11,043 - root - INFO - step: 79360 loss: 16.1798 memory: 44.58GiB(31.99%) tps: 81,229 tflops: 280.34 mfu: 28.35% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 07:02:11,043 - root - INFO - lr: 5.7936e-05 gnorm: 0.96 [17:27:08< 4:32:20] +[titan] 2025-07-10 07:02:11,198 - root - INFO - Dumping profiler traces at step 79360 +[titan] 2025-07-10 07:02:11,232 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:02:15,202 - root - INFO - step: 79365 loss: 16.3870 memory: 44.58GiB(31.99%) tps: 78,781 tflops: 271.89 mfu: 27.49% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8797 +[titan] 2025-07-10 07:02:15,203 - root - INFO - lr: 5.7923e-05 gnorm: 1.03 [17:27:12< 4:32:16] +[titan] 2025-07-10 07:02:19,114 - root - INFO - step: 79370 loss: 15.9415 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4369 global_avg_mtp_loss: 13.5047 +[titan] 2025-07-10 07:02:19,115 - root - INFO - lr: 5.7910e-05 gnorm: 0.96 [17:27:16< 4:32:12] +[titan] 2025-07-10 07:02:23,048 - root - INFO - step: 79375 loss: 16.3395 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8290 +[titan] 2025-07-10 07:02:23,048 - root - INFO - lr: 5.7897e-05 gnorm: 0.97 [17:27:20< 4:32:08] +[titan] 2025-07-10 07:02:26,965 - root - INFO - step: 79380 loss: 16.3176 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.8293 +[titan] 2025-07-10 07:02:26,965 - root - INFO - lr: 5.7884e-05 gnorm: 0.93 [17:27:24< 4:32:04] +[titan] 2025-07-10 07:02:30,878 - root - INFO - step: 79385 loss: 16.0437 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5945 +[titan] 2025-07-10 07:02:30,878 - root - INFO - lr: 5.7871e-05 gnorm: 0.97 [17:27:28< 4:32:00] +[titan] 2025-07-10 07:02:34,777 - root - INFO - step: 79390 loss: 16.1195 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6629 +[titan] 2025-07-10 07:02:34,777 - root - INFO - lr: 5.7858e-05 gnorm: 0.94 [17:27:32< 4:31:56] +[titan] 2025-07-10 07:02:38,697 - root - INFO - step: 79395 loss: 16.4699 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.9491 +[titan] 2025-07-10 07:02:38,697 - root - INFO - lr: 5.7845e-05 gnorm: 0.96 [17:27:36< 4:31:52] +[titan] 2025-07-10 07:02:41,818 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:02:42,611 - root - INFO - step: 79400 loss: 16.0298 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.5688 +[titan] 2025-07-10 07:02:42,612 - root - INFO - lr: 5.7832e-05 gnorm: 1.01 [17:27:39< 4:31:48] +[titan] 2025-07-10 07:02:46,518 - root - INFO - step: 79405 loss: 16.1100 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4646 global_avg_mtp_loss: 13.6454 +[titan] 2025-07-10 07:02:46,518 - root - INFO - lr: 5.7819e-05 gnorm: 0.99 [17:27:43< 4:31:44] +[titan] 2025-07-10 07:02:50,420 - root - INFO - step: 79410 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.5207 global_avg_mtp_loss: 13.8187 +[titan] 2025-07-10 07:02:50,420 - root - INFO - lr: 5.7806e-05 gnorm: 1.05 [17:27:47< 4:31:40] +[titan] 2025-07-10 07:02:54,326 - root - INFO - step: 79415 loss: 16.2103 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7387 +[titan] 2025-07-10 07:02:54,326 - root - INFO - lr: 5.7793e-05 gnorm: 1.02 [17:27:51< 4:31:36] +[titan] 2025-07-10 07:02:58,236 - root - INFO - step: 79420 loss: 16.3200 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 07:02:58,237 - root - INFO - lr: 5.7780e-05 gnorm: 0.98 [17:27:55< 4:31:32] +[titan] 2025-07-10 07:03:02,197 - root - INFO - step: 79425 loss: 16.3261 memory: 44.58GiB(31.99%) tps: 82,736 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8342 +[titan] 2025-07-10 07:03:02,198 - root - INFO - lr: 5.7766e-05 gnorm: 0.98 [17:27:59< 4:31:28] +[titan] 2025-07-10 07:03:06,123 - root - INFO - step: 79430 loss: 16.3944 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.8917 +[titan] 2025-07-10 07:03:06,123 - root - INFO - lr: 5.7753e-05 gnorm: 0.92 [17:28:03< 4:31:24] +[titan] 2025-07-10 07:03:10,048 - root - INFO - step: 79435 loss: 15.9539 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4210 global_avg_mtp_loss: 13.5329 +[titan] 2025-07-10 07:03:10,048 - root - INFO - lr: 5.7740e-05 gnorm: 0.93 [17:28:07< 4:31:20] +[titan] 2025-07-10 07:03:13,987 - root - INFO - step: 79440 loss: 16.2936 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.8185 +[titan] 2025-07-10 07:03:13,987 - root - INFO - lr: 5.7727e-05 gnorm: 0.99 [17:28:11< 4:31:17] +[titan] 2025-07-10 07:03:17,951 - root - INFO - step: 79445 loss: 16.1118 memory: 44.58GiB(31.99%) tps: 82,673 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6453 +[titan] 2025-07-10 07:03:17,951 - root - INFO - lr: 5.7714e-05 gnorm: 0.97 [17:28:15< 4:31:13] +[titan] 2025-07-10 07:03:21,071 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:03:21,862 - root - INFO - step: 79450 loss: 16.0928 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.6168 +[titan] 2025-07-10 07:03:21,862 - root - INFO - lr: 5.7701e-05 gnorm: 1.03 [17:28:19< 4:31:09] +[titan] 2025-07-10 07:03:25,762 - root - INFO - step: 79455 loss: 16.3597 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8678 +[titan] 2025-07-10 07:03:25,762 - root - INFO - lr: 5.7688e-05 gnorm: 0.98 [17:28:23< 4:31:05] +[titan] 2025-07-10 07:03:29,695 - root - INFO - step: 79460 loss: 16.3483 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-10 07:03:29,696 - root - INFO - lr: 5.7675e-05 gnorm: 0.95 [17:28:27< 4:31:01] +[titan] 2025-07-10 07:03:33,612 - root - INFO - step: 79465 loss: 16.4020 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8929 +[titan] 2025-07-10 07:03:33,612 - root - INFO - lr: 5.7662e-05 gnorm: 0.98 [17:28:30< 4:30:57] +[titan] 2025-07-10 07:03:37,541 - root - INFO - step: 79470 loss: 16.2525 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-10 07:03:37,541 - root - INFO - lr: 5.7649e-05 gnorm: 0.92 [17:28:34< 4:30:53] +[titan] 2025-07-10 07:03:41,512 - root - INFO - step: 79475 loss: 16.1764 memory: 44.58GiB(31.99%) tps: 82,522 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.7063 +[titan] 2025-07-10 07:03:41,512 - root - INFO - lr: 5.7637e-05 gnorm: 0.93 [17:28:38< 4:30:49] +[titan] 2025-07-10 07:03:45,432 - root - INFO - step: 79480 loss: 16.1536 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.6767 +[titan] 2025-07-10 07:03:45,432 - root - INFO - lr: 5.7624e-05 gnorm: 1.00 [17:28:42< 4:30:45] +[titan] 2025-07-10 07:03:49,381 - root - INFO - step: 79485 loss: 16.2082 memory: 44.58GiB(31.99%) tps: 82,978 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7335 +[titan] 2025-07-10 07:03:49,382 - root - INFO - lr: 5.7611e-05 gnorm: 0.95 [17:28:46< 4:30:41] +[titan] 2025-07-10 07:03:53,338 - root - INFO - step: 79490 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8902 +[titan] 2025-07-10 07:03:53,338 - root - INFO - lr: 5.7598e-05 gnorm: 1.01 [17:28:50< 4:30:37] +[titan] 2025-07-10 07:03:57,274 - root - INFO - step: 79495 loss: 16.1466 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.6837 +[titan] 2025-07-10 07:03:57,275 - root - INFO - lr: 5.7585e-05 gnorm: 0.97 [17:28:54< 4:30:33] +[titan] 2025-07-10 07:04:00,376 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:04:01,176 - root - INFO - step: 79500 loss: 16.4249 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.9161 +[titan] 2025-07-10 07:04:01,176 - root - INFO - lr: 5.7572e-05 gnorm: 1.01 [17:28:58< 4:30:29] +[titan] 2025-07-10 07:04:05,105 - root - INFO - step: 79505 loss: 16.3159 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8164 +[titan] 2025-07-10 07:04:05,105 - root - INFO - lr: 5.7559e-05 gnorm: 0.92 [17:29:02< 4:30:25] +[titan] 2025-07-10 07:04:09,050 - root - INFO - step: 79510 loss: 16.2951 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.8116 +[titan] 2025-07-10 07:04:09,051 - root - INFO - lr: 5.7546e-05 gnorm: 0.98 [17:29:06< 4:30:21] +[titan] 2025-07-10 07:04:13,011 - root - INFO - step: 79515 loss: 16.2351 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7556 +[titan] 2025-07-10 07:04:13,011 - root - INFO - lr: 5.7533e-05 gnorm: 0.95 [17:29:10< 4:30:17] +[titan] 2025-07-10 07:04:16,930 - root - INFO - step: 79520 loss: 16.4535 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5255 global_avg_mtp_loss: 13.9281 +[titan] 2025-07-10 07:04:16,930 - root - INFO - lr: 5.7520e-05 gnorm: 1.00 [17:29:14< 4:30:13] +[titan] 2025-07-10 07:04:20,835 - root - INFO - step: 79525 loss: 16.4128 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8906 +[titan] 2025-07-10 07:04:20,836 - root - INFO - lr: 5.7507e-05 gnorm: 0.93 [17:29:18< 4:30:09] +[titan] 2025-07-10 07:04:24,755 - root - INFO - step: 79530 loss: 16.0842 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4483 global_avg_mtp_loss: 13.6359 +[titan] 2025-07-10 07:04:24,755 - root - INFO - lr: 5.7494e-05 gnorm: 0.92 [17:29:22< 4:30:05] +[titan] 2025-07-10 07:04:28,681 - root - INFO - step: 79535 loss: 16.4718 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5256 global_avg_mtp_loss: 13.9462 +[titan] 2025-07-10 07:04:28,681 - root - INFO - lr: 5.7481e-05 gnorm: 0.92 [17:29:25< 4:30:01] +[titan] 2025-07-10 07:04:32,594 - root - INFO - step: 79540 loss: 16.3146 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8218 +[titan] 2025-07-10 07:04:32,594 - root - INFO - lr: 5.7468e-05 gnorm: 0.92 [17:29:29< 4:29:57] +[titan] 2025-07-10 07:04:36,534 - root - INFO - step: 79545 loss: 16.0032 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4326 global_avg_mtp_loss: 13.5707 +[titan] 2025-07-10 07:04:36,534 - root - INFO - lr: 5.7455e-05 gnorm: 0.96 [17:29:33< 4:29:53] +[titan] 2025-07-10 07:04:39,651 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:04:40,439 - root - INFO - step: 79550 loss: 16.0505 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.6125 +[titan] 2025-07-10 07:04:40,439 - root - INFO - lr: 5.7442e-05 gnorm: 0.92 [17:29:37< 4:29:49] +[titan] 2025-07-10 07:04:44,349 - root - INFO - step: 79555 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8293 +[titan] 2025-07-10 07:04:44,349 - root - INFO - lr: 5.7429e-05 gnorm: 0.96 [17:29:41< 4:29:45] +[titan] 2025-07-10 07:04:48,248 - root - INFO - step: 79560 loss: 16.1565 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.6844 +[titan] 2025-07-10 07:04:48,249 - root - INFO - lr: 5.7416e-05 gnorm: 0.96 [17:29:45< 4:29:41] +[titan] 2025-07-10 07:04:52,160 - root - INFO - step: 79565 loss: 16.1249 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.6602 +[titan] 2025-07-10 07:04:52,160 - root - INFO - lr: 5.7403e-05 gnorm: 0.98 [17:29:49< 4:29:37] +[titan] 2025-07-10 07:04:56,081 - root - INFO - step: 79570 loss: 16.2491 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7673 +[titan] 2025-07-10 07:04:56,082 - root - INFO - lr: 5.7390e-05 gnorm: 0.97 [17:29:53< 4:29:33] +[titan] 2025-07-10 07:04:59,986 - root - INFO - step: 79575 loss: 16.0862 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6344 +[titan] 2025-07-10 07:04:59,987 - root - INFO - lr: 5.7377e-05 gnorm: 0.91 [17:29:57< 4:29:29] +[titan] 2025-07-10 07:05:03,912 - root - INFO - step: 79580 loss: 15.9429 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4333 global_avg_mtp_loss: 13.5096 +[titan] 2025-07-10 07:05:03,912 - root - INFO - lr: 5.7364e-05 gnorm: 0.97 [17:30:01< 4:29:25] +[titan] 2025-07-10 07:05:07,823 - root - INFO - step: 79585 loss: 16.2887 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 07:05:07,823 - root - INFO - lr: 5.7351e-05 gnorm: 0.96 [17:30:05< 4:29:21] +[titan] 2025-07-10 07:05:11,749 - root - INFO - step: 79590 loss: 16.3761 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8710 +[titan] 2025-07-10 07:05:11,750 - root - INFO - lr: 5.7339e-05 gnorm: 0.93 [17:30:09< 4:29:17] +[titan] 2025-07-10 07:05:15,691 - root - INFO - step: 79595 loss: 16.3202 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 07:05:15,692 - root - INFO - lr: 5.7326e-05 gnorm: 0.97 [17:30:12< 4:29:14] +[titan] 2025-07-10 07:05:18,816 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:05:19,606 - root - INFO - step: 79600 loss: 16.4298 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.9194 +[titan] 2025-07-10 07:05:19,606 - root - INFO - lr: 5.7313e-05 gnorm: 0.94 [17:30:16< 4:29:10] +[titan] 2025-07-10 07:05:23,547 - root - INFO - step: 79605 loss: 16.2993 memory: 44.58GiB(31.99%) tps: 83,156 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.7861 +[titan] 2025-07-10 07:05:23,547 - root - INFO - lr: 5.7300e-05 gnorm: 0.95 [17:30:20< 4:29:06] +[titan] 2025-07-10 07:05:27,439 - root - INFO - step: 79610 loss: 16.2482 memory: 44.58GiB(31.99%) tps: 84,189 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7622 +[titan] 2025-07-10 07:05:27,439 - root - INFO - lr: 5.7287e-05 gnorm: 0.97 [17:30:24< 4:29:02] +[titan] 2025-07-10 07:05:31,360 - root - INFO - step: 79615 loss: 16.3306 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.8411 +[titan] 2025-07-10 07:05:31,360 - root - INFO - lr: 5.7274e-05 gnorm: 0.97 [17:30:28< 4:28:58] +[titan] 2025-07-10 07:05:35,289 - root - INFO - step: 79620 loss: 16.3225 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 07:05:35,289 - root - INFO - lr: 5.7261e-05 gnorm: 0.95 [17:30:32< 4:28:54] +[titan] 2025-07-10 07:05:39,200 - root - INFO - step: 79625 loss: 16.2263 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7570 +[titan] 2025-07-10 07:05:39,200 - root - INFO - lr: 5.7248e-05 gnorm: 0.96 [17:30:36< 4:28:50] +[titan] 2025-07-10 07:05:43,106 - root - INFO - step: 79630 loss: 15.9643 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4397 global_avg_mtp_loss: 13.5246 +[titan] 2025-07-10 07:05:43,106 - root - INFO - lr: 5.7235e-05 gnorm: 0.97 [17:30:40< 4:28:46] +[titan] 2025-07-10 07:05:47,082 - root - INFO - step: 79635 loss: 16.3267 memory: 44.58GiB(31.99%) tps: 82,437 tflops: 284.50 mfu: 28.77% global_avg_ntp_loss: 2.4941 global_avg_mtp_loss: 13.8326 +[titan] 2025-07-10 07:05:47,082 - root - INFO - lr: 5.7222e-05 gnorm: 1.02 [17:30:44< 4:28:42] +[titan] 2025-07-10 07:05:51,007 - root - INFO - step: 79640 loss: 16.0788 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6258 +[titan] 2025-07-10 07:05:51,007 - root - INFO - lr: 5.7209e-05 gnorm: 1.03 [17:30:48< 4:28:38] +[titan] 2025-07-10 07:05:54,976 - root - INFO - step: 79645 loss: 16.2994 memory: 44.58GiB(31.99%) tps: 82,564 tflops: 284.94 mfu: 28.81% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8083 +[titan] 2025-07-10 07:05:54,976 - root - INFO - lr: 5.7197e-05 gnorm: 0.99 [17:30:52< 4:28:34] +[titan] 2025-07-10 07:05:58,131 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:05:58,917 - root - INFO - step: 79650 loss: 16.1590 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4536 global_avg_mtp_loss: 13.7053 +[titan] 2025-07-10 07:05:58,917 - root - INFO - lr: 5.7184e-05 gnorm: 1.04 [17:30:56< 4:28:30] +[titan] 2025-07-10 07:06:02,835 - root - INFO - step: 79655 loss: 16.2256 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7392 +[titan] 2025-07-10 07:06:02,836 - root - INFO - lr: 5.7171e-05 gnorm: 0.97 [17:31:00< 4:28:26] +[titan] 2025-07-10 07:06:06,804 - root - INFO - step: 79660 loss: 16.4921 memory: 44.58GiB(31.99%) tps: 82,577 tflops: 284.99 mfu: 28.82% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 13.9554 +[titan] 2025-07-10 07:06:06,804 - root - INFO - lr: 5.7158e-05 gnorm: 0.97 [17:31:04< 4:28:22] +[titan] 2025-07-10 07:06:10,725 - root - INFO - step: 79665 loss: 16.0945 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6295 +[titan] 2025-07-10 07:06:10,726 - root - INFO - lr: 5.7145e-05 gnorm: 1.04 [17:31:08< 4:28:18] +[titan] 2025-07-10 07:06:14,651 - root - INFO - step: 79670 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.7904 +[titan] 2025-07-10 07:06:14,651 - root - INFO - lr: 5.7132e-05 gnorm: 0.97 [17:31:11< 4:28:14] +[titan] 2025-07-10 07:06:18,574 - root - INFO - step: 79675 loss: 16.2642 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.7627 +[titan] 2025-07-10 07:06:18,574 - root - INFO - lr: 5.7119e-05 gnorm: 1.01 [17:31:15< 4:28:10] +[titan] 2025-07-10 07:06:22,501 - root - INFO - step: 79680 loss: 16.2686 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7790 +[titan] 2025-07-10 07:06:22,501 - root - INFO - lr: 5.7106e-05 gnorm: 1.03 [17:31:19< 4:28:06] +[titan] 2025-07-10 07:06:26,401 - root - INFO - step: 79685 loss: 16.4022 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8980 +[titan] 2025-07-10 07:06:26,401 - root - INFO - lr: 5.7094e-05 gnorm: 0.98 [17:31:23< 4:28:02] +[titan] 2025-07-10 07:06:30,315 - root - INFO - step: 79690 loss: 16.1347 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6647 +[titan] 2025-07-10 07:06:30,316 - root - INFO - lr: 5.7081e-05 gnorm: 1.01 [17:31:27< 4:27:58] +[titan] 2025-07-10 07:06:34,237 - root - INFO - step: 79695 loss: 15.8520 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4341 global_avg_mtp_loss: 13.4179 +[titan] 2025-07-10 07:06:34,237 - root - INFO - lr: 5.7068e-05 gnorm: 1.01 [17:31:31< 4:27:54] +[titan] 2025-07-10 07:06:37,350 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:06:38,141 - root - INFO - step: 79700 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8461 +[titan] 2025-07-10 07:06:38,141 - root - INFO - lr: 5.7055e-05 gnorm: 0.96 [17:31:35< 4:27:50] +[titan] 2025-07-10 07:06:42,097 - root - INFO - step: 79705 loss: 16.3487 memory: 44.58GiB(31.99%) tps: 82,836 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.8573 +[titan] 2025-07-10 07:06:42,097 - root - INFO - lr: 5.7042e-05 gnorm: 0.94 [17:31:39< 4:27:46] +[titan] 2025-07-10 07:06:46,040 - root - INFO - step: 79710 loss: 16.1959 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7132 +[titan] 2025-07-10 07:06:46,040 - root - INFO - lr: 5.7029e-05 gnorm: 0.97 [17:31:43< 4:27:42] +[titan] 2025-07-10 07:06:49,964 - root - INFO - step: 79715 loss: 16.3183 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.8355 +[titan] 2025-07-10 07:06:49,964 - root - INFO - lr: 5.7016e-05 gnorm: 0.95 [17:31:47< 4:27:38] +[titan] 2025-07-10 07:06:53,892 - root - INFO - step: 79720 loss: 16.2240 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7445 +[titan] 2025-07-10 07:06:53,892 - root - INFO - lr: 5.7003e-05 gnorm: 0.95 [17:31:51< 4:27:34] +[titan] 2025-07-10 07:06:57,819 - root - INFO - step: 79725 loss: 16.3134 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.8243 +[titan] 2025-07-10 07:06:57,820 - root - INFO - lr: 5.6991e-05 gnorm: 0.96 [17:31:55< 4:27:30] +[titan] 2025-07-10 07:07:01,757 - root - INFO - step: 79730 loss: 16.0522 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.5922 +[titan] 2025-07-10 07:07:01,757 - root - INFO - lr: 5.6978e-05 gnorm: 0.96 [17:31:59< 4:27:26] +[titan] 2025-07-10 07:07:05,671 - root - INFO - step: 79735 loss: 16.3637 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8552 +[titan] 2025-07-10 07:07:05,672 - root - INFO - lr: 5.6965e-05 gnorm: 0.99 [17:32:02< 4:27:22] +[titan] 2025-07-10 07:07:09,612 - root - INFO - step: 79740 loss: 16.3386 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-10 07:07:09,612 - root - INFO - lr: 5.6952e-05 gnorm: 0.99 [17:32:06< 4:27:19] +[titan] 2025-07-10 07:07:13,556 - root - INFO - step: 79745 loss: 16.5678 memory: 44.58GiB(31.99%) tps: 83,083 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5495 global_avg_mtp_loss: 14.0183 +[titan] 2025-07-10 07:07:13,557 - root - INFO - lr: 5.6939e-05 gnorm: 0.97 [17:32:10< 4:27:15] +[titan] 2025-07-10 07:07:16,674 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:07:17,463 - root - INFO - step: 79750 loss: 16.0913 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.6486 +[titan] 2025-07-10 07:07:17,463 - root - INFO - lr: 5.6926e-05 gnorm: 1.01 [17:32:14< 4:27:11] +[titan] 2025-07-10 07:07:21,363 - root - INFO - step: 79755 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7221 +[titan] 2025-07-10 07:07:21,363 - root - INFO - lr: 5.6914e-05 gnorm: 0.98 [17:32:18< 4:27:07] +[titan] 2025-07-10 07:07:25,274 - root - INFO - step: 79760 loss: 16.2957 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 07:07:25,274 - root - INFO - lr: 5.6901e-05 gnorm: 0.93 [17:32:22< 4:27:03] +[titan] 2025-07-10 07:07:29,228 - root - INFO - step: 79765 loss: 15.8079 memory: 44.58GiB(31.99%) tps: 82,878 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.3975 global_avg_mtp_loss: 13.4104 +[titan] 2025-07-10 07:07:29,228 - root - INFO - lr: 5.6888e-05 gnorm: 0.99 [17:32:26< 4:26:59] +[titan] 2025-07-10 07:07:33,136 - root - INFO - step: 79770 loss: 15.9433 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4299 global_avg_mtp_loss: 13.5134 +[titan] 2025-07-10 07:07:33,136 - root - INFO - lr: 5.6875e-05 gnorm: 0.95 [17:32:30< 4:26:55] +[titan] 2025-07-10 07:07:37,054 - root - INFO - step: 79775 loss: 16.2579 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7769 +[titan] 2025-07-10 07:07:37,054 - root - INFO - lr: 5.6862e-05 gnorm: 0.95 [17:32:34< 4:26:51] +[titan] 2025-07-10 07:07:40,956 - root - INFO - step: 79780 loss: 15.9937 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4386 global_avg_mtp_loss: 13.5551 +[titan] 2025-07-10 07:07:40,956 - root - INFO - lr: 5.6849e-05 gnorm: 1.00 [17:32:38< 4:26:47] +[titan] 2025-07-10 07:07:44,881 - root - INFO - step: 79785 loss: 16.4057 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8989 +[titan] 2025-07-10 07:07:44,881 - root - INFO - lr: 5.6837e-05 gnorm: 0.95 [17:32:42< 4:26:43] +[titan] 2025-07-10 07:07:48,802 - root - INFO - step: 79790 loss: 16.2236 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.7421 +[titan] 2025-07-10 07:07:48,803 - root - INFO - lr: 5.6824e-05 gnorm: 0.98 [17:32:46< 4:26:39] +[titan] 2025-07-10 07:07:52,701 - root - INFO - step: 79795 loss: 16.1793 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.6961 +[titan] 2025-07-10 07:07:52,701 - root - INFO - lr: 5.6811e-05 gnorm: 0.93 [17:32:49< 4:26:35] +[titan] 2025-07-10 07:07:55,822 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:07:56,606 - root - INFO - step: 79800 loss: 16.1670 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.7030 +[titan] 2025-07-10 07:07:56,606 - root - INFO - lr: 5.6798e-05 gnorm: 0.99 [17:32:53< 4:26:31] +[titan] 2025-07-10 07:08:00,543 - root - INFO - step: 79805 loss: 16.1610 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.6880 +[titan] 2025-07-10 07:08:00,544 - root - INFO - lr: 5.6785e-05 gnorm: 0.99 [17:32:57< 4:26:27] +[titan] 2025-07-10 07:08:04,452 - root - INFO - step: 79810 loss: 16.2030 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7261 +[titan] 2025-07-10 07:08:04,452 - root - INFO - lr: 5.6773e-05 gnorm: 0.93 [17:33:01< 4:26:23] +[titan] 2025-07-10 07:08:08,362 - root - INFO - step: 79815 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 07:08:08,362 - root - INFO - lr: 5.6760e-05 gnorm: 0.98 [17:33:05< 4:26:19] +[titan] 2025-07-10 07:08:12,266 - root - INFO - step: 79820 loss: 15.9705 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4310 global_avg_mtp_loss: 13.5395 +[titan] 2025-07-10 07:08:12,266 - root - INFO - lr: 5.6747e-05 gnorm: 0.94 [17:33:09< 4:26:15] +[titan] 2025-07-10 07:08:16,197 - root - INFO - step: 79825 loss: 16.5224 memory: 44.58GiB(31.99%) tps: 83,368 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.5411 global_avg_mtp_loss: 13.9813 +[titan] 2025-07-10 07:08:16,197 - root - INFO - lr: 5.6734e-05 gnorm: 0.96 [17:33:13< 4:26:11] +[titan] 2025-07-10 07:08:20,106 - root - INFO - step: 79830 loss: 16.0690 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.6157 +[titan] 2025-07-10 07:08:20,107 - root - INFO - lr: 5.6721e-05 gnorm: 0.97 [17:33:17< 4:26:07] +[titan] 2025-07-10 07:08:24,038 - root - INFO - step: 79835 loss: 15.8690 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4263 global_avg_mtp_loss: 13.4427 +[titan] 2025-07-10 07:08:24,038 - root - INFO - lr: 5.6709e-05 gnorm: 1.06 [17:33:21< 4:26:03] +[titan] 2025-07-10 07:08:27,942 - root - INFO - step: 79840 loss: 16.0361 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4529 global_avg_mtp_loss: 13.5832 +[titan] 2025-07-10 07:08:27,942 - root - INFO - lr: 5.6696e-05 gnorm: 0.95 [17:33:25< 4:25:59] +[titan] 2025-07-10 07:08:31,914 - root - INFO - step: 79845 loss: 16.3181 memory: 44.58GiB(31.99%) tps: 82,496 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8188 +[titan] 2025-07-10 07:08:31,915 - root - INFO - lr: 5.6683e-05 gnorm: 0.96 [17:33:29< 4:25:55] +[titan] 2025-07-10 07:08:35,031 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:08:35,824 - root - INFO - step: 79850 loss: 16.3690 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 07:08:35,824 - root - INFO - lr: 5.6670e-05 gnorm: 1.03 [17:33:33< 4:25:51] +[titan] 2025-07-10 07:08:39,733 - root - INFO - step: 79855 loss: 16.1838 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.7170 +[titan] 2025-07-10 07:08:39,733 - root - INFO - lr: 5.6657e-05 gnorm: 0.98 [17:33:37< 4:25:47] +[titan] 2025-07-10 07:08:43,654 - root - INFO - step: 79860 loss: 16.2643 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7924 +[titan] 2025-07-10 07:08:43,654 - root - INFO - lr: 5.6645e-05 gnorm: 0.98 [17:33:40< 4:25:43] +[titan] 2025-07-10 07:08:47,595 - root - INFO - step: 79865 loss: 16.3130 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.8256 +[titan] 2025-07-10 07:08:47,596 - root - INFO - lr: 5.6632e-05 gnorm: 0.97 [17:33:44< 4:25:39] +[titan] 2025-07-10 07:08:51,579 - root - INFO - step: 79870 loss: 16.1721 memory: 44.58GiB(31.99%) tps: 82,255 tflops: 283.87 mfu: 28.70% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6926 +[titan] 2025-07-10 07:08:51,580 - root - INFO - lr: 5.6619e-05 gnorm: 1.01 [17:33:48< 4:25:35] +[titan] 2025-07-10 07:08:53,296 - root - INFO - Dumping profiler traces at step 79872 +[titan] 2025-07-10 07:08:53,330 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:08:55,696 - root - INFO - step: 79875 loss: 16.2634 memory: 44.58GiB(31.99%) tps: 79,610 tflops: 274.75 mfu: 27.78% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7834 +[titan] 2025-07-10 07:08:55,696 - root - INFO - lr: 5.6606e-05 gnorm: 0.94 [17:33:52< 4:25:31] +[titan] 2025-07-10 07:08:59,594 - root - INFO - step: 79880 loss: 16.2466 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.7571 +[titan] 2025-07-10 07:08:59,594 - root - INFO - lr: 5.6594e-05 gnorm: 0.98 [17:33:56< 4:25:27] +[titan] 2025-07-10 07:09:03,545 - root - INFO - step: 79885 loss: 16.3568 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.8536 +[titan] 2025-07-10 07:09:03,545 - root - INFO - lr: 5.6581e-05 gnorm: 0.99 [17:34:00< 4:25:24] +[titan] 2025-07-10 07:09:07,485 - root - INFO - step: 79890 loss: 16.2019 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7282 +[titan] 2025-07-10 07:09:07,485 - root - INFO - lr: 5.6568e-05 gnorm: 0.96 [17:34:04< 4:25:20] +[titan] 2025-07-10 07:09:11,412 - root - INFO - step: 79895 loss: 16.1697 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.6909 +[titan] 2025-07-10 07:09:11,412 - root - INFO - lr: 5.6555e-05 gnorm: 1.03 [17:34:08< 4:25:16] +[titan] 2025-07-10 07:09:14,552 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:09:15,339 - root - INFO - step: 79900 loss: 16.4558 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.5125 global_avg_mtp_loss: 13.9432 +[titan] 2025-07-10 07:09:15,340 - root - INFO - lr: 5.6543e-05 gnorm: 0.98 [17:34:12< 4:25:12] +[titan] 2025-07-10 07:09:19,251 - root - INFO - step: 79905 loss: 16.0818 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4454 global_avg_mtp_loss: 13.6365 +[titan] 2025-07-10 07:09:19,251 - root - INFO - lr: 5.6530e-05 gnorm: 0.96 [17:34:16< 4:25:08] +[titan] 2025-07-10 07:09:23,174 - root - INFO - step: 79910 loss: 16.2722 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.7877 +[titan] 2025-07-10 07:09:23,174 - root - INFO - lr: 5.6517e-05 gnorm: 0.95 [17:34:20< 4:25:04] +[titan] 2025-07-10 07:09:27,132 - root - INFO - step: 79915 loss: 16.3493 memory: 44.58GiB(31.99%) tps: 82,789 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8496 +[titan] 2025-07-10 07:09:27,133 - root - INFO - lr: 5.6504e-05 gnorm: 0.93 [17:34:24< 4:25:00] +[titan] 2025-07-10 07:09:31,034 - root - INFO - step: 79920 loss: 16.1261 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6568 +[titan] 2025-07-10 07:09:31,034 - root - INFO - lr: 5.6492e-05 gnorm: 0.92 [17:34:28< 4:24:56] +[titan] 2025-07-10 07:09:34,948 - root - INFO - step: 79925 loss: 16.2448 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7660 +[titan] 2025-07-10 07:09:34,948 - root - INFO - lr: 5.6479e-05 gnorm: 0.94 [17:34:32< 4:24:52] +[titan] 2025-07-10 07:09:38,859 - root - INFO - step: 79930 loss: 16.3584 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8497 +[titan] 2025-07-10 07:09:38,859 - root - INFO - lr: 5.6466e-05 gnorm: 0.97 [17:34:36< 4:24:48] +[titan] 2025-07-10 07:09:42,808 - root - INFO - step: 79935 loss: 16.4322 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.9176 +[titan] 2025-07-10 07:09:42,808 - root - INFO - lr: 5.6453e-05 gnorm: 0.97 [17:34:40< 4:24:44] +[titan] 2025-07-10 07:09:46,745 - root - INFO - step: 79940 loss: 16.2494 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.7537 +[titan] 2025-07-10 07:09:46,746 - root - INFO - lr: 5.6441e-05 gnorm: 0.99 [17:34:44< 4:24:40] +[titan] 2025-07-10 07:09:50,656 - root - INFO - step: 79945 loss: 16.2754 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7980 +[titan] 2025-07-10 07:09:50,656 - root - INFO - lr: 5.6428e-05 gnorm: 0.97 [17:34:47< 4:24:36] +[titan] 2025-07-10 07:09:53,782 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:09:54,573 - root - INFO - step: 79950 loss: 16.1126 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6537 +[titan] 2025-07-10 07:09:54,573 - root - INFO - lr: 5.6415e-05 gnorm: 0.97 [17:34:51< 4:24:32] +[titan] 2025-07-10 07:09:58,544 - root - INFO - step: 79955 loss: 16.3801 memory: 44.58GiB(31.99%) tps: 82,522 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8819 +[titan] 2025-07-10 07:09:58,544 - root - INFO - lr: 5.6402e-05 gnorm: 1.01 [17:34:55< 4:24:28] +[titan] 2025-07-10 07:10:02,476 - root - INFO - step: 79960 loss: 16.3724 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8635 +[titan] 2025-07-10 07:10:02,476 - root - INFO - lr: 5.6390e-05 gnorm: 0.96 [17:34:59< 4:24:24] +[titan] 2025-07-10 07:10:06,384 - root - INFO - step: 79965 loss: 16.3187 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.8276 +[titan] 2025-07-10 07:10:06,385 - root - INFO - lr: 5.6377e-05 gnorm: 0.92 [17:35:03< 4:24:20] +[titan] 2025-07-10 07:10:10,309 - root - INFO - step: 79970 loss: 16.3900 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.8727 +[titan] 2025-07-10 07:10:10,310 - root - INFO - lr: 5.6364e-05 gnorm: 0.97 [17:35:07< 4:24:16] +[titan] 2025-07-10 07:10:14,242 - root - INFO - step: 79975 loss: 16.2317 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7416 +[titan] 2025-07-10 07:10:14,242 - root - INFO - lr: 5.6352e-05 gnorm: 0.99 [17:35:11< 4:24:12] +[titan] 2025-07-10 07:10:18,148 - root - INFO - step: 79980 loss: 16.2380 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.7591 +[titan] 2025-07-10 07:10:18,148 - root - INFO - lr: 5.6339e-05 gnorm: 0.96 [17:35:15< 4:24:08] +[titan] 2025-07-10 07:10:22,059 - root - INFO - step: 79985 loss: 16.3209 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.8323 +[titan] 2025-07-10 07:10:22,059 - root - INFO - lr: 5.6326e-05 gnorm: 0.97 [17:35:19< 4:24:04] +[titan] 2025-07-10 07:10:25,960 - root - INFO - step: 79990 loss: 16.0996 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6376 +[titan] 2025-07-10 07:10:25,960 - root - INFO - lr: 5.6313e-05 gnorm: 0.93 [17:35:23< 4:24:00] +[titan] 2025-07-10 07:10:29,876 - root - INFO - step: 79995 loss: 16.1633 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.6965 +[titan] 2025-07-10 07:10:29,877 - root - INFO - lr: 5.6301e-05 gnorm: 1.00 [17:35:27< 4:23:56] +[titan] 2025-07-10 07:10:32,991 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:10:33,786 - root - INFO - step: 80000 loss: 16.2686 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7847 +[titan] 2025-07-10 07:10:33,786 - root - INFO - lr: 5.6288e-05 gnorm: 0.96 [17:35:31< 4:23:52] +[titan] 2025-07-10 07:10:33,786 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 07:10:36,262 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 07:10:36,262 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.48 seconds. +[titan] 2025-07-10 07:11:54,032 - root - INFO - step: 80005 loss: 16.3942 memory: 44.58GiB(31.99%) tps: 4,083 tflops: 14.09 mfu: 1.42% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8841 +[titan] 2025-07-10 07:11:54,032 - root - INFO - lr: 5.6275e-05 gnorm: 0.95 [17:36:51< 4:24:07] +[titan] 2025-07-10 07:11:57,931 - root - INFO - step: 80010 loss: 16.2049 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7221 +[titan] 2025-07-10 07:11:57,931 - root - INFO - lr: 5.6263e-05 gnorm: 0.98 [17:36:55< 4:24:03] +[titan] 2025-07-10 07:12:01,833 - root - INFO - step: 80015 loss: 16.1526 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.6818 +[titan] 2025-07-10 07:12:01,834 - root - INFO - lr: 5.6250e-05 gnorm: 0.96 [17:36:59< 4:23:59] +[titan] 2025-07-10 07:12:05,741 - root - INFO - step: 80020 loss: 16.2789 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7861 +[titan] 2025-07-10 07:12:05,741 - root - INFO - lr: 5.6237e-05 gnorm: 0.97 [17:37:03< 4:23:55] +[titan] 2025-07-10 07:12:09,658 - root - INFO - step: 80025 loss: 16.0121 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5704 +[titan] 2025-07-10 07:12:09,658 - root - INFO - lr: 5.6225e-05 gnorm: 0.94 [17:37:06< 4:23:51] +[titan] 2025-07-10 07:12:13,711 - root - INFO - step: 80030 loss: 16.4646 memory: 44.58GiB(31.99%) tps: 80,844 tflops: 279.01 mfu: 28.21% global_avg_ntp_loss: 2.5187 global_avg_mtp_loss: 13.9459 +[titan] 2025-07-10 07:12:13,712 - root - INFO - lr: 5.6212e-05 gnorm: 0.93 [17:37:10< 4:23:48] +[titan] 2025-07-10 07:12:17,624 - root - INFO - step: 80035 loss: 16.1399 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.6769 +[titan] 2025-07-10 07:12:17,624 - root - INFO - lr: 5.6199e-05 gnorm: 0.99 [17:37:14< 4:23:44] +[titan] 2025-07-10 07:12:21,579 - root - INFO - step: 80040 loss: 16.3816 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8840 +[titan] 2025-07-10 07:12:21,579 - root - INFO - lr: 5.6186e-05 gnorm: 0.93 [17:37:18< 4:23:40] +[titan] 2025-07-10 07:12:25,521 - root - INFO - step: 80045 loss: 16.0540 memory: 44.58GiB(31.99%) tps: 83,143 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4460 global_avg_mtp_loss: 13.6080 +[titan] 2025-07-10 07:12:25,521 - root - INFO - lr: 5.6174e-05 gnorm: 0.96 [17:37:22< 4:23:36] +[titan] 2025-07-10 07:12:28,681 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:12:29,471 - root - INFO - step: 80050 loss: 16.2134 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.4714 global_avg_mtp_loss: 13.7420 +[titan] 2025-07-10 07:12:29,471 - root - INFO - lr: 5.6161e-05 gnorm: 0.93 [17:37:26< 4:23:32] +[titan] 2025-07-10 07:12:33,401 - root - INFO - step: 80055 loss: 16.0978 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.6462 +[titan] 2025-07-10 07:12:33,401 - root - INFO - lr: 5.6148e-05 gnorm: 0.95 [17:37:30< 4:23:28] +[titan] 2025-07-10 07:12:37,368 - root - INFO - step: 80060 loss: 16.2648 memory: 44.58GiB(31.99%) tps: 82,615 tflops: 285.12 mfu: 28.83% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7813 +[titan] 2025-07-10 07:12:37,368 - root - INFO - lr: 5.6136e-05 gnorm: 1.00 [17:37:34< 4:23:24] +[titan] 2025-07-10 07:12:41,332 - root - INFO - step: 80065 loss: 16.3168 memory: 44.58GiB(31.99%) tps: 82,664 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8270 +[titan] 2025-07-10 07:12:41,333 - root - INFO - lr: 5.6123e-05 gnorm: 0.93 [17:37:38< 4:23:20] +[titan] 2025-07-10 07:12:45,321 - root - INFO - step: 80070 loss: 16.2252 memory: 44.58GiB(31.99%) tps: 82,154 tflops: 283.53 mfu: 28.67% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7334 +[titan] 2025-07-10 07:12:45,322 - root - INFO - lr: 5.6110e-05 gnorm: 0.96 [17:37:42< 4:23:16] +[titan] 2025-07-10 07:12:49,249 - root - INFO - step: 80075 loss: 16.5527 memory: 44.58GiB(31.99%) tps: 83,437 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.5620 global_avg_mtp_loss: 13.9907 +[titan] 2025-07-10 07:12:49,249 - root - INFO - lr: 5.6098e-05 gnorm: 0.96 [17:37:46< 4:23:12] +[titan] 2025-07-10 07:12:53,197 - root - INFO - step: 80080 loss: 16.5068 memory: 44.58GiB(31.99%) tps: 83,011 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.9869 +[titan] 2025-07-10 07:12:53,197 - root - INFO - lr: 5.6085e-05 gnorm: 0.95 [17:37:50< 4:23:08] +[titan] 2025-07-10 07:12:57,111 - root - INFO - step: 80085 loss: 16.1223 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 07:12:57,111 - root - INFO - lr: 5.6073e-05 gnorm: 0.99 [17:37:54< 4:23:04] +[titan] 2025-07-10 07:13:01,067 - root - INFO - step: 80090 loss: 16.4014 memory: 44.58GiB(31.99%) tps: 82,836 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.5139 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 07:13:01,068 - root - INFO - lr: 5.6060e-05 gnorm: 0.96 [17:37:58< 4:23:00] +[titan] 2025-07-10 07:13:04,994 - root - INFO - step: 80095 loss: 16.0866 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6252 +[titan] 2025-07-10 07:13:04,995 - root - INFO - lr: 5.6047e-05 gnorm: 0.99 [17:38:02< 4:22:56] +[titan] 2025-07-10 07:13:08,119 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:13:08,908 - root - INFO - step: 80100 loss: 16.3315 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.8388 +[titan] 2025-07-10 07:13:08,909 - root - INFO - lr: 5.6035e-05 gnorm: 0.98 [17:38:06< 4:22:52] +[titan] 2025-07-10 07:13:12,845 - root - INFO - step: 80105 loss: 15.9895 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4347 global_avg_mtp_loss: 13.5547 +[titan] 2025-07-10 07:13:12,845 - root - INFO - lr: 5.6022e-05 gnorm: 0.93 [17:38:10< 4:22:48] +[titan] 2025-07-10 07:13:16,784 - root - INFO - step: 80110 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.8038 +[titan] 2025-07-10 07:13:16,784 - root - INFO - lr: 5.6009e-05 gnorm: 1.01 [17:38:14< 4:22:44] +[titan] 2025-07-10 07:13:20,687 - root - INFO - step: 80115 loss: 16.1668 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 07:13:20,688 - root - INFO - lr: 5.5997e-05 gnorm: 0.98 [17:38:17< 4:22:40] +[titan] 2025-07-10 07:13:24,606 - root - INFO - step: 80120 loss: 16.1088 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.6505 +[titan] 2025-07-10 07:13:24,606 - root - INFO - lr: 5.5984e-05 gnorm: 0.94 [17:38:21< 4:22:36] +[titan] 2025-07-10 07:13:28,545 - root - INFO - step: 80125 loss: 16.2067 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7199 +[titan] 2025-07-10 07:13:28,545 - root - INFO - lr: 5.5971e-05 gnorm: 1.01 [17:38:25< 4:22:32] +[titan] 2025-07-10 07:13:32,480 - root - INFO - step: 80130 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8774 +[titan] 2025-07-10 07:13:32,480 - root - INFO - lr: 5.5959e-05 gnorm: 0.92 [17:38:29< 4:22:28] +[titan] 2025-07-10 07:13:36,376 - root - INFO - step: 80135 loss: 16.2558 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 07:13:36,377 - root - INFO - lr: 5.5946e-05 gnorm: 0.97 [17:38:33< 4:22:24] +[titan] 2025-07-10 07:13:40,283 - root - INFO - step: 80140 loss: 15.9804 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4203 global_avg_mtp_loss: 13.5600 +[titan] 2025-07-10 07:13:40,283 - root - INFO - lr: 5.5933e-05 gnorm: 1.01 [17:38:37< 4:22:20] +[titan] 2025-07-10 07:13:44,201 - root - INFO - step: 80145 loss: 16.4725 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.9556 +[titan] 2025-07-10 07:13:44,202 - root - INFO - lr: 5.5921e-05 gnorm: 0.94 [17:38:41< 4:22:16] +[titan] 2025-07-10 07:13:47,318 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:13:48,112 - root - INFO - step: 80150 loss: 16.3254 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-10 07:13:48,112 - root - INFO - lr: 5.5908e-05 gnorm: 0.92 [17:38:45< 4:22:12] +[titan] 2025-07-10 07:13:52,017 - root - INFO - step: 80155 loss: 16.3112 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 07:13:52,017 - root - INFO - lr: 5.5896e-05 gnorm: 0.99 [17:38:49< 4:22:08] +[titan] 2025-07-10 07:13:55,955 - root - INFO - step: 80160 loss: 16.3290 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8338 +[titan] 2025-07-10 07:13:55,955 - root - INFO - lr: 5.5883e-05 gnorm: 0.96 [17:38:53< 4:22:04] +[titan] 2025-07-10 07:13:59,883 - root - INFO - step: 80165 loss: 16.1886 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7081 +[titan] 2025-07-10 07:13:59,884 - root - INFO - lr: 5.5870e-05 gnorm: 0.97 [17:38:57< 4:22:00] +[titan] 2025-07-10 07:14:03,804 - root - INFO - step: 80170 loss: 16.5558 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 14.0225 +[titan] 2025-07-10 07:14:03,804 - root - INFO - lr: 5.5858e-05 gnorm: 1.05 [17:39:01< 4:21:56] +[titan] 2025-07-10 07:14:07,693 - root - INFO - step: 80175 loss: 16.2648 memory: 44.58GiB(31.99%) tps: 84,259 tflops: 290.79 mfu: 29.40% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7686 +[titan] 2025-07-10 07:14:07,693 - root - INFO - lr: 5.5845e-05 gnorm: 1.00 [17:39:04< 4:21:52] +[titan] 2025-07-10 07:14:11,633 - root - INFO - step: 80180 loss: 16.1687 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.6992 +[titan] 2025-07-10 07:14:11,634 - root - INFO - lr: 5.5833e-05 gnorm: 0.99 [17:39:08< 4:21:48] +[titan] 2025-07-10 07:14:15,529 - root - INFO - step: 80185 loss: 16.3094 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8077 +[titan] 2025-07-10 07:14:15,529 - root - INFO - lr: 5.5820e-05 gnorm: 0.98 [17:39:12< 4:21:44] +[titan] 2025-07-10 07:14:19,425 - root - INFO - step: 80190 loss: 16.1470 memory: 44.58GiB(31.99%) tps: 84,122 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.4544 global_avg_mtp_loss: 13.6927 +[titan] 2025-07-10 07:14:19,425 - root - INFO - lr: 5.5807e-05 gnorm: 0.95 [17:39:16< 4:21:40] +[titan] 2025-07-10 07:14:23,339 - root - INFO - step: 80195 loss: 16.2375 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.7441 +[titan] 2025-07-10 07:14:23,339 - root - INFO - lr: 5.5795e-05 gnorm: 0.98 [17:39:20< 4:21:36] +[titan] 2025-07-10 07:14:26,461 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:14:27,257 - root - INFO - step: 80200 loss: 16.5461 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5598 global_avg_mtp_loss: 13.9863 +[titan] 2025-07-10 07:14:27,258 - root - INFO - lr: 5.5782e-05 gnorm: 1.00 [17:39:24< 4:21:32] +[titan] 2025-07-10 07:14:31,176 - root - INFO - step: 80205 loss: 16.1837 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7155 +[titan] 2025-07-10 07:14:31,177 - root - INFO - lr: 5.5770e-05 gnorm: 0.95 [17:39:28< 4:21:29] +[titan] 2025-07-10 07:14:35,115 - root - INFO - step: 80210 loss: 16.0719 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 07:14:35,115 - root - INFO - lr: 5.5757e-05 gnorm: 0.97 [17:39:32< 4:21:25] +[titan] 2025-07-10 07:14:39,019 - root - INFO - step: 80215 loss: 16.3777 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8794 +[titan] 2025-07-10 07:14:39,019 - root - INFO - lr: 5.5744e-05 gnorm: 0.93 [17:39:36< 4:21:21] +[titan] 2025-07-10 07:14:42,917 - root - INFO - step: 80220 loss: 16.2505 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7768 +[titan] 2025-07-10 07:14:42,917 - root - INFO - lr: 5.5732e-05 gnorm: 0.96 [17:39:40< 4:21:17] +[titan] 2025-07-10 07:14:46,859 - root - INFO - step: 80225 loss: 15.9254 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.4780 +[titan] 2025-07-10 07:14:46,859 - root - INFO - lr: 5.5719e-05 gnorm: 0.98 [17:39:44< 4:21:13] +[titan] 2025-07-10 07:14:50,767 - root - INFO - step: 80230 loss: 16.1691 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.7029 +[titan] 2025-07-10 07:14:50,768 - root - INFO - lr: 5.5707e-05 gnorm: 0.95 [17:39:48< 4:21:09] +[titan] 2025-07-10 07:14:54,705 - root - INFO - step: 80235 loss: 16.1039 memory: 44.58GiB(31.99%) tps: 83,224 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6502 +[titan] 2025-07-10 07:14:54,705 - root - INFO - lr: 5.5694e-05 gnorm: 0.95 [17:39:51< 4:21:05] +[titan] 2025-07-10 07:14:58,635 - root - INFO - step: 80240 loss: 16.4160 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.9073 +[titan] 2025-07-10 07:14:58,635 - root - INFO - lr: 5.5682e-05 gnorm: 0.96 [17:39:55< 4:21:01] +[titan] 2025-07-10 07:15:02,556 - root - INFO - step: 80245 loss: 16.3064 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8076 +[titan] 2025-07-10 07:15:02,556 - root - INFO - lr: 5.5669e-05 gnorm: 0.98 [17:39:59< 4:20:57] +[titan] 2025-07-10 07:15:05,684 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:15:06,474 - root - INFO - step: 80250 loss: 16.2675 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7799 +[titan] 2025-07-10 07:15:06,474 - root - INFO - lr: 5.5656e-05 gnorm: 0.99 [17:40:03< 4:20:53] +[titan] 2025-07-10 07:15:10,396 - root - INFO - step: 80255 loss: 16.2493 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7626 +[titan] 2025-07-10 07:15:10,396 - root - INFO - lr: 5.5644e-05 gnorm: 0.99 [17:40:07< 4:20:49] +[titan] 2025-07-10 07:15:14,340 - root - INFO - step: 80260 loss: 16.3040 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.8176 +[titan] 2025-07-10 07:15:14,340 - root - INFO - lr: 5.5631e-05 gnorm: 0.93 [17:40:11< 4:20:45] +[titan] 2025-07-10 07:15:18,248 - root - INFO - step: 80265 loss: 16.0027 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4446 global_avg_mtp_loss: 13.5582 +[titan] 2025-07-10 07:15:18,248 - root - INFO - lr: 5.5619e-05 gnorm: 0.96 [17:40:15< 4:20:41] +[titan] 2025-07-10 07:15:22,168 - root - INFO - step: 80270 loss: 16.2403 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7639 +[titan] 2025-07-10 07:15:22,168 - root - INFO - lr: 5.5606e-05 gnorm: 0.97 [17:40:19< 4:20:37] +[titan] 2025-07-10 07:15:26,117 - root - INFO - step: 80275 loss: 16.2399 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7656 +[titan] 2025-07-10 07:15:26,118 - root - INFO - lr: 5.5594e-05 gnorm: 0.93 [17:40:23< 4:20:33] +[titan] 2025-07-10 07:15:30,056 - root - INFO - step: 80280 loss: 16.1361 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6720 +[titan] 2025-07-10 07:15:30,056 - root - INFO - lr: 5.5581e-05 gnorm: 0.97 [17:40:27< 4:20:29] +[titan] 2025-07-10 07:15:34,009 - root - INFO - step: 80285 loss: 15.9129 memory: 44.58GiB(31.99%) tps: 82,890 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.4193 global_avg_mtp_loss: 13.4936 +[titan] 2025-07-10 07:15:34,009 - root - INFO - lr: 5.5569e-05 gnorm: 0.96 [17:40:31< 4:20:25] +[titan] 2025-07-10 07:15:37,934 - root - INFO - step: 80290 loss: 16.3381 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-10 07:15:37,935 - root - INFO - lr: 5.5556e-05 gnorm: 0.95 [17:40:35< 4:20:21] +[titan] 2025-07-10 07:15:41,841 - root - INFO - step: 80295 loss: 16.2117 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.7441 +[titan] 2025-07-10 07:15:41,842 - root - INFO - lr: 5.5543e-05 gnorm: 1.02 [17:40:39< 4:20:17] +[titan] 2025-07-10 07:15:44,977 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:15:45,768 - root - INFO - step: 80300 loss: 16.3613 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.8641 +[titan] 2025-07-10 07:15:45,768 - root - INFO - lr: 5.5531e-05 gnorm: 0.94 [17:40:43< 4:20:13] +[titan] 2025-07-10 07:15:49,676 - root - INFO - step: 80305 loss: 16.4345 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.9172 +[titan] 2025-07-10 07:15:49,676 - root - INFO - lr: 5.5518e-05 gnorm: 0.94 [17:40:46< 4:20:09] +[titan] 2025-07-10 07:15:53,581 - root - INFO - step: 80310 loss: 15.9851 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.5389 +[titan] 2025-07-10 07:15:53,581 - root - INFO - lr: 5.5506e-05 gnorm: 0.99 [17:40:50< 4:20:05] +[titan] 2025-07-10 07:15:57,504 - root - INFO - step: 80315 loss: 16.2048 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.7363 +[titan] 2025-07-10 07:15:57,505 - root - INFO - lr: 5.5493e-05 gnorm: 1.04 [17:40:54< 4:20:01] +[titan] 2025-07-10 07:16:01,435 - root - INFO - step: 80320 loss: 16.2657 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7850 +[titan] 2025-07-10 07:16:01,436 - root - INFO - lr: 5.5481e-05 gnorm: 0.96 [17:40:58< 4:19:57] +[titan] 2025-07-10 07:16:05,327 - root - INFO - step: 80325 loss: 16.1470 memory: 44.58GiB(31.99%) tps: 84,208 tflops: 290.62 mfu: 29.38% global_avg_ntp_loss: 2.4494 global_avg_mtp_loss: 13.6976 +[titan] 2025-07-10 07:16:05,328 - root - INFO - lr: 5.5468e-05 gnorm: 0.97 [17:41:02< 4:19:53] +[titan] 2025-07-10 07:16:09,265 - root - INFO - step: 80330 loss: 16.2341 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.7659 +[titan] 2025-07-10 07:16:09,265 - root - INFO - lr: 5.5456e-05 gnorm: 0.99 [17:41:06< 4:19:49] +[titan] 2025-07-10 07:16:13,167 - root - INFO - step: 80335 loss: 16.5412 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5629 global_avg_mtp_loss: 13.9783 +[titan] 2025-07-10 07:16:13,167 - root - INFO - lr: 5.5443e-05 gnorm: 1.05 [17:41:10< 4:19:45] +[titan] 2025-07-10 07:16:17,083 - root - INFO - step: 80340 loss: 16.1342 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 07:16:17,083 - root - INFO - lr: 5.5431e-05 gnorm: 0.98 [17:41:14< 4:19:41] +[titan] 2025-07-10 07:16:20,989 - root - INFO - step: 80345 loss: 16.2574 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7790 +[titan] 2025-07-10 07:16:20,990 - root - INFO - lr: 5.5418e-05 gnorm: 0.95 [17:41:18< 4:19:37] +[titan] 2025-07-10 07:16:24,114 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:16:24,904 - root - INFO - step: 80350 loss: 16.0191 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.5578 +[titan] 2025-07-10 07:16:24,904 - root - INFO - lr: 5.5406e-05 gnorm: 0.98 [17:41:22< 4:19:33] +[titan] 2025-07-10 07:16:28,807 - root - INFO - step: 80355 loss: 16.4114 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.8914 +[titan] 2025-07-10 07:16:28,807 - root - INFO - lr: 5.5393e-05 gnorm: 0.94 [17:41:26< 4:19:29] +[titan] 2025-07-10 07:16:32,760 - root - INFO - step: 80360 loss: 16.2195 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7393 +[titan] 2025-07-10 07:16:32,760 - root - INFO - lr: 5.5381e-05 gnorm: 0.95 [17:41:29< 4:19:25] +[titan] 2025-07-10 07:16:36,663 - root - INFO - step: 80365 loss: 16.2541 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7712 +[titan] 2025-07-10 07:16:36,663 - root - INFO - lr: 5.5368e-05 gnorm: 0.94 [17:41:33< 4:19:21] +[titan] 2025-07-10 07:16:40,563 - root - INFO - step: 80370 loss: 16.3937 memory: 44.58GiB(31.99%) tps: 84,036 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8979 +[titan] 2025-07-10 07:16:40,563 - root - INFO - lr: 5.5356e-05 gnorm: 0.95 [17:41:37< 4:19:17] +[titan] 2025-07-10 07:16:44,489 - root - INFO - step: 80375 loss: 16.2870 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.7896 +[titan] 2025-07-10 07:16:44,489 - root - INFO - lr: 5.5343e-05 gnorm: 0.95 [17:41:41< 4:19:13] +[titan] 2025-07-10 07:16:48,414 - root - INFO - step: 80380 loss: 16.1708 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.7219 +[titan] 2025-07-10 07:16:48,414 - root - INFO - lr: 5.5331e-05 gnorm: 0.95 [17:41:45< 4:19:09] +[titan] 2025-07-10 07:16:51,775 - root - INFO - Dumping profiler traces at step 80384 +[titan] 2025-07-10 07:16:51,809 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:16:52,599 - root - INFO - step: 80385 loss: 16.1598 memory: 44.58GiB(31.99%) tps: 78,297 tflops: 270.21 mfu: 27.32% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6940 +[titan] 2025-07-10 07:16:52,599 - root - INFO - lr: 5.5318e-05 gnorm: 0.98 [17:41:49< 4:19:06] +[titan] 2025-07-10 07:16:56,501 - root - INFO - step: 80390 loss: 16.1254 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.6558 +[titan] 2025-07-10 07:16:56,501 - root - INFO - lr: 5.5306e-05 gnorm: 0.94 [17:41:53< 4:19:02] +[titan] 2025-07-10 07:17:00,412 - root - INFO - step: 80395 loss: 16.1438 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.6698 +[titan] 2025-07-10 07:17:00,412 - root - INFO - lr: 5.5293e-05 gnorm: 1.07 [17:41:57< 4:18:58] +[titan] 2025-07-10 07:17:03,546 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:17:04,336 - root - INFO - step: 80400 loss: 16.3474 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8396 +[titan] 2025-07-10 07:17:04,336 - root - INFO - lr: 5.5281e-05 gnorm: 0.99 [17:42:01< 4:18:54] +[titan] 2025-07-10 07:17:08,289 - root - INFO - step: 80405 loss: 16.1930 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.7313 +[titan] 2025-07-10 07:17:08,289 - root - INFO - lr: 5.5268e-05 gnorm: 1.01 [17:42:05< 4:18:50] +[titan] 2025-07-10 07:17:12,246 - root - INFO - step: 80410 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 82,822 tflops: 285.83 mfu: 28.90% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8711 +[titan] 2025-07-10 07:17:12,246 - root - INFO - lr: 5.5256e-05 gnorm: 0.98 [17:42:09< 4:18:46] +[titan] 2025-07-10 07:17:16,167 - root - INFO - step: 80415 loss: 16.1283 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6669 +[titan] 2025-07-10 07:17:16,167 - root - INFO - lr: 5.5243e-05 gnorm: 0.98 [17:42:13< 4:18:42] +[titan] 2025-07-10 07:17:20,078 - root - INFO - step: 80420 loss: 16.1005 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6414 +[titan] 2025-07-10 07:17:20,079 - root - INFO - lr: 5.5231e-05 gnorm: 0.94 [17:42:17< 4:18:38] +[titan] 2025-07-10 07:17:23,977 - root - INFO - step: 80425 loss: 16.1358 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6776 +[titan] 2025-07-10 07:17:23,977 - root - INFO - lr: 5.5218e-05 gnorm: 0.95 [17:42:21< 4:18:34] +[titan] 2025-07-10 07:17:27,894 - root - INFO - step: 80430 loss: 16.3172 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 07:17:27,894 - root - INFO - lr: 5.5206e-05 gnorm: 0.94 [17:42:25< 4:18:30] +[titan] 2025-07-10 07:17:31,815 - root - INFO - step: 80435 loss: 16.2407 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7626 +[titan] 2025-07-10 07:17:31,816 - root - INFO - lr: 5.5194e-05 gnorm: 0.97 [17:42:29< 4:18:26] +[titan] 2025-07-10 07:17:35,717 - root - INFO - step: 80440 loss: 16.3061 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4983 global_avg_mtp_loss: 13.8078 +[titan] 2025-07-10 07:17:35,717 - root - INFO - lr: 5.5181e-05 gnorm: 1.01 [17:42:32< 4:18:22] +[titan] 2025-07-10 07:17:39,620 - root - INFO - step: 80445 loss: 16.2166 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.7466 +[titan] 2025-07-10 07:17:39,620 - root - INFO - lr: 5.5169e-05 gnorm: 0.99 [17:42:36< 4:18:18] +[titan] 2025-07-10 07:17:42,806 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:17:43,609 - root - INFO - step: 80450 loss: 16.3119 memory: 44.58GiB(31.99%) tps: 82,156 tflops: 283.53 mfu: 28.67% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.8284 +[titan] 2025-07-10 07:17:43,609 - root - INFO - lr: 5.5156e-05 gnorm: 0.94 [17:42:40< 4:18:14] +[titan] 2025-07-10 07:17:47,550 - root - INFO - step: 80455 loss: 16.3824 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8724 +[titan] 2025-07-10 07:17:47,551 - root - INFO - lr: 5.5144e-05 gnorm: 0.95 [17:42:44< 4:18:10] +[titan] 2025-07-10 07:17:51,486 - root - INFO - step: 80460 loss: 16.3400 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.8471 +[titan] 2025-07-10 07:17:51,487 - root - INFO - lr: 5.5131e-05 gnorm: 0.95 [17:42:48< 4:18:06] +[titan] 2025-07-10 07:17:55,374 - root - INFO - step: 80465 loss: 16.2438 memory: 44.58GiB(31.99%) tps: 84,302 tflops: 290.94 mfu: 29.42% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7674 +[titan] 2025-07-10 07:17:55,374 - root - INFO - lr: 5.5119e-05 gnorm: 0.96 [17:42:52< 4:18:02] +[titan] 2025-07-10 07:17:59,277 - root - INFO - step: 80470 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.8032 +[titan] 2025-07-10 07:17:59,277 - root - INFO - lr: 5.5106e-05 gnorm: 0.90 [17:42:56< 4:17:58] +[titan] 2025-07-10 07:18:03,189 - root - INFO - step: 80475 loss: 16.2818 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7946 +[titan] 2025-07-10 07:18:03,189 - root - INFO - lr: 5.5094e-05 gnorm: 0.98 [17:43:00< 4:17:54] +[titan] 2025-07-10 07:18:07,109 - root - INFO - step: 80480 loss: 16.2573 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 07:18:07,109 - root - INFO - lr: 5.5081e-05 gnorm: 0.93 [17:43:04< 4:17:50] +[titan] 2025-07-10 07:18:11,030 - root - INFO - step: 80485 loss: 16.2649 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7861 +[titan] 2025-07-10 07:18:11,031 - root - INFO - lr: 5.5069e-05 gnorm: 0.94 [17:43:08< 4:17:46] +[titan] 2025-07-10 07:18:14,940 - root - INFO - step: 80490 loss: 16.4287 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.9203 +[titan] 2025-07-10 07:18:14,941 - root - INFO - lr: 5.5057e-05 gnorm: 0.93 [17:43:12< 4:17:42] +[titan] 2025-07-10 07:18:18,840 - root - INFO - step: 80495 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.8971 +[titan] 2025-07-10 07:18:18,840 - root - INFO - lr: 5.5044e-05 gnorm: 1.01 [17:43:16< 4:17:38] +[titan] 2025-07-10 07:18:21,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:18:22,738 - root - INFO - step: 80500 loss: 15.9964 memory: 44.58GiB(31.99%) tps: 84,071 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4415 global_avg_mtp_loss: 13.5549 +[titan] 2025-07-10 07:18:22,738 - root - INFO - lr: 5.5032e-05 gnorm: 0.97 [17:43:19< 4:17:34] +[titan] 2025-07-10 07:18:26,639 - root - INFO - step: 80505 loss: 16.3527 memory: 44.58GiB(31.99%) tps: 84,001 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8451 +[titan] 2025-07-10 07:18:26,639 - root - INFO - lr: 5.5019e-05 gnorm: 0.96 [17:43:23< 4:17:30] +[titan] 2025-07-10 07:18:30,540 - root - INFO - step: 80510 loss: 16.1059 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.6565 +[titan] 2025-07-10 07:18:30,541 - root - INFO - lr: 5.5007e-05 gnorm: 0.99 [17:43:27< 4:17:26] +[titan] 2025-07-10 07:18:34,482 - root - INFO - step: 80515 loss: 16.1833 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7031 +[titan] 2025-07-10 07:18:34,483 - root - INFO - lr: 5.4994e-05 gnorm: 0.95 [17:43:31< 4:17:22] +[titan] 2025-07-10 07:18:38,392 - root - INFO - step: 80520 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.8173 +[titan] 2025-07-10 07:18:38,393 - root - INFO - lr: 5.4982e-05 gnorm: 0.97 [17:43:35< 4:17:18] +[titan] 2025-07-10 07:18:42,333 - root - INFO - step: 80525 loss: 16.1214 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6564 +[titan] 2025-07-10 07:18:42,333 - root - INFO - lr: 5.4970e-05 gnorm: 1.04 [17:43:39< 4:17:14] +[titan] 2025-07-10 07:18:46,276 - root - INFO - step: 80530 loss: 16.1430 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.6757 +[titan] 2025-07-10 07:18:46,276 - root - INFO - lr: 5.4957e-05 gnorm: 0.98 [17:43:43< 4:17:10] +[titan] 2025-07-10 07:18:50,191 - root - INFO - step: 80535 loss: 16.1405 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6803 +[titan] 2025-07-10 07:18:50,191 - root - INFO - lr: 5.4945e-05 gnorm: 1.00 [17:43:47< 4:17:06] +[titan] 2025-07-10 07:18:54,121 - root - INFO - step: 80540 loss: 16.1706 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.7078 +[titan] 2025-07-10 07:18:54,122 - root - INFO - lr: 5.4932e-05 gnorm: 1.00 [17:43:51< 4:17:02] +[titan] 2025-07-10 07:18:58,024 - root - INFO - step: 80545 loss: 16.3329 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8353 +[titan] 2025-07-10 07:18:58,024 - root - INFO - lr: 5.4920e-05 gnorm: 1.00 [17:43:55< 4:16:58] +[titan] 2025-07-10 07:19:01,137 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:19:01,926 - root - INFO - step: 80550 loss: 16.1115 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6597 +[titan] 2025-07-10 07:19:01,927 - root - INFO - lr: 5.4908e-05 gnorm: 0.97 [17:43:59< 4:16:54] +[titan] 2025-07-10 07:19:05,881 - root - INFO - step: 80555 loss: 16.3279 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8182 +[titan] 2025-07-10 07:19:05,881 - root - INFO - lr: 5.4895e-05 gnorm: 1.01 [17:44:03< 4:16:50] +[titan] 2025-07-10 07:19:09,788 - root - INFO - step: 80560 loss: 16.2707 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7891 +[titan] 2025-07-10 07:19:09,788 - root - INFO - lr: 5.4883e-05 gnorm: 0.98 [17:44:06< 4:16:46] +[titan] 2025-07-10 07:19:13,731 - root - INFO - step: 80565 loss: 16.3105 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8157 +[titan] 2025-07-10 07:19:13,731 - root - INFO - lr: 5.4870e-05 gnorm: 1.17 [17:44:10< 4:16:43] +[titan] 2025-07-10 07:19:17,638 - root - INFO - step: 80570 loss: 15.9462 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4228 global_avg_mtp_loss: 13.5234 +[titan] 2025-07-10 07:19:17,639 - root - INFO - lr: 5.4858e-05 gnorm: 0.98 [17:44:14< 4:16:39] +[titan] 2025-07-10 07:19:21,533 - root - INFO - step: 80575 loss: 16.3652 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8649 +[titan] 2025-07-10 07:19:21,533 - root - INFO - lr: 5.4846e-05 gnorm: 0.96 [17:44:18< 4:16:35] +[titan] 2025-07-10 07:19:25,429 - root - INFO - step: 80580 loss: 16.1212 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6693 +[titan] 2025-07-10 07:19:25,429 - root - INFO - lr: 5.4833e-05 gnorm: 0.97 [17:44:22< 4:16:31] +[titan] 2025-07-10 07:19:29,360 - root - INFO - step: 80585 loss: 16.0820 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4524 global_avg_mtp_loss: 13.6296 +[titan] 2025-07-10 07:19:29,361 - root - INFO - lr: 5.4821e-05 gnorm: 0.99 [17:44:26< 4:16:27] +[titan] 2025-07-10 07:19:33,250 - root - INFO - step: 80590 loss: 16.5911 memory: 44.58GiB(31.99%) tps: 84,247 tflops: 290.75 mfu: 29.40% global_avg_ntp_loss: 2.5668 global_avg_mtp_loss: 14.0244 +[titan] 2025-07-10 07:19:33,251 - root - INFO - lr: 5.4809e-05 gnorm: 0.94 [17:44:30< 4:16:23] +[titan] 2025-07-10 07:19:37,160 - root - INFO - step: 80595 loss: 16.2816 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.7954 +[titan] 2025-07-10 07:19:37,160 - root - INFO - lr: 5.4796e-05 gnorm: 0.95 [17:44:34< 4:16:19] +[titan] 2025-07-10 07:19:40,293 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:19:41,083 - root - INFO - step: 80600 loss: 16.1596 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.6819 +[titan] 2025-07-10 07:19:41,083 - root - INFO - lr: 5.4784e-05 gnorm: 0.99 [17:44:38< 4:16:15] +[titan] 2025-07-10 07:19:44,997 - root - INFO - step: 80605 loss: 16.2284 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7521 +[titan] 2025-07-10 07:19:44,997 - root - INFO - lr: 5.4771e-05 gnorm: 0.99 [17:44:42< 4:16:11] +[titan] 2025-07-10 07:19:48,897 - root - INFO - step: 80610 loss: 16.2631 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 07:19:48,898 - root - INFO - lr: 5.4759e-05 gnorm: 0.95 [17:44:46< 4:16:07] +[titan] 2025-07-10 07:19:52,815 - root - INFO - step: 80615 loss: 16.1183 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6485 +[titan] 2025-07-10 07:19:52,815 - root - INFO - lr: 5.4747e-05 gnorm: 0.97 [17:44:50< 4:16:03] +[titan] 2025-07-10 07:19:56,726 - root - INFO - step: 80620 loss: 16.2482 memory: 44.58GiB(31.99%) tps: 83,799 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7720 +[titan] 2025-07-10 07:19:56,726 - root - INFO - lr: 5.4734e-05 gnorm: 0.93 [17:44:53< 4:15:59] +[titan] 2025-07-10 07:20:00,650 - root - INFO - step: 80625 loss: 16.0874 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6347 +[titan] 2025-07-10 07:20:00,650 - root - INFO - lr: 5.4722e-05 gnorm: 0.95 [17:44:57< 4:15:55] +[titan] 2025-07-10 07:20:04,560 - root - INFO - step: 80630 loss: 16.1633 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 07:20:04,560 - root - INFO - lr: 5.4710e-05 gnorm: 0.93 [17:45:01< 4:15:51] +[titan] 2025-07-10 07:20:08,473 - root - INFO - step: 80635 loss: 16.2156 memory: 44.58GiB(31.99%) tps: 83,759 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7355 +[titan] 2025-07-10 07:20:08,473 - root - INFO - lr: 5.4697e-05 gnorm: 0.96 [17:45:05< 4:15:47] +[titan] 2025-07-10 07:20:12,397 - root - INFO - step: 80640 loss: 16.2474 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7676 +[titan] 2025-07-10 07:20:12,397 - root - INFO - lr: 5.4685e-05 gnorm: 0.97 [17:45:09< 4:15:43] +[titan] 2025-07-10 07:20:16,300 - root - INFO - step: 80645 loss: 16.2926 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.8113 +[titan] 2025-07-10 07:20:16,300 - root - INFO - lr: 5.4673e-05 gnorm: 0.94 [17:45:13< 4:15:39] +[titan] 2025-07-10 07:20:19,424 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:20:20,214 - root - INFO - step: 80650 loss: 16.3470 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8339 +[titan] 2025-07-10 07:20:20,214 - root - INFO - lr: 5.4660e-05 gnorm: 1.06 [17:45:17< 4:15:35] +[titan] 2025-07-10 07:20:24,175 - root - INFO - step: 80655 loss: 16.3410 memory: 44.58GiB(31.99%) tps: 82,728 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8434 +[titan] 2025-07-10 07:20:24,175 - root - INFO - lr: 5.4648e-05 gnorm: 0.96 [17:45:21< 4:15:31] +[titan] 2025-07-10 07:20:28,087 - root - INFO - step: 80660 loss: 16.0527 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.6126 +[titan] 2025-07-10 07:20:28,087 - root - INFO - lr: 5.4636e-05 gnorm: 0.96 [17:45:25< 4:15:27] +[titan] 2025-07-10 07:20:31,988 - root - INFO - step: 80665 loss: 16.3360 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.8232 +[titan] 2025-07-10 07:20:31,989 - root - INFO - lr: 5.4623e-05 gnorm: 0.95 [17:45:29< 4:15:23] +[titan] 2025-07-10 07:20:35,909 - root - INFO - step: 80670 loss: 16.1561 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.7009 +[titan] 2025-07-10 07:20:35,909 - root - INFO - lr: 5.4611e-05 gnorm: 1.00 [17:45:33< 4:15:19] +[titan] 2025-07-10 07:20:39,805 - root - INFO - step: 80675 loss: 16.2111 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.7268 +[titan] 2025-07-10 07:20:39,805 - root - INFO - lr: 5.4599e-05 gnorm: 0.94 [17:45:37< 4:15:15] +[titan] 2025-07-10 07:20:43,705 - root - INFO - step: 80680 loss: 16.1330 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6689 +[titan] 2025-07-10 07:20:43,706 - root - INFO - lr: 5.4586e-05 gnorm: 0.95 [17:45:40< 4:15:11] +[titan] 2025-07-10 07:20:47,618 - root - INFO - step: 80685 loss: 15.9730 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4484 global_avg_mtp_loss: 13.5245 +[titan] 2025-07-10 07:20:47,618 - root - INFO - lr: 5.4574e-05 gnorm: 1.00 [17:45:44< 4:15:07] +[titan] 2025-07-10 07:20:51,547 - root - INFO - step: 80690 loss: 16.3300 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8376 +[titan] 2025-07-10 07:20:51,547 - root - INFO - lr: 5.4562e-05 gnorm: 0.93 [17:45:48< 4:15:03] +[titan] 2025-07-10 07:20:55,507 - root - INFO - step: 80695 loss: 16.4458 memory: 44.58GiB(31.99%) tps: 82,769 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.9265 +[titan] 2025-07-10 07:20:55,507 - root - INFO - lr: 5.4549e-05 gnorm: 0.98 [17:45:52< 4:14:59] +[titan] 2025-07-10 07:20:58,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:20:59,426 - root - INFO - step: 80700 loss: 16.2363 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7467 +[titan] 2025-07-10 07:20:59,426 - root - INFO - lr: 5.4537e-05 gnorm: 0.97 [17:45:56< 4:14:55] +[titan] 2025-07-10 07:21:03,414 - root - INFO - step: 80705 loss: 16.3223 memory: 44.58GiB(31.99%) tps: 82,168 tflops: 283.58 mfu: 28.67% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.8335 +[titan] 2025-07-10 07:21:03,415 - root - INFO - lr: 5.4525e-05 gnorm: 0.96 [17:46:00< 4:14:51] +[titan] 2025-07-10 07:21:07,319 - root - INFO - step: 80710 loss: 16.2091 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7307 +[titan] 2025-07-10 07:21:07,319 - root - INFO - lr: 5.4512e-05 gnorm: 0.99 [17:46:04< 4:14:47] +[titan] 2025-07-10 07:21:11,237 - root - INFO - step: 80715 loss: 16.2230 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7433 +[titan] 2025-07-10 07:21:11,237 - root - INFO - lr: 5.4500e-05 gnorm: 1.01 [17:46:08< 4:14:43] +[titan] 2025-07-10 07:21:15,182 - root - INFO - step: 80720 loss: 16.2732 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7803 +[titan] 2025-07-10 07:21:15,182 - root - INFO - lr: 5.4488e-05 gnorm: 0.94 [17:46:12< 4:14:39] +[titan] 2025-07-10 07:21:19,126 - root - INFO - step: 80725 loss: 16.1051 memory: 44.58GiB(31.99%) tps: 83,081 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.6423 +[titan] 2025-07-10 07:21:19,127 - root - INFO - lr: 5.4475e-05 gnorm: 1.01 [17:46:16< 4:14:35] +[titan] 2025-07-10 07:21:23,026 - root - INFO - step: 80730 loss: 16.2130 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7407 +[titan] 2025-07-10 07:21:23,026 - root - INFO - lr: 5.4463e-05 gnorm: 0.95 [17:46:20< 4:14:31] +[titan] 2025-07-10 07:21:27,010 - root - INFO - step: 80735 loss: 16.2615 memory: 44.58GiB(31.99%) tps: 82,256 tflops: 283.88 mfu: 28.70% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.7670 +[titan] 2025-07-10 07:21:27,010 - root - INFO - lr: 5.4451e-05 gnorm: 0.98 [17:46:24< 4:14:27] +[titan] 2025-07-10 07:21:30,896 - root - INFO - step: 80740 loss: 15.8839 memory: 44.58GiB(31.99%) tps: 84,331 tflops: 291.04 mfu: 29.43% global_avg_ntp_loss: 2.4115 global_avg_mtp_loss: 13.4724 +[titan] 2025-07-10 07:21:30,896 - root - INFO - lr: 5.4439e-05 gnorm: 1.06 [17:46:28< 4:14:23] +[titan] 2025-07-10 07:21:34,818 - root - INFO - step: 80745 loss: 16.4321 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.9207 +[titan] 2025-07-10 07:21:34,818 - root - INFO - lr: 5.4426e-05 gnorm: 0.96 [17:46:32< 4:14:19] +[titan] 2025-07-10 07:21:37,953 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:21:38,740 - root - INFO - step: 80750 loss: 16.3950 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8960 +[titan] 2025-07-10 07:21:38,740 - root - INFO - lr: 5.4414e-05 gnorm: 0.94 [17:46:35< 4:14:15] +[titan] 2025-07-10 07:21:42,698 - root - INFO - step: 80755 loss: 16.3249 memory: 44.58GiB(31.99%) tps: 82,791 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.8394 +[titan] 2025-07-10 07:21:42,698 - root - INFO - lr: 5.4402e-05 gnorm: 0.99 [17:46:39< 4:14:12] +[titan] 2025-07-10 07:21:46,610 - root - INFO - step: 80760 loss: 16.2678 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7819 +[titan] 2025-07-10 07:21:46,610 - root - INFO - lr: 5.4389e-05 gnorm: 0.95 [17:46:43< 4:14:08] +[titan] 2025-07-10 07:21:50,501 - root - INFO - step: 80765 loss: 16.2658 memory: 44.58GiB(31.99%) tps: 84,225 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7791 +[titan] 2025-07-10 07:21:50,501 - root - INFO - lr: 5.4377e-05 gnorm: 0.99 [17:46:47< 4:14:04] +[titan] 2025-07-10 07:21:54,427 - root - INFO - step: 80770 loss: 16.4101 memory: 44.58GiB(31.99%) tps: 83,470 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.8898 +[titan] 2025-07-10 07:21:54,427 - root - INFO - lr: 5.4365e-05 gnorm: 0.97 [17:46:51< 4:14:00] +[titan] 2025-07-10 07:21:58,381 - root - INFO - step: 80775 loss: 16.3296 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.8389 +[titan] 2025-07-10 07:21:58,381 - root - INFO - lr: 5.4353e-05 gnorm: 0.96 [17:46:55< 4:13:56] +[titan] 2025-07-10 07:22:02,318 - root - INFO - step: 80780 loss: 16.0637 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.6300 +[titan] 2025-07-10 07:22:02,318 - root - INFO - lr: 5.4340e-05 gnorm: 0.96 [17:46:59< 4:13:52] +[titan] 2025-07-10 07:22:06,222 - root - INFO - step: 80785 loss: 16.0484 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.5994 +[titan] 2025-07-10 07:22:06,223 - root - INFO - lr: 5.4328e-05 gnorm: 0.97 [17:47:03< 4:13:48] +[titan] 2025-07-10 07:22:10,130 - root - INFO - step: 80790 loss: 16.1304 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 07:22:10,130 - root - INFO - lr: 5.4316e-05 gnorm: 1.01 [17:47:07< 4:13:44] +[titan] 2025-07-10 07:22:14,049 - root - INFO - step: 80795 loss: 16.0368 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.5818 +[titan] 2025-07-10 07:22:14,049 - root - INFO - lr: 5.4304e-05 gnorm: 0.97 [17:47:11< 4:13:40] +[titan] 2025-07-10 07:22:17,162 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:22:17,951 - root - INFO - step: 80800 loss: 16.0692 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6123 +[titan] 2025-07-10 07:22:17,951 - root - INFO - lr: 5.4291e-05 gnorm: 0.95 [17:47:15< 4:13:36] +[titan] 2025-07-10 07:22:21,865 - root - INFO - step: 80805 loss: 16.3356 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8395 +[titan] 2025-07-10 07:22:21,866 - root - INFO - lr: 5.4279e-05 gnorm: 0.97 [17:47:19< 4:13:32] +[titan] 2025-07-10 07:22:25,772 - root - INFO - step: 80810 loss: 16.0632 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4423 global_avg_mtp_loss: 13.6209 +[titan] 2025-07-10 07:22:25,772 - root - INFO - lr: 5.4267e-05 gnorm: 0.99 [17:47:22< 4:13:28] +[titan] 2025-07-10 07:22:29,682 - root - INFO - step: 80815 loss: 16.3117 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8050 +[titan] 2025-07-10 07:22:29,682 - root - INFO - lr: 5.4255e-05 gnorm: 0.95 [17:47:26< 4:13:24] +[titan] 2025-07-10 07:22:33,577 - root - INFO - step: 80820 loss: 16.2268 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.7558 +[titan] 2025-07-10 07:22:33,577 - root - INFO - lr: 5.4242e-05 gnorm: 0.99 [17:47:30< 4:13:20] +[titan] 2025-07-10 07:22:37,468 - root - INFO - step: 80825 loss: 16.3055 memory: 44.58GiB(31.99%) tps: 84,209 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8134 +[titan] 2025-07-10 07:22:37,469 - root - INFO - lr: 5.4230e-05 gnorm: 0.98 [17:47:34< 4:13:16] +[titan] 2025-07-10 07:22:41,401 - root - INFO - step: 80830 loss: 16.0849 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6293 +[titan] 2025-07-10 07:22:41,401 - root - INFO - lr: 5.4218e-05 gnorm: 0.97 [17:47:38< 4:13:12] +[titan] 2025-07-10 07:22:45,330 - root - INFO - step: 80835 loss: 16.2229 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7435 +[titan] 2025-07-10 07:22:45,330 - root - INFO - lr: 5.4206e-05 gnorm: 0.99 [17:47:42< 4:13:08] +[titan] 2025-07-10 07:22:49,298 - root - INFO - step: 80840 loss: 16.1541 memory: 44.58GiB(31.99%) tps: 82,580 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.6766 +[titan] 2025-07-10 07:22:49,298 - root - INFO - lr: 5.4193e-05 gnorm: 0.96 [17:47:46< 4:13:04] +[titan] 2025-07-10 07:22:53,223 - root - INFO - step: 80845 loss: 16.2031 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7271 +[titan] 2025-07-10 07:22:53,223 - root - INFO - lr: 5.4181e-05 gnorm: 1.02 [17:47:50< 4:13:00] +[titan] 2025-07-10 07:22:56,397 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:22:57,197 - root - INFO - step: 80850 loss: 16.5769 memory: 44.58GiB(31.99%) tps: 82,458 tflops: 284.57 mfu: 28.77% global_avg_ntp_loss: 2.5364 global_avg_mtp_loss: 14.0405 +[titan] 2025-07-10 07:22:57,197 - root - INFO - lr: 5.4169e-05 gnorm: 0.98 [17:47:54< 4:12:56] +[titan] 2025-07-10 07:23:01,155 - root - INFO - step: 80855 loss: 16.4142 memory: 44.58GiB(31.99%) tps: 82,797 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.5098 global_avg_mtp_loss: 13.9044 +[titan] 2025-07-10 07:23:01,155 - root - INFO - lr: 5.4157e-05 gnorm: 0.98 [17:47:58< 4:12:52] +[titan] 2025-07-10 07:23:05,051 - root - INFO - step: 80860 loss: 16.0179 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.5804 +[titan] 2025-07-10 07:23:05,051 - root - INFO - lr: 5.4144e-05 gnorm: 0.98 [17:48:02< 4:12:48] +[titan] 2025-07-10 07:23:08,985 - root - INFO - step: 80865 loss: 16.3949 memory: 44.58GiB(31.99%) tps: 83,301 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8919 +[titan] 2025-07-10 07:23:08,985 - root - INFO - lr: 5.4132e-05 gnorm: 0.99 [17:48:06< 4:12:44] +[titan] 2025-07-10 07:23:12,908 - root - INFO - step: 80870 loss: 16.4420 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5182 global_avg_mtp_loss: 13.9238 +[titan] 2025-07-10 07:23:12,908 - root - INFO - lr: 5.4120e-05 gnorm: 1.00 [17:48:10< 4:12:40] +[titan] 2025-07-10 07:23:16,826 - root - INFO - step: 80875 loss: 16.4324 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5311 global_avg_mtp_loss: 13.9012 +[titan] 2025-07-10 07:23:16,826 - root - INFO - lr: 5.4108e-05 gnorm: 1.00 [17:48:14< 4:12:36] +[titan] 2025-07-10 07:23:20,756 - root - INFO - step: 80880 loss: 16.4639 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.9490 +[titan] 2025-07-10 07:23:20,757 - root - INFO - lr: 5.4096e-05 gnorm: 1.00 [17:48:17< 4:12:32] +[titan] 2025-07-10 07:23:24,662 - root - INFO - step: 80885 loss: 16.3712 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8580 +[titan] 2025-07-10 07:23:24,663 - root - INFO - lr: 5.4083e-05 gnorm: 0.99 [17:48:21< 4:12:28] +[titan] 2025-07-10 07:23:28,575 - root - INFO - step: 80890 loss: 16.3635 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8593 +[titan] 2025-07-10 07:23:28,575 - root - INFO - lr: 5.4071e-05 gnorm: 0.98 [17:48:25< 4:12:24] +[titan] 2025-07-10 07:23:32,570 - root - INFO - step: 80895 loss: 16.3380 memory: 44.58GiB(31.99%) tps: 82,033 tflops: 283.11 mfu: 28.63% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.8515 +[titan] 2025-07-10 07:23:32,570 - root - INFO - lr: 5.4059e-05 gnorm: 0.96 [17:48:29< 4:12:20] +[titan] 2025-07-10 07:23:33,509 - root - INFO - Dumping profiler traces at step 80896 +[titan] 2025-07-10 07:23:33,542 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:23:35,896 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:23:36,690 - root - INFO - step: 80900 loss: 16.1273 memory: 44.58GiB(31.99%) tps: 79,537 tflops: 274.50 mfu: 27.75% global_avg_ntp_loss: 2.4521 global_avg_mtp_loss: 13.6752 +[titan] 2025-07-10 07:23:36,690 - root - INFO - lr: 5.4047e-05 gnorm: 1.02 [17:48:33< 4:12:16] +[titan] 2025-07-10 07:23:40,623 - root - INFO - step: 80905 loss: 16.3573 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8505 +[titan] 2025-07-10 07:23:40,623 - root - INFO - lr: 5.4034e-05 gnorm: 1.03 [17:48:37< 4:12:12] +[titan] 2025-07-10 07:23:44,568 - root - INFO - step: 80910 loss: 16.1780 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7020 +[titan] 2025-07-10 07:23:44,569 - root - INFO - lr: 5.4022e-05 gnorm: 0.97 [17:48:41< 4:12:08] +[titan] 2025-07-10 07:23:48,493 - root - INFO - step: 80915 loss: 16.2452 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7666 +[titan] 2025-07-10 07:23:48,494 - root - INFO - lr: 5.4010e-05 gnorm: 0.95 [17:48:45< 4:12:04] +[titan] 2025-07-10 07:23:52,433 - root - INFO - step: 80920 loss: 16.2867 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.02% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 07:23:52,434 - root - INFO - lr: 5.3998e-05 gnorm: 0.95 [17:48:49< 4:12:01] +[titan] 2025-07-10 07:23:56,386 - root - INFO - step: 80925 loss: 16.3177 memory: 44.58GiB(31.99%) tps: 82,912 tflops: 286.14 mfu: 28.93% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8208 +[titan] 2025-07-10 07:23:56,386 - root - INFO - lr: 5.3986e-05 gnorm: 0.90 [17:48:53< 4:11:57] +[titan] 2025-07-10 07:24:00,392 - root - INFO - step: 80930 loss: 16.4794 memory: 44.58GiB(31.99%) tps: 81,807 tflops: 282.33 mfu: 28.55% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9532 +[titan] 2025-07-10 07:24:00,392 - root - INFO - lr: 5.3974e-05 gnorm: 0.97 [17:48:57< 4:11:53] +[titan] 2025-07-10 07:24:04,331 - root - INFO - step: 80935 loss: 16.2888 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.7836 +[titan] 2025-07-10 07:24:04,332 - root - INFO - lr: 5.3961e-05 gnorm: 0.96 [17:49:01< 4:11:49] +[titan] 2025-07-10 07:24:08,247 - root - INFO - step: 80940 loss: 16.2236 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7527 +[titan] 2025-07-10 07:24:08,248 - root - INFO - lr: 5.3949e-05 gnorm: 0.92 [17:49:05< 4:11:45] +[titan] 2025-07-10 07:24:12,150 - root - INFO - step: 80945 loss: 16.0892 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4523 global_avg_mtp_loss: 13.6368 +[titan] 2025-07-10 07:24:12,150 - root - INFO - lr: 5.3937e-05 gnorm: 0.98 [17:49:09< 4:11:41] +[titan] 2025-07-10 07:24:15,272 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:24:16,060 - root - INFO - step: 80950 loss: 16.2629 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7832 +[titan] 2025-07-10 07:24:16,061 - root - INFO - lr: 5.3925e-05 gnorm: 0.96 [17:49:13< 4:11:37] +[titan] 2025-07-10 07:24:20,022 - root - INFO - step: 80955 loss: 16.3834 memory: 44.58GiB(31.99%) tps: 82,718 tflops: 285.47 mfu: 28.86% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8723 +[titan] 2025-07-10 07:24:20,022 - root - INFO - lr: 5.3913e-05 gnorm: 0.98 [17:49:17< 4:11:33] +[titan] 2025-07-10 07:24:23,932 - root - INFO - step: 80960 loss: 16.3329 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 07:24:23,932 - root - INFO - lr: 5.3900e-05 gnorm: 0.94 [17:49:21< 4:11:29] +[titan] 2025-07-10 07:24:27,834 - root - INFO - step: 80965 loss: 16.3713 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.8727 +[titan] 2025-07-10 07:24:27,834 - root - INFO - lr: 5.3888e-05 gnorm: 0.97 [17:49:25< 4:11:25] +[titan] 2025-07-10 07:24:31,751 - root - INFO - step: 80970 loss: 16.1492 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6889 +[titan] 2025-07-10 07:24:31,751 - root - INFO - lr: 5.3876e-05 gnorm: 0.99 [17:49:28< 4:11:21] +[titan] 2025-07-10 07:24:35,649 - root - INFO - step: 80975 loss: 15.8542 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4197 global_avg_mtp_loss: 13.4345 +[titan] 2025-07-10 07:24:35,649 - root - INFO - lr: 5.3864e-05 gnorm: 1.02 [17:49:32< 4:11:17] +[titan] 2025-07-10 07:24:39,602 - root - INFO - step: 80980 loss: 16.0382 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.5942 +[titan] 2025-07-10 07:24:39,603 - root - INFO - lr: 5.3852e-05 gnorm: 1.02 [17:49:36< 4:11:13] +[titan] 2025-07-10 07:24:43,556 - root - INFO - step: 80985 loss: 16.1402 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4663 global_avg_mtp_loss: 13.6739 +[titan] 2025-07-10 07:24:43,557 - root - INFO - lr: 5.3840e-05 gnorm: 0.97 [17:49:40< 4:11:09] +[titan] 2025-07-10 07:24:47,483 - root - INFO - step: 80990 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.8164 +[titan] 2025-07-10 07:24:47,484 - root - INFO - lr: 5.3827e-05 gnorm: 0.96 [17:49:44< 4:11:05] +[titan] 2025-07-10 07:24:51,418 - root - INFO - step: 80995 loss: 16.0571 memory: 44.58GiB(31.99%) tps: 83,282 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6092 +[titan] 2025-07-10 07:24:51,419 - root - INFO - lr: 5.3815e-05 gnorm: 0.94 [17:49:48< 4:11:01] +[titan] 2025-07-10 07:24:54,565 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:24:55,364 - root - INFO - step: 81000 loss: 16.3588 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 07:24:55,364 - root - INFO - lr: 5.3803e-05 gnorm: 0.94 [17:49:52< 4:10:57] +[titan] 2025-07-10 07:24:59,292 - root - INFO - step: 81005 loss: 15.9839 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4331 global_avg_mtp_loss: 13.5508 +[titan] 2025-07-10 07:24:59,292 - root - INFO - lr: 5.3791e-05 gnorm: 1.00 [17:49:56< 4:10:53] +[titan] 2025-07-10 07:25:03,207 - root - INFO - step: 81010 loss: 15.9617 memory: 44.58GiB(31.99%) tps: 83,705 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4365 global_avg_mtp_loss: 13.5253 +[titan] 2025-07-10 07:25:03,207 - root - INFO - lr: 5.3779e-05 gnorm: 1.02 [17:50:00< 4:10:49] +[titan] 2025-07-10 07:25:07,146 - root - INFO - step: 81015 loss: 16.2607 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7748 +[titan] 2025-07-10 07:25:07,146 - root - INFO - lr: 5.3767e-05 gnorm: 0.96 [17:50:04< 4:10:45] +[titan] 2025-07-10 07:25:11,060 - root - INFO - step: 81020 loss: 16.5224 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5307 global_avg_mtp_loss: 13.9917 +[titan] 2025-07-10 07:25:11,060 - root - INFO - lr: 5.3755e-05 gnorm: 1.01 [17:50:08< 4:10:41] +[titan] 2025-07-10 07:25:14,993 - root - INFO - step: 81025 loss: 15.9082 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4188 global_avg_mtp_loss: 13.4894 +[titan] 2025-07-10 07:25:14,993 - root - INFO - lr: 5.3742e-05 gnorm: 1.01 [17:50:12< 4:10:37] +[titan] 2025-07-10 07:25:18,908 - root - INFO - step: 81030 loss: 16.1386 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.6800 +[titan] 2025-07-10 07:25:18,909 - root - INFO - lr: 5.3730e-05 gnorm: 1.01 [17:50:16< 4:10:33] +[titan] 2025-07-10 07:25:22,805 - root - INFO - step: 81035 loss: 16.3758 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 07:25:22,805 - root - INFO - lr: 5.3718e-05 gnorm: 0.99 [17:50:19< 4:10:29] +[titan] 2025-07-10 07:25:26,724 - root - INFO - step: 81040 loss: 16.1509 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.6875 +[titan] 2025-07-10 07:25:26,724 - root - INFO - lr: 5.3706e-05 gnorm: 1.01 [17:50:23< 4:10:25] +[titan] 2025-07-10 07:25:30,688 - root - INFO - step: 81045 loss: 16.2411 memory: 44.58GiB(31.99%) tps: 82,673 tflops: 285.32 mfu: 28.85% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7587 +[titan] 2025-07-10 07:25:30,688 - root - INFO - lr: 5.3694e-05 gnorm: 0.99 [17:50:27< 4:10:21] +[titan] 2025-07-10 07:25:33,835 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:25:34,626 - root - INFO - step: 81050 loss: 16.4696 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9418 +[titan] 2025-07-10 07:25:34,626 - root - INFO - lr: 5.3682e-05 gnorm: 1.05 [17:50:31< 4:10:17] +[titan] 2025-07-10 07:25:38,527 - root - INFO - step: 81055 loss: 16.2428 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7556 +[titan] 2025-07-10 07:25:38,527 - root - INFO - lr: 5.3670e-05 gnorm: 0.96 [17:50:35< 4:10:13] +[titan] 2025-07-10 07:25:42,486 - root - INFO - step: 81060 loss: 16.5068 memory: 44.58GiB(31.99%) tps: 82,790 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.5295 global_avg_mtp_loss: 13.9773 +[titan] 2025-07-10 07:25:42,486 - root - INFO - lr: 5.3658e-05 gnorm: 0.98 [17:50:39< 4:10:09] +[titan] 2025-07-10 07:25:46,394 - root - INFO - step: 81065 loss: 16.2810 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 07:25:46,394 - root - INFO - lr: 5.3646e-05 gnorm: 0.98 [17:50:43< 4:10:05] +[titan] 2025-07-10 07:25:50,307 - root - INFO - step: 81070 loss: 15.7947 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4136 global_avg_mtp_loss: 13.3812 +[titan] 2025-07-10 07:25:50,307 - root - INFO - lr: 5.3633e-05 gnorm: 0.95 [17:50:47< 4:10:01] +[titan] 2025-07-10 07:25:54,227 - root - INFO - step: 81075 loss: 16.0193 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4438 global_avg_mtp_loss: 13.5755 +[titan] 2025-07-10 07:25:54,228 - root - INFO - lr: 5.3621e-05 gnorm: 1.05 [17:50:51< 4:09:57] +[titan] 2025-07-10 07:25:58,180 - root - INFO - step: 81080 loss: 16.3967 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 07:25:58,180 - root - INFO - lr: 5.3609e-05 gnorm: 1.00 [17:50:55< 4:09:53] +[titan] 2025-07-10 07:26:02,088 - root - INFO - step: 81085 loss: 16.2847 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.8002 +[titan] 2025-07-10 07:26:02,088 - root - INFO - lr: 5.3597e-05 gnorm: 0.98 [17:50:59< 4:09:49] +[titan] 2025-07-10 07:26:05,999 - root - INFO - step: 81090 loss: 16.1778 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 07:26:06,000 - root - INFO - lr: 5.3585e-05 gnorm: 0.98 [17:51:03< 4:09:46] +[titan] 2025-07-10 07:26:09,911 - root - INFO - step: 81095 loss: 16.2484 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7623 +[titan] 2025-07-10 07:26:09,912 - root - INFO - lr: 5.3573e-05 gnorm: 1.02 [17:51:07< 4:09:42] +[titan] 2025-07-10 07:26:13,022 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:26:13,809 - root - INFO - step: 81100 loss: 16.3005 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.8101 +[titan] 2025-07-10 07:26:13,809 - root - INFO - lr: 5.3561e-05 gnorm: 0.97 [17:51:10< 4:09:38] +[titan] 2025-07-10 07:26:17,742 - root - INFO - step: 81105 loss: 16.3011 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8096 +[titan] 2025-07-10 07:26:17,742 - root - INFO - lr: 5.3549e-05 gnorm: 0.99 [17:51:14< 4:09:34] +[titan] 2025-07-10 07:26:21,644 - root - INFO - step: 81110 loss: 16.4749 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9415 +[titan] 2025-07-10 07:26:21,644 - root - INFO - lr: 5.3537e-05 gnorm: 1.00 [17:51:18< 4:09:30] +[titan] 2025-07-10 07:26:25,545 - root - INFO - step: 81115 loss: 16.0300 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4342 global_avg_mtp_loss: 13.5958 +[titan] 2025-07-10 07:26:25,545 - root - INFO - lr: 5.3525e-05 gnorm: 1.00 [17:51:22< 4:09:26] +[titan] 2025-07-10 07:26:29,469 - root - INFO - step: 81120 loss: 16.1172 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 07:26:29,469 - root - INFO - lr: 5.3512e-05 gnorm: 0.98 [17:51:26< 4:09:22] +[titan] 2025-07-10 07:26:33,380 - root - INFO - step: 81125 loss: 16.3238 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.8402 +[titan] 2025-07-10 07:26:33,380 - root - INFO - lr: 5.3500e-05 gnorm: 0.93 [17:51:30< 4:09:18] +[titan] 2025-07-10 07:26:37,283 - root - INFO - step: 81130 loss: 16.3026 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8060 +[titan] 2025-07-10 07:26:37,283 - root - INFO - lr: 5.3488e-05 gnorm: 0.90 [17:51:34< 4:09:14] +[titan] 2025-07-10 07:26:41,193 - root - INFO - step: 81135 loss: 16.2903 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.8066 +[titan] 2025-07-10 07:26:41,193 - root - INFO - lr: 5.3476e-05 gnorm: 0.94 [17:51:38< 4:09:10] +[titan] 2025-07-10 07:26:45,098 - root - INFO - step: 81140 loss: 16.4826 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9561 +[titan] 2025-07-10 07:26:45,098 - root - INFO - lr: 5.3464e-05 gnorm: 0.97 [17:51:42< 4:09:06] +[titan] 2025-07-10 07:26:49,035 - root - INFO - step: 81145 loss: 16.3416 memory: 44.58GiB(31.99%) tps: 83,244 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8470 +[titan] 2025-07-10 07:26:49,035 - root - INFO - lr: 5.3452e-05 gnorm: 0.93 [17:51:46< 4:09:02] +[titan] 2025-07-10 07:26:52,185 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:26:53,001 - root - INFO - step: 81150 loss: 16.1867 memory: 44.58GiB(31.99%) tps: 82,618 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7138 +[titan] 2025-07-10 07:26:53,002 - root - INFO - lr: 5.3440e-05 gnorm: 1.01 [17:51:50< 4:08:58] +[titan] 2025-07-10 07:26:56,936 - root - INFO - step: 81155 loss: 16.1954 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.7232 +[titan] 2025-07-10 07:26:56,936 - root - INFO - lr: 5.3428e-05 gnorm: 0.95 [17:51:54< 4:08:54] +[titan] 2025-07-10 07:27:00,865 - root - INFO - step: 81160 loss: 16.3943 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8942 +[titan] 2025-07-10 07:27:00,865 - root - INFO - lr: 5.3416e-05 gnorm: 0.95 [17:51:58< 4:08:50] +[titan] 2025-07-10 07:27:04,788 - root - INFO - step: 81165 loss: 16.2228 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7444 +[titan] 2025-07-10 07:27:04,788 - root - INFO - lr: 5.3404e-05 gnorm: 0.98 [17:52:01< 4:08:46] +[titan] 2025-07-10 07:27:08,707 - root - INFO - step: 81170 loss: 15.9598 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.5222 +[titan] 2025-07-10 07:27:08,708 - root - INFO - lr: 5.3392e-05 gnorm: 1.00 [17:52:05< 4:08:42] +[titan] 2025-07-10 07:27:12,604 - root - INFO - step: 81175 loss: 16.0605 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.4469 global_avg_mtp_loss: 13.6136 +[titan] 2025-07-10 07:27:12,604 - root - INFO - lr: 5.3380e-05 gnorm: 1.04 [17:52:09< 4:08:38] +[titan] 2025-07-10 07:27:16,530 - root - INFO - step: 81180 loss: 16.1399 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6835 +[titan] 2025-07-10 07:27:16,530 - root - INFO - lr: 5.3368e-05 gnorm: 1.01 [17:52:13< 4:08:34] +[titan] 2025-07-10 07:27:20,442 - root - INFO - step: 81185 loss: 16.2535 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7696 +[titan] 2025-07-10 07:27:20,443 - root - INFO - lr: 5.3356e-05 gnorm: 0.97 [17:52:17< 4:08:30] +[titan] 2025-07-10 07:27:24,414 - root - INFO - step: 81190 loss: 16.2240 memory: 44.58GiB(31.99%) tps: 82,522 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7385 +[titan] 2025-07-10 07:27:24,414 - root - INFO - lr: 5.3344e-05 gnorm: 0.98 [17:52:21< 4:08:26] +[titan] 2025-07-10 07:27:28,332 - root - INFO - step: 81195 loss: 16.1574 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 07:27:28,332 - root - INFO - lr: 5.3332e-05 gnorm: 0.93 [17:52:25< 4:08:22] +[titan] 2025-07-10 07:27:31,450 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:27:32,251 - root - INFO - step: 81200 loss: 16.0944 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6369 +[titan] 2025-07-10 07:27:32,251 - root - INFO - lr: 5.3320e-05 gnorm: 1.01 [17:52:29< 4:08:18] +[titan] 2025-07-10 07:27:36,174 - root - INFO - step: 81205 loss: 16.0354 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4495 global_avg_mtp_loss: 13.5859 +[titan] 2025-07-10 07:27:36,175 - root - INFO - lr: 5.3308e-05 gnorm: 0.96 [17:52:33< 4:08:14] +[titan] 2025-07-10 07:27:40,078 - root - INFO - step: 81210 loss: 15.7756 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4081 global_avg_mtp_loss: 13.3675 +[titan] 2025-07-10 07:27:40,078 - root - INFO - lr: 5.3296e-05 gnorm: 1.00 [17:52:37< 4:08:10] +[titan] 2025-07-10 07:27:44,018 - root - INFO - step: 81215 loss: 16.1553 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.6821 +[titan] 2025-07-10 07:27:44,018 - root - INFO - lr: 5.3284e-05 gnorm: 1.00 [17:52:41< 4:08:06] +[titan] 2025-07-10 07:27:47,922 - root - INFO - step: 81220 loss: 16.1609 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 07:27:47,922 - root - INFO - lr: 5.3271e-05 gnorm: 1.01 [17:52:45< 4:08:02] +[titan] 2025-07-10 07:27:51,858 - root - INFO - step: 81225 loss: 16.3398 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4955 global_avg_mtp_loss: 13.8443 +[titan] 2025-07-10 07:27:51,858 - root - INFO - lr: 5.3259e-05 gnorm: 0.99 [17:52:49< 4:07:58] +[titan] 2025-07-10 07:27:55,803 - root - INFO - step: 81230 loss: 16.3123 memory: 44.58GiB(31.99%) tps: 83,066 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.8295 +[titan] 2025-07-10 07:27:55,803 - root - INFO - lr: 5.3247e-05 gnorm: 0.97 [17:52:52< 4:07:54] +[titan] 2025-07-10 07:27:59,774 - root - INFO - step: 81235 loss: 15.9626 memory: 44.58GiB(31.99%) tps: 82,524 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.4353 global_avg_mtp_loss: 13.5273 +[titan] 2025-07-10 07:27:59,774 - root - INFO - lr: 5.3235e-05 gnorm: 0.99 [17:52:56< 4:07:50] +[titan] 2025-07-10 07:28:03,707 - root - INFO - step: 81240 loss: 15.9396 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4206 global_avg_mtp_loss: 13.5190 +[titan] 2025-07-10 07:28:03,708 - root - INFO - lr: 5.3223e-05 gnorm: 0.96 [17:53:00< 4:07:46] +[titan] 2025-07-10 07:28:07,662 - root - INFO - step: 81245 loss: 15.9660 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.4423 global_avg_mtp_loss: 13.5236 +[titan] 2025-07-10 07:28:07,662 - root - INFO - lr: 5.3211e-05 gnorm: 1.00 [17:53:04< 4:07:42] +[titan] 2025-07-10 07:28:10,774 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:28:11,559 - root - INFO - step: 81250 loss: 15.7552 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.3893 global_avg_mtp_loss: 13.3659 +[titan] 2025-07-10 07:28:11,559 - root - INFO - lr: 5.3199e-05 gnorm: 0.99 [17:53:08< 4:07:38] +[titan] 2025-07-10 07:28:15,463 - root - INFO - step: 81255 loss: 15.9737 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4287 global_avg_mtp_loss: 13.5451 +[titan] 2025-07-10 07:28:15,463 - root - INFO - lr: 5.3187e-05 gnorm: 0.99 [17:53:12< 4:07:34] +[titan] 2025-07-10 07:28:19,388 - root - INFO - step: 81260 loss: 16.4058 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8914 +[titan] 2025-07-10 07:28:19,389 - root - INFO - lr: 5.3175e-05 gnorm: 0.98 [17:53:16< 4:07:30] +[titan] 2025-07-10 07:28:23,313 - root - INFO - step: 81265 loss: 16.1674 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 07:28:23,313 - root - INFO - lr: 5.3163e-05 gnorm: 1.01 [17:53:20< 4:07:27] +[titan] 2025-07-10 07:28:27,235 - root - INFO - step: 81270 loss: 16.4529 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5320 global_avg_mtp_loss: 13.9209 +[titan] 2025-07-10 07:28:27,235 - root - INFO - lr: 5.3151e-05 gnorm: 0.99 [17:53:24< 4:07:23] +[titan] 2025-07-10 07:28:31,143 - root - INFO - step: 81275 loss: 16.2171 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.7499 +[titan] 2025-07-10 07:28:31,143 - root - INFO - lr: 5.3139e-05 gnorm: 0.95 [17:53:28< 4:07:19] +[titan] 2025-07-10 07:28:35,053 - root - INFO - step: 81280 loss: 16.2427 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.7547 +[titan] 2025-07-10 07:28:35,053 - root - INFO - lr: 5.3127e-05 gnorm: 0.95 [17:53:32< 4:07:15] +[titan] 2025-07-10 07:28:38,958 - root - INFO - step: 81285 loss: 15.7210 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.63 mfu: 29.28% global_avg_ntp_loss: 2.3778 global_avg_mtp_loss: 13.3432 +[titan] 2025-07-10 07:28:38,958 - root - INFO - lr: 5.3115e-05 gnorm: 1.04 [17:53:36< 4:07:11] +[titan] 2025-07-10 07:28:42,874 - root - INFO - step: 81290 loss: 16.6320 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5483 global_avg_mtp_loss: 14.0837 +[titan] 2025-07-10 07:28:42,874 - root - INFO - lr: 5.3103e-05 gnorm: 0.95 [17:53:40< 4:07:07] +[titan] 2025-07-10 07:28:46,774 - root - INFO - step: 81295 loss: 16.3202 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.8346 +[titan] 2025-07-10 07:28:46,774 - root - INFO - lr: 5.3091e-05 gnorm: 0.95 [17:53:43< 4:07:03] +[titan] 2025-07-10 07:28:49,912 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:28:50,702 - root - INFO - step: 81300 loss: 16.4046 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8984 +[titan] 2025-07-10 07:28:50,702 - root - INFO - lr: 5.3079e-05 gnorm: 1.01 [17:53:47< 4:06:59] +[titan] 2025-07-10 07:28:54,662 - root - INFO - step: 81305 loss: 16.0978 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6499 +[titan] 2025-07-10 07:28:54,662 - root - INFO - lr: 5.3067e-05 gnorm: 0.99 [17:53:51< 4:06:55] +[titan] 2025-07-10 07:28:58,580 - root - INFO - step: 81310 loss: 16.1366 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6718 +[titan] 2025-07-10 07:28:58,581 - root - INFO - lr: 5.3056e-05 gnorm: 1.00 [17:53:55< 4:06:51] +[titan] 2025-07-10 07:29:02,507 - root - INFO - step: 81315 loss: 16.1669 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4703 global_avg_mtp_loss: 13.6966 +[titan] 2025-07-10 07:29:02,507 - root - INFO - lr: 5.3044e-05 gnorm: 0.98 [17:53:59< 4:06:47] +[titan] 2025-07-10 07:29:06,406 - root - INFO - step: 81320 loss: 16.3141 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 07:29:06,406 - root - INFO - lr: 5.3032e-05 gnorm: 0.98 [17:54:03< 4:06:43] +[titan] 2025-07-10 07:29:10,310 - root - INFO - step: 81325 loss: 16.2495 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7630 +[titan] 2025-07-10 07:29:10,310 - root - INFO - lr: 5.3020e-05 gnorm: 0.99 [17:54:07< 4:06:39] +[titan] 2025-07-10 07:29:14,211 - root - INFO - step: 81330 loss: 16.2875 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.8075 +[titan] 2025-07-10 07:29:14,211 - root - INFO - lr: 5.3008e-05 gnorm: 0.93 [17:54:11< 4:06:35] +[titan] 2025-07-10 07:29:18,114 - root - INFO - step: 81335 loss: 16.3005 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8039 +[titan] 2025-07-10 07:29:18,114 - root - INFO - lr: 5.2996e-05 gnorm: 0.98 [17:54:15< 4:06:31] +[titan] 2025-07-10 07:29:22,007 - root - INFO - step: 81340 loss: 16.1551 memory: 44.58GiB(31.99%) tps: 84,184 tflops: 290.53 mfu: 29.38% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.6865 +[titan] 2025-07-10 07:29:22,007 - root - INFO - lr: 5.2984e-05 gnorm: 0.98 [17:54:19< 4:06:27] +[titan] 2025-07-10 07:29:25,922 - root - INFO - step: 81345 loss: 16.0090 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4300 global_avg_mtp_loss: 13.5789 +[titan] 2025-07-10 07:29:25,922 - root - INFO - lr: 5.2972e-05 gnorm: 0.97 [17:54:23< 4:06:23] +[titan] 2025-07-10 07:29:29,046 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:29:29,833 - root - INFO - step: 81350 loss: 16.1376 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6695 +[titan] 2025-07-10 07:29:29,834 - root - INFO - lr: 5.2960e-05 gnorm: 0.99 [17:54:26< 4:06:19] +[titan] 2025-07-10 07:29:33,733 - root - INFO - step: 81355 loss: 16.3809 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8640 +[titan] 2025-07-10 07:29:33,733 - root - INFO - lr: 5.2948e-05 gnorm: 1.00 [17:54:30< 4:06:15] +[titan] 2025-07-10 07:29:37,639 - root - INFO - step: 81360 loss: 16.0704 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6068 +[titan] 2025-07-10 07:29:37,639 - root - INFO - lr: 5.2936e-05 gnorm: 1.05 [17:54:34< 4:06:11] +[titan] 2025-07-10 07:29:41,542 - root - INFO - step: 81365 loss: 16.0825 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.6210 +[titan] 2025-07-10 07:29:41,543 - root - INFO - lr: 5.2924e-05 gnorm: 0.97 [17:54:38< 4:06:07] +[titan] 2025-07-10 07:29:45,454 - root - INFO - step: 81370 loss: 16.0079 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5662 +[titan] 2025-07-10 07:29:45,454 - root - INFO - lr: 5.2912e-05 gnorm: 0.97 [17:54:42< 4:06:03] +[titan] 2025-07-10 07:29:49,360 - root - INFO - step: 81375 loss: 16.1838 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.7093 +[titan] 2025-07-10 07:29:49,360 - root - INFO - lr: 5.2900e-05 gnorm: 0.95 [17:54:46< 4:05:59] +[titan] 2025-07-10 07:29:53,260 - root - INFO - step: 81380 loss: 16.4606 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9402 +[titan] 2025-07-10 07:29:53,260 - root - INFO - lr: 5.2888e-05 gnorm: 0.96 [17:54:50< 4:05:55] +[titan] 2025-07-10 07:29:57,172 - root - INFO - step: 81385 loss: 16.0488 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4624 global_avg_mtp_loss: 13.5864 +[titan] 2025-07-10 07:29:57,173 - root - INFO - lr: 5.2876e-05 gnorm: 0.93 [17:54:54< 4:05:51] +[titan] 2025-07-10 07:30:01,079 - root - INFO - step: 81390 loss: 16.4221 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 07:30:01,079 - root - INFO - lr: 5.2864e-05 gnorm: 0.95 [17:54:58< 4:05:47] +[titan] 2025-07-10 07:30:05,030 - root - INFO - step: 81395 loss: 16.3968 memory: 44.58GiB(31.99%) tps: 82,953 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8954 +[titan] 2025-07-10 07:30:05,030 - root - INFO - lr: 5.2852e-05 gnorm: 0.94 [17:55:02< 4:05:43] +[titan] 2025-07-10 07:30:08,160 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:30:08,960 - root - INFO - step: 81400 loss: 16.1254 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6656 +[titan] 2025-07-10 07:30:08,961 - root - INFO - lr: 5.2840e-05 gnorm: 1.04 [17:55:06< 4:05:39] +[titan] 2025-07-10 07:30:12,996 - root - INFO - step: 81405 loss: 16.3832 memory: 44.58GiB(31.99%) tps: 81,210 tflops: 280.27 mfu: 28.34% global_avg_ntp_loss: 2.5240 global_avg_mtp_loss: 13.8593 +[titan] 2025-07-10 07:30:12,996 - root - INFO - lr: 5.2829e-05 gnorm: 0.94 [17:55:10< 4:05:35] +[titan] 2025-07-10 07:30:15,484 - root - INFO - Dumping profiler traces at step 81408 +[titan] 2025-07-10 07:30:15,516 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:30:17,087 - root - INFO - step: 81410 loss: 16.3830 memory: 44.58GiB(31.99%) tps: 80,094 tflops: 276.42 mfu: 27.95% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8769 +[titan] 2025-07-10 07:30:17,088 - root - INFO - lr: 5.2817e-05 gnorm: 0.96 [17:55:14< 4:05:31] +[titan] 2025-07-10 07:30:20,999 - root - INFO - step: 81415 loss: 16.1327 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6755 +[titan] 2025-07-10 07:30:20,999 - root - INFO - lr: 5.2805e-05 gnorm: 0.98 [17:55:18< 4:05:27] +[titan] 2025-07-10 07:30:24,937 - root - INFO - step: 81420 loss: 16.0641 memory: 44.58GiB(31.99%) tps: 83,225 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.5986 +[titan] 2025-07-10 07:30:24,937 - root - INFO - lr: 5.2793e-05 gnorm: 1.00 [17:55:22< 4:05:23] +[titan] 2025-07-10 07:30:28,864 - root - INFO - step: 81425 loss: 16.2420 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.7470 +[titan] 2025-07-10 07:30:28,864 - root - INFO - lr: 5.2781e-05 gnorm: 1.02 [17:55:25< 4:05:19] +[titan] 2025-07-10 07:30:32,775 - root - INFO - step: 81430 loss: 16.2497 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7580 +[titan] 2025-07-10 07:30:32,775 - root - INFO - lr: 5.2769e-05 gnorm: 0.98 [17:55:29< 4:05:15] +[titan] 2025-07-10 07:30:36,682 - root - INFO - step: 81435 loss: 16.2361 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.7475 +[titan] 2025-07-10 07:30:36,682 - root - INFO - lr: 5.2757e-05 gnorm: 1.02 [17:55:33< 4:05:11] +[titan] 2025-07-10 07:30:40,593 - root - INFO - step: 81440 loss: 16.2337 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7544 +[titan] 2025-07-10 07:30:40,593 - root - INFO - lr: 5.2745e-05 gnorm: 0.97 [17:55:37< 4:05:08] +[titan] 2025-07-10 07:30:44,509 - root - INFO - step: 81445 loss: 16.1210 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6663 +[titan] 2025-07-10 07:30:44,509 - root - INFO - lr: 5.2733e-05 gnorm: 0.94 [17:55:41< 4:05:04] +[titan] 2025-07-10 07:30:47,671 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:30:48,456 - root - INFO - step: 81450 loss: 16.3214 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8256 +[titan] 2025-07-10 07:30:48,456 - root - INFO - lr: 5.2721e-05 gnorm: 1.00 [17:55:45< 4:05:00] +[titan] 2025-07-10 07:30:52,377 - root - INFO - step: 81455 loss: 16.0189 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4428 global_avg_mtp_loss: 13.5761 +[titan] 2025-07-10 07:30:52,377 - root - INFO - lr: 5.2710e-05 gnorm: 0.93 [17:55:49< 4:04:56] +[titan] 2025-07-10 07:30:56,333 - root - INFO - step: 81460 loss: 16.0602 memory: 44.58GiB(31.99%) tps: 82,840 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6030 +[titan] 2025-07-10 07:30:56,333 - root - INFO - lr: 5.2698e-05 gnorm: 0.95 [17:55:53< 4:04:52] +[titan] 2025-07-10 07:31:00,238 - root - INFO - step: 81465 loss: 16.1853 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7113 +[titan] 2025-07-10 07:31:00,238 - root - INFO - lr: 5.2686e-05 gnorm: 1.00 [17:55:57< 4:04:48] +[titan] 2025-07-10 07:31:04,166 - root - INFO - step: 81470 loss: 16.1777 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7079 +[titan] 2025-07-10 07:31:04,166 - root - INFO - lr: 5.2674e-05 gnorm: 0.98 [17:56:01< 4:04:44] +[titan] 2025-07-10 07:31:08,083 - root - INFO - step: 81475 loss: 16.1040 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6342 +[titan] 2025-07-10 07:31:08,084 - root - INFO - lr: 5.2662e-05 gnorm: 0.95 [17:56:05< 4:04:40] +[titan] 2025-07-10 07:31:11,984 - root - INFO - step: 81480 loss: 15.8858 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4180 global_avg_mtp_loss: 13.4679 +[titan] 2025-07-10 07:31:11,984 - root - INFO - lr: 5.2650e-05 gnorm: 0.99 [17:56:09< 4:04:36] +[titan] 2025-07-10 07:31:15,933 - root - INFO - step: 81485 loss: 16.5907 memory: 44.58GiB(31.99%) tps: 82,985 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.5493 global_avg_mtp_loss: 14.0414 +[titan] 2025-07-10 07:31:15,933 - root - INFO - lr: 5.2638e-05 gnorm: 0.94 [17:56:13< 4:04:32] +[titan] 2025-07-10 07:31:19,850 - root - INFO - step: 81490 loss: 16.2563 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7675 +[titan] 2025-07-10 07:31:19,850 - root - INFO - lr: 5.2626e-05 gnorm: 0.99 [17:56:16< 4:04:28] +[titan] 2025-07-10 07:31:23,804 - root - INFO - step: 81495 loss: 16.1777 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7056 +[titan] 2025-07-10 07:31:23,804 - root - INFO - lr: 5.2614e-05 gnorm: 1.01 [17:56:20< 4:04:24] +[titan] 2025-07-10 07:31:26,949 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:31:27,750 - root - INFO - step: 81500 loss: 16.1164 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4512 global_avg_mtp_loss: 13.6652 +[titan] 2025-07-10 07:31:27,750 - root - INFO - lr: 5.2603e-05 gnorm: 1.03 [17:56:24< 4:04:20] +[titan] 2025-07-10 07:31:31,668 - root - INFO - step: 81505 loss: 16.1132 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.6695 +[titan] 2025-07-10 07:31:31,668 - root - INFO - lr: 5.2591e-05 gnorm: 1.00 [17:56:28< 4:04:16] +[titan] 2025-07-10 07:31:35,575 - root - INFO - step: 81510 loss: 16.3781 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.8600 +[titan] 2025-07-10 07:31:35,575 - root - INFO - lr: 5.2579e-05 gnorm: 0.98 [17:56:32< 4:04:12] +[titan] 2025-07-10 07:31:39,479 - root - INFO - step: 81515 loss: 16.0396 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.5828 +[titan] 2025-07-10 07:31:39,479 - root - INFO - lr: 5.2567e-05 gnorm: 0.99 [17:56:36< 4:04:08] +[titan] 2025-07-10 07:31:43,391 - root - INFO - step: 81520 loss: 16.4732 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.5264 global_avg_mtp_loss: 13.9468 +[titan] 2025-07-10 07:31:43,391 - root - INFO - lr: 5.2555e-05 gnorm: 0.97 [17:56:40< 4:04:04] +[titan] 2025-07-10 07:31:47,307 - root - INFO - step: 81525 loss: 16.0116 memory: 44.58GiB(31.99%) tps: 83,700 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4331 global_avg_mtp_loss: 13.5785 +[titan] 2025-07-10 07:31:47,307 - root - INFO - lr: 5.2543e-05 gnorm: 0.93 [17:56:44< 4:04:00] +[titan] 2025-07-10 07:31:51,250 - root - INFO - step: 81530 loss: 16.2515 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7641 +[titan] 2025-07-10 07:31:51,251 - root - INFO - lr: 5.2531e-05 gnorm: 0.99 [17:56:48< 4:03:56] +[titan] 2025-07-10 07:31:55,172 - root - INFO - step: 81535 loss: 16.1176 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.6687 +[titan] 2025-07-10 07:31:55,172 - root - INFO - lr: 5.2520e-05 gnorm: 1.06 [17:56:52< 4:03:52] +[titan] 2025-07-10 07:31:59,124 - root - INFO - step: 81540 loss: 16.1360 memory: 44.58GiB(31.99%) tps: 82,918 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.4593 global_avg_mtp_loss: 13.6767 +[titan] 2025-07-10 07:31:59,124 - root - INFO - lr: 5.2508e-05 gnorm: 1.02 [17:56:56< 4:03:48] +[titan] 2025-07-10 07:32:03,041 - root - INFO - step: 81545 loss: 16.3298 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8217 +[titan] 2025-07-10 07:32:03,041 - root - INFO - lr: 5.2496e-05 gnorm: 1.03 [17:57:00< 4:03:44] +[titan] 2025-07-10 07:32:06,173 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:32:06,962 - root - INFO - step: 81550 loss: 16.4375 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.9208 +[titan] 2025-07-10 07:32:06,963 - root - INFO - lr: 5.2484e-05 gnorm: 1.03 [17:57:04< 4:03:40] +[titan] 2025-07-10 07:32:10,881 - root - INFO - step: 81555 loss: 16.2519 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 07:32:10,881 - root - INFO - lr: 5.2472e-05 gnorm: 0.96 [17:57:07< 4:03:36] +[titan] 2025-07-10 07:32:14,794 - root - INFO - step: 81560 loss: 16.2307 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.7489 +[titan] 2025-07-10 07:32:14,794 - root - INFO - lr: 5.2460e-05 gnorm: 1.02 [17:57:11< 4:03:32] +[titan] 2025-07-10 07:32:18,733 - root - INFO - step: 81565 loss: 16.2615 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 07:32:18,733 - root - INFO - lr: 5.2449e-05 gnorm: 0.97 [17:57:15< 4:03:28] +[titan] 2025-07-10 07:32:22,628 - root - INFO - step: 81570 loss: 16.1701 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6906 +[titan] 2025-07-10 07:32:22,628 - root - INFO - lr: 5.2437e-05 gnorm: 0.98 [17:57:19< 4:03:24] +[titan] 2025-07-10 07:32:26,525 - root - INFO - step: 81575 loss: 16.0367 memory: 44.58GiB(31.99%) tps: 84,081 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4503 global_avg_mtp_loss: 13.5864 +[titan] 2025-07-10 07:32:26,526 - root - INFO - lr: 5.2425e-05 gnorm: 0.94 [17:57:23< 4:03:20] +[titan] 2025-07-10 07:32:30,446 - root - INFO - step: 81580 loss: 16.1091 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.6463 +[titan] 2025-07-10 07:32:30,446 - root - INFO - lr: 5.2413e-05 gnorm: 0.96 [17:57:27< 4:03:16] +[titan] 2025-07-10 07:32:34,374 - root - INFO - step: 81585 loss: 16.3441 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8390 +[titan] 2025-07-10 07:32:34,375 - root - INFO - lr: 5.2401e-05 gnorm: 0.98 [17:57:31< 4:03:12] +[titan] 2025-07-10 07:32:38,327 - root - INFO - step: 81590 loss: 16.0752 memory: 44.58GiB(31.99%) tps: 82,916 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.4502 global_avg_mtp_loss: 13.6250 +[titan] 2025-07-10 07:32:38,327 - root - INFO - lr: 5.2390e-05 gnorm: 1.05 [17:57:35< 4:03:08] +[titan] 2025-07-10 07:32:42,235 - root - INFO - step: 81595 loss: 16.1378 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6835 +[titan] 2025-07-10 07:32:42,235 - root - INFO - lr: 5.2378e-05 gnorm: 0.98 [17:57:39< 4:03:04] +[titan] 2025-07-10 07:32:45,357 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:32:46,144 - root - INFO - step: 81600 loss: 16.1302 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.6702 +[titan] 2025-07-10 07:32:46,144 - root - INFO - lr: 5.2366e-05 gnorm: 0.95 [17:57:43< 4:03:00] +[titan] 2025-07-10 07:32:50,030 - root - INFO - step: 81605 loss: 15.8936 memory: 44.58GiB(31.99%) tps: 84,328 tflops: 291.03 mfu: 29.43% global_avg_ntp_loss: 2.4344 global_avg_mtp_loss: 13.4592 +[titan] 2025-07-10 07:32:50,030 - root - INFO - lr: 5.2354e-05 gnorm: 1.00 [17:57:47< 4:02:56] +[titan] 2025-07-10 07:32:53,966 - root - INFO - step: 81610 loss: 16.1375 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4562 global_avg_mtp_loss: 13.6813 +[titan] 2025-07-10 07:32:53,966 - root - INFO - lr: 5.2342e-05 gnorm: 1.03 [17:57:51< 4:02:52] +[titan] 2025-07-10 07:32:57,881 - root - INFO - step: 81615 loss: 16.3072 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8102 +[titan] 2025-07-10 07:32:57,882 - root - INFO - lr: 5.2330e-05 gnorm: 0.96 [17:57:54< 4:02:49] +[titan] 2025-07-10 07:33:01,791 - root - INFO - step: 81620 loss: 16.0959 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.6491 +[titan] 2025-07-10 07:33:01,791 - root - INFO - lr: 5.2319e-05 gnorm: 0.93 [17:57:58< 4:02:45] +[titan] 2025-07-10 07:33:05,725 - root - INFO - step: 81625 loss: 16.4189 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.9074 +[titan] 2025-07-10 07:33:05,725 - root - INFO - lr: 5.2307e-05 gnorm: 0.98 [17:58:02< 4:02:41] +[titan] 2025-07-10 07:33:09,656 - root - INFO - step: 81630 loss: 16.1290 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.6695 +[titan] 2025-07-10 07:33:09,656 - root - INFO - lr: 5.2295e-05 gnorm: 0.96 [17:58:06< 4:02:37] +[titan] 2025-07-10 07:33:13,589 - root - INFO - step: 81635 loss: 16.0693 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4477 global_avg_mtp_loss: 13.6216 +[titan] 2025-07-10 07:33:13,590 - root - INFO - lr: 5.2283e-05 gnorm: 0.95 [17:58:10< 4:02:33] +[titan] 2025-07-10 07:33:17,509 - root - INFO - step: 81640 loss: 16.0945 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.6413 +[titan] 2025-07-10 07:33:17,509 - root - INFO - lr: 5.2272e-05 gnorm: 1.02 [17:58:14< 4:02:29] +[titan] 2025-07-10 07:33:21,417 - root - INFO - step: 81645 loss: 16.4007 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8904 +[titan] 2025-07-10 07:33:21,417 - root - INFO - lr: 5.2260e-05 gnorm: 0.97 [17:58:18< 4:02:25] +[titan] 2025-07-10 07:33:24,540 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:33:25,330 - root - INFO - step: 81650 loss: 16.1481 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.6842 +[titan] 2025-07-10 07:33:25,330 - root - INFO - lr: 5.2248e-05 gnorm: 1.04 [17:58:22< 4:02:21] +[titan] 2025-07-10 07:33:29,268 - root - INFO - step: 81655 loss: 16.1697 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.6899 +[titan] 2025-07-10 07:33:29,268 - root - INFO - lr: 5.2236e-05 gnorm: 0.99 [17:58:26< 4:02:17] +[titan] 2025-07-10 07:33:33,183 - root - INFO - step: 81660 loss: 16.2921 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8004 +[titan] 2025-07-10 07:33:33,184 - root - INFO - lr: 5.2224e-05 gnorm: 0.99 [17:58:30< 4:02:13] +[titan] 2025-07-10 07:33:37,102 - root - INFO - step: 81665 loss: 16.2381 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7697 +[titan] 2025-07-10 07:33:37,103 - root - INFO - lr: 5.2213e-05 gnorm: 0.97 [17:58:34< 4:02:09] +[titan] 2025-07-10 07:33:41,030 - root - INFO - step: 81670 loss: 16.1558 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 07:33:41,031 - root - INFO - lr: 5.2201e-05 gnorm: 0.99 [17:58:38< 4:02:05] +[titan] 2025-07-10 07:33:44,969 - root - INFO - step: 81675 loss: 16.3412 memory: 44.58GiB(31.99%) tps: 83,203 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8424 +[titan] 2025-07-10 07:33:44,969 - root - INFO - lr: 5.2189e-05 gnorm: 0.94 [17:58:42< 4:02:01] +[titan] 2025-07-10 07:33:48,877 - root - INFO - step: 81680 loss: 16.1916 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7182 +[titan] 2025-07-10 07:33:48,877 - root - INFO - lr: 5.2177e-05 gnorm: 0.94 [17:58:45< 4:01:57] +[titan] 2025-07-10 07:33:52,773 - root - INFO - step: 81685 loss: 16.1771 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 07:33:52,773 - root - INFO - lr: 5.2166e-05 gnorm: 0.97 [17:58:49< 4:01:53] +[titan] 2025-07-10 07:33:56,730 - root - INFO - step: 81690 loss: 16.1645 memory: 44.58GiB(31.99%) tps: 82,811 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.4833 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 07:33:56,730 - root - INFO - lr: 5.2154e-05 gnorm: 0.98 [17:58:53< 4:01:49] +[titan] 2025-07-10 07:34:00,641 - root - INFO - step: 81695 loss: 16.5954 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5633 global_avg_mtp_loss: 14.0321 +[titan] 2025-07-10 07:34:00,641 - root - INFO - lr: 5.2142e-05 gnorm: 0.99 [17:58:57< 4:01:45] +[titan] 2025-07-10 07:34:03,752 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:34:04,542 - root - INFO - step: 81700 loss: 16.1252 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6612 +[titan] 2025-07-10 07:34:04,542 - root - INFO - lr: 5.2130e-05 gnorm: 0.98 [17:59:01< 4:01:41] +[titan] 2025-07-10 07:34:08,452 - root - INFO - step: 81705 loss: 16.3366 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8391 +[titan] 2025-07-10 07:34:08,453 - root - INFO - lr: 5.2119e-05 gnorm: 0.96 [17:59:05< 4:01:37] +[titan] 2025-07-10 07:34:12,395 - root - INFO - step: 81710 loss: 16.4057 memory: 44.58GiB(31.99%) tps: 83,125 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5183 global_avg_mtp_loss: 13.8873 +[titan] 2025-07-10 07:34:12,395 - root - INFO - lr: 5.2107e-05 gnorm: 0.98 [17:59:09< 4:01:33] +[titan] 2025-07-10 07:34:16,325 - root - INFO - step: 81715 loss: 16.3295 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8270 +[titan] 2025-07-10 07:34:16,325 - root - INFO - lr: 5.2095e-05 gnorm: 0.96 [17:59:13< 4:01:29] +[titan] 2025-07-10 07:34:20,212 - root - INFO - step: 81720 loss: 16.4121 memory: 44.58GiB(31.99%) tps: 84,305 tflops: 290.95 mfu: 29.42% global_avg_ntp_loss: 2.5118 global_avg_mtp_loss: 13.9003 +[titan] 2025-07-10 07:34:20,213 - root - INFO - lr: 5.2083e-05 gnorm: 0.95 [17:59:17< 4:01:25] +[titan] 2025-07-10 07:34:24,111 - root - INFO - step: 81725 loss: 16.1333 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6804 +[titan] 2025-07-10 07:34:24,111 - root - INFO - lr: 5.2072e-05 gnorm: 0.97 [17:59:21< 4:01:21] +[titan] 2025-07-10 07:34:28,008 - root - INFO - step: 81730 loss: 16.1284 memory: 44.58GiB(31.99%) tps: 84,086 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6728 +[titan] 2025-07-10 07:34:28,009 - root - INFO - lr: 5.2060e-05 gnorm: 0.93 [17:59:25< 4:01:17] +[titan] 2025-07-10 07:34:31,904 - root - INFO - step: 81735 loss: 16.3291 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8354 +[titan] 2025-07-10 07:34:31,904 - root - INFO - lr: 5.2048e-05 gnorm: 1.00 [17:59:29< 4:01:13] +[titan] 2025-07-10 07:34:35,825 - root - INFO - step: 81740 loss: 15.9856 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4315 global_avg_mtp_loss: 13.5541 +[titan] 2025-07-10 07:34:35,826 - root - INFO - lr: 5.2036e-05 gnorm: 1.00 [17:59:32< 4:01:09] +[titan] 2025-07-10 07:34:39,732 - root - INFO - step: 81745 loss: 16.5014 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 13.9682 +[titan] 2025-07-10 07:34:39,733 - root - INFO - lr: 5.2025e-05 gnorm: 0.96 [17:59:36< 4:01:05] +[titan] 2025-07-10 07:34:42,879 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:34:43,666 - root - INFO - step: 81750 loss: 16.2327 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7559 +[titan] 2025-07-10 07:34:43,666 - root - INFO - lr: 5.2013e-05 gnorm: 0.98 [17:59:40< 4:01:01] +[titan] 2025-07-10 07:34:47,560 - root - INFO - step: 81755 loss: 16.2431 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7625 +[titan] 2025-07-10 07:34:47,561 - root - INFO - lr: 5.2001e-05 gnorm: 1.02 [17:59:44< 4:00:57] +[titan] 2025-07-10 07:34:51,475 - root - INFO - step: 81760 loss: 16.0944 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.6383 +[titan] 2025-07-10 07:34:51,475 - root - INFO - lr: 5.1989e-05 gnorm: 0.97 [17:59:48< 4:00:53] +[titan] 2025-07-10 07:34:55,434 - root - INFO - step: 81765 loss: 16.3264 memory: 44.58GiB(31.99%) tps: 82,776 tflops: 285.67 mfu: 28.89% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8319 +[titan] 2025-07-10 07:34:55,434 - root - INFO - lr: 5.1978e-05 gnorm: 1.00 [17:59:52< 4:00:49] +[titan] 2025-07-10 07:34:59,396 - root - INFO - step: 81770 loss: 16.2517 memory: 44.58GiB(31.99%) tps: 82,705 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.7491 +[titan] 2025-07-10 07:34:59,396 - root - INFO - lr: 5.1966e-05 gnorm: 1.00 [17:59:56< 4:00:45] +[titan] 2025-07-10 07:35:03,321 - root - INFO - step: 81775 loss: 16.0673 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.6247 +[titan] 2025-07-10 07:35:03,321 - root - INFO - lr: 5.1954e-05 gnorm: 0.97 [18:00:00< 4:00:41] +[titan] 2025-07-10 07:35:07,234 - root - INFO - step: 81780 loss: 16.1934 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7290 +[titan] 2025-07-10 07:35:07,234 - root - INFO - lr: 5.1943e-05 gnorm: 0.95 [18:00:04< 4:00:37] +[titan] 2025-07-10 07:35:11,165 - root - INFO - step: 81785 loss: 16.5286 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5335 global_avg_mtp_loss: 13.9950 +[titan] 2025-07-10 07:35:11,165 - root - INFO - lr: 5.1931e-05 gnorm: 1.00 [18:00:08< 4:00:33] +[titan] 2025-07-10 07:35:15,091 - root - INFO - step: 81790 loss: 16.3756 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 07:35:15,092 - root - INFO - lr: 5.1919e-05 gnorm: 1.01 [18:00:12< 4:00:30] +[titan] 2025-07-10 07:35:18,990 - root - INFO - step: 81795 loss: 16.1358 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.6664 +[titan] 2025-07-10 07:35:18,990 - root - INFO - lr: 5.1908e-05 gnorm: 0.92 [18:00:16< 4:00:26] +[titan] 2025-07-10 07:35:22,096 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:35:22,882 - root - INFO - step: 81800 loss: 16.3260 memory: 44.58GiB(31.99%) tps: 84,192 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.8373 +[titan] 2025-07-10 07:35:22,883 - root - INFO - lr: 5.1896e-05 gnorm: 0.95 [18:00:19< 4:00:22] +[titan] 2025-07-10 07:35:26,787 - root - INFO - step: 81805 loss: 16.3370 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.8350 +[titan] 2025-07-10 07:35:26,788 - root - INFO - lr: 5.1884e-05 gnorm: 0.99 [18:00:23< 4:00:18] +[titan] 2025-07-10 07:35:30,759 - root - INFO - step: 81810 loss: 16.1650 memory: 44.58GiB(31.99%) tps: 82,521 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.6991 +[titan] 2025-07-10 07:35:30,759 - root - INFO - lr: 5.1872e-05 gnorm: 0.98 [18:00:27< 4:00:14] +[titan] 2025-07-10 07:35:34,682 - root - INFO - step: 81815 loss: 16.1385 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6734 +[titan] 2025-07-10 07:35:34,683 - root - INFO - lr: 5.1861e-05 gnorm: 0.95 [18:00:31< 4:00:10] +[titan] 2025-07-10 07:35:38,614 - root - INFO - step: 81820 loss: 16.2567 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7761 +[titan] 2025-07-10 07:35:38,614 - root - INFO - lr: 5.1849e-05 gnorm: 1.00 [18:00:35< 4:00:06] +[titan] 2025-07-10 07:35:42,533 - root - INFO - step: 81825 loss: 16.3006 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.8159 +[titan] 2025-07-10 07:35:42,534 - root - INFO - lr: 5.1837e-05 gnorm: 1.00 [18:00:39< 4:00:02] +[titan] 2025-07-10 07:35:46,443 - root - INFO - step: 81830 loss: 16.2499 memory: 44.58GiB(31.99%) tps: 83,824 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.7565 +[titan] 2025-07-10 07:35:46,443 - root - INFO - lr: 5.1826e-05 gnorm: 0.95 [18:00:43< 3:59:58] +[titan] 2025-07-10 07:35:50,371 - root - INFO - step: 81835 loss: 16.3958 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5071 global_avg_mtp_loss: 13.8886 +[titan] 2025-07-10 07:35:50,371 - root - INFO - lr: 5.1814e-05 gnorm: 0.96 [18:00:47< 3:59:54] +[titan] 2025-07-10 07:35:54,292 - root - INFO - step: 81840 loss: 16.1904 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7084 +[titan] 2025-07-10 07:35:54,292 - root - INFO - lr: 5.1802e-05 gnorm: 0.93 [18:00:51< 3:59:50] +[titan] 2025-07-10 07:35:58,225 - root - INFO - step: 81845 loss: 16.2777 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7880 +[titan] 2025-07-10 07:35:58,225 - root - INFO - lr: 5.1791e-05 gnorm: 0.93 [18:00:55< 3:59:46] +[titan] 2025-07-10 07:36:01,335 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:36:02,127 - root - INFO - step: 81850 loss: 15.9882 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4415 global_avg_mtp_loss: 13.5467 +[titan] 2025-07-10 07:36:02,127 - root - INFO - lr: 5.1779e-05 gnorm: 0.98 [18:00:59< 3:59:42] +[titan] 2025-07-10 07:36:06,054 - root - INFO - step: 81855 loss: 16.3556 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 07:36:06,054 - root - INFO - lr: 5.1767e-05 gnorm: 0.97 [18:01:03< 3:59:38] +[titan] 2025-07-10 07:36:09,958 - root - INFO - step: 81860 loss: 16.1165 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6569 +[titan] 2025-07-10 07:36:09,959 - root - INFO - lr: 5.1756e-05 gnorm: 0.95 [18:01:07< 3:59:34] +[titan] 2025-07-10 07:36:13,903 - root - INFO - step: 81865 loss: 16.0692 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.6259 +[titan] 2025-07-10 07:36:13,903 - root - INFO - lr: 5.1744e-05 gnorm: 0.95 [18:01:10< 3:59:30] +[titan] 2025-07-10 07:36:17,813 - root - INFO - step: 81870 loss: 16.2578 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7709 +[titan] 2025-07-10 07:36:17,814 - root - INFO - lr: 5.1732e-05 gnorm: 0.95 [18:01:14< 3:59:26] +[titan] 2025-07-10 07:36:21,756 - root - INFO - step: 81875 loss: 16.1560 memory: 44.58GiB(31.99%) tps: 83,113 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.6804 +[titan] 2025-07-10 07:36:21,757 - root - INFO - lr: 5.1721e-05 gnorm: 0.93 [18:01:18< 3:59:22] +[titan] 2025-07-10 07:36:25,667 - root - INFO - step: 81880 loss: 16.3477 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8483 +[titan] 2025-07-10 07:36:25,667 - root - INFO - lr: 5.1709e-05 gnorm: 0.96 [18:01:22< 3:59:18] +[titan] 2025-07-10 07:36:29,573 - root - INFO - step: 81885 loss: 16.4394 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9118 +[titan] 2025-07-10 07:36:29,573 - root - INFO - lr: 5.1697e-05 gnorm: 0.95 [18:01:26< 3:59:14] +[titan] 2025-07-10 07:36:33,482 - root - INFO - step: 81890 loss: 16.3924 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8814 +[titan] 2025-07-10 07:36:33,483 - root - INFO - lr: 5.1686e-05 gnorm: 0.95 [18:01:30< 3:59:10] +[titan] 2025-07-10 07:36:37,385 - root - INFO - step: 81895 loss: 16.1386 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 07:36:37,385 - root - INFO - lr: 5.1674e-05 gnorm: 0.96 [18:01:34< 3:59:06] +[titan] 2025-07-10 07:36:40,544 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:36:41,333 - root - INFO - step: 81900 loss: 16.1136 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.4654 global_avg_mtp_loss: 13.6482 +[titan] 2025-07-10 07:36:41,333 - root - INFO - lr: 5.1663e-05 gnorm: 0.97 [18:01:38< 3:59:02] +[titan] 2025-07-10 07:36:45,258 - root - INFO - step: 81905 loss: 16.2270 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7561 +[titan] 2025-07-10 07:36:45,258 - root - INFO - lr: 5.1651e-05 gnorm: 1.01 [18:01:42< 3:58:58] +[titan] 2025-07-10 07:36:49,222 - root - INFO - step: 81910 loss: 16.1225 memory: 44.58GiB(31.99%) tps: 82,666 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6527 +[titan] 2025-07-10 07:36:49,223 - root - INFO - lr: 5.1639e-05 gnorm: 1.00 [18:01:46< 3:58:54] +[titan] 2025-07-10 07:36:53,123 - root - INFO - step: 81915 loss: 16.2363 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7568 +[titan] 2025-07-10 07:36:53,123 - root - INFO - lr: 5.1628e-05 gnorm: 0.99 [18:01:50< 3:58:50] +[titan] 2025-07-10 07:36:57,131 - root - INFO - step: 81920 loss: 16.1389 memory: 44.58GiB(31.99%) tps: 81,757 tflops: 282.16 mfu: 28.53% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6679 +[titan] 2025-07-10 07:36:57,132 - root - INFO - lr: 5.1616e-05 gnorm: 1.02 [18:01:54< 3:58:46] +[titan] 2025-07-10 07:36:57,285 - root - INFO - Dumping profiler traces at step 81920 +[titan] 2025-07-10 07:36:57,318 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:37:01,301 - root - INFO - step: 81925 loss: 16.3292 memory: 44.58GiB(31.99%) tps: 78,585 tflops: 271.21 mfu: 27.42% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-10 07:37:01,302 - root - INFO - lr: 5.1604e-05 gnorm: 0.96 [18:01:58< 3:58:42] +[titan] 2025-07-10 07:37:05,225 - root - INFO - step: 81930 loss: 16.5018 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5337 global_avg_mtp_loss: 13.9681 +[titan] 2025-07-10 07:37:05,225 - root - INFO - lr: 5.1593e-05 gnorm: 1.00 [18:02:02< 3:58:38] +[titan] 2025-07-10 07:37:09,138 - root - INFO - step: 81935 loss: 16.3838 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8699 +[titan] 2025-07-10 07:37:09,139 - root - INFO - lr: 5.1581e-05 gnorm: 1.01 [18:02:06< 3:58:34] +[titan] 2025-07-10 07:37:13,049 - root - INFO - step: 81940 loss: 16.2653 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.8021 +[titan] 2025-07-10 07:37:13,049 - root - INFO - lr: 5.1570e-05 gnorm: 1.01 [18:02:10< 3:58:30] +[titan] 2025-07-10 07:37:16,963 - root - INFO - step: 81945 loss: 16.3037 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.8224 +[titan] 2025-07-10 07:37:16,964 - root - INFO - lr: 5.1558e-05 gnorm: 0.99 [18:02:14< 3:58:26] +[titan] 2025-07-10 07:37:20,088 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:37:20,884 - root - INFO - step: 81950 loss: 16.1168 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6621 +[titan] 2025-07-10 07:37:20,884 - root - INFO - lr: 5.1546e-05 gnorm: 1.03 [18:02:17< 3:58:22] +[titan] 2025-07-10 07:37:24,811 - root - INFO - step: 81955 loss: 16.3720 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8617 +[titan] 2025-07-10 07:37:24,811 - root - INFO - lr: 5.1535e-05 gnorm: 0.96 [18:02:21< 3:58:19] +[titan] 2025-07-10 07:37:28,725 - root - INFO - step: 81960 loss: 16.2232 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7423 +[titan] 2025-07-10 07:37:28,725 - root - INFO - lr: 5.1523e-05 gnorm: 0.99 [18:02:25< 3:58:15] +[titan] 2025-07-10 07:37:32,663 - root - INFO - step: 81965 loss: 16.3427 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-10 07:37:32,663 - root - INFO - lr: 5.1511e-05 gnorm: 0.97 [18:02:29< 3:58:11] +[titan] 2025-07-10 07:37:36,630 - root - INFO - step: 81970 loss: 16.2078 memory: 44.58GiB(31.99%) tps: 82,605 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7309 +[titan] 2025-07-10 07:37:36,630 - root - INFO - lr: 5.1500e-05 gnorm: 1.03 [18:02:33< 3:58:07] +[titan] 2025-07-10 07:37:40,551 - root - INFO - step: 81975 loss: 16.3294 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8279 +[titan] 2025-07-10 07:37:40,551 - root - INFO - lr: 5.1488e-05 gnorm: 0.95 [18:02:37< 3:58:03] +[titan] 2025-07-10 07:37:44,476 - root - INFO - step: 81980 loss: 16.1607 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6966 +[titan] 2025-07-10 07:37:44,476 - root - INFO - lr: 5.1477e-05 gnorm: 1.02 [18:02:41< 3:57:59] +[titan] 2025-07-10 07:37:48,409 - root - INFO - step: 81985 loss: 16.2144 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7391 +[titan] 2025-07-10 07:37:48,410 - root - INFO - lr: 5.1465e-05 gnorm: 1.08 [18:02:45< 3:57:55] +[titan] 2025-07-10 07:37:52,348 - root - INFO - step: 81990 loss: 16.2695 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7839 +[titan] 2025-07-10 07:37:52,349 - root - INFO - lr: 5.1454e-05 gnorm: 0.96 [18:02:49< 3:57:51] +[titan] 2025-07-10 07:37:56,282 - root - INFO - step: 81995 loss: 16.2710 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4893 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 07:37:56,282 - root - INFO - lr: 5.1442e-05 gnorm: 0.98 [18:02:53< 3:57:47] +[titan] 2025-07-10 07:37:59,460 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:38:00,251 - root - INFO - step: 82000 loss: 16.3762 memory: 44.58GiB(31.99%) tps: 82,576 tflops: 284.98 mfu: 28.82% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8761 +[titan] 2025-07-10 07:38:00,251 - root - INFO - lr: 5.1430e-05 gnorm: 0.97 [18:02:57< 3:57:43] +[titan] 2025-07-10 07:38:04,229 - root - INFO - step: 82005 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 82,369 tflops: 284.27 mfu: 28.74% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.7994 +[titan] 2025-07-10 07:38:04,230 - root - INFO - lr: 5.1419e-05 gnorm: 0.99 [18:03:01< 3:57:39] +[titan] 2025-07-10 07:38:08,126 - root - INFO - step: 82010 loss: 16.1576 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.6869 +[titan] 2025-07-10 07:38:08,126 - root - INFO - lr: 5.1407e-05 gnorm: 0.97 [18:03:05< 3:57:35] +[titan] 2025-07-10 07:38:12,026 - root - INFO - step: 82015 loss: 16.3212 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.8410 +[titan] 2025-07-10 07:38:12,027 - root - INFO - lr: 5.1396e-05 gnorm: 0.97 [18:03:09< 3:57:31] +[titan] 2025-07-10 07:38:15,942 - root - INFO - step: 82020 loss: 15.9518 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4302 global_avg_mtp_loss: 13.5216 +[titan] 2025-07-10 07:38:15,942 - root - INFO - lr: 5.1384e-05 gnorm: 0.99 [18:03:13< 3:57:27] +[titan] 2025-07-10 07:38:19,854 - root - INFO - step: 82025 loss: 16.2361 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7530 +[titan] 2025-07-10 07:38:19,854 - root - INFO - lr: 5.1372e-05 gnorm: 0.94 [18:03:16< 3:57:23] +[titan] 2025-07-10 07:38:23,809 - root - INFO - step: 82030 loss: 16.3774 memory: 44.58GiB(31.99%) tps: 82,856 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8822 +[titan] 2025-07-10 07:38:23,809 - root - INFO - lr: 5.1361e-05 gnorm: 0.99 [18:03:20< 3:57:19] +[titan] 2025-07-10 07:38:27,725 - root - INFO - step: 82035 loss: 16.0095 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.5602 +[titan] 2025-07-10 07:38:27,725 - root - INFO - lr: 5.1349e-05 gnorm: 0.95 [18:03:24< 3:57:15] +[titan] 2025-07-10 07:38:31,651 - root - INFO - step: 82040 loss: 16.5152 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5279 global_avg_mtp_loss: 13.9873 +[titan] 2025-07-10 07:38:31,651 - root - INFO - lr: 5.1338e-05 gnorm: 0.93 [18:03:28< 3:57:11] +[titan] 2025-07-10 07:38:35,556 - root - INFO - step: 82045 loss: 16.2182 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4621 global_avg_mtp_loss: 13.7561 +[titan] 2025-07-10 07:38:35,557 - root - INFO - lr: 5.1326e-05 gnorm: 1.08 [18:03:32< 3:57:07] +[titan] 2025-07-10 07:38:38,695 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:38:39,484 - root - INFO - step: 82050 loss: 16.1763 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4679 global_avg_mtp_loss: 13.7085 +[titan] 2025-07-10 07:38:39,484 - root - INFO - lr: 5.1315e-05 gnorm: 0.97 [18:03:36< 3:57:03] +[titan] 2025-07-10 07:38:43,435 - root - INFO - step: 82055 loss: 16.3198 memory: 44.58GiB(31.99%) tps: 82,939 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8237 +[titan] 2025-07-10 07:38:43,436 - root - INFO - lr: 5.1303e-05 gnorm: 0.96 [18:03:40< 3:56:59] +[titan] 2025-07-10 07:38:47,357 - root - INFO - step: 82060 loss: 16.2664 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 07:38:47,358 - root - INFO - lr: 5.1292e-05 gnorm: 0.96 [18:03:44< 3:56:55] +[titan] 2025-07-10 07:38:51,261 - root - INFO - step: 82065 loss: 16.2663 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7863 +[titan] 2025-07-10 07:38:51,262 - root - INFO - lr: 5.1280e-05 gnorm: 1.00 [18:03:48< 3:56:51] +[titan] 2025-07-10 07:38:55,177 - root - INFO - step: 82070 loss: 16.0805 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4574 global_avg_mtp_loss: 13.6232 +[titan] 2025-07-10 07:38:55,177 - root - INFO - lr: 5.1269e-05 gnorm: 1.04 [18:03:52< 3:56:47] +[titan] 2025-07-10 07:38:59,111 - root - INFO - step: 82075 loss: 16.3739 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 07:38:59,111 - root - INFO - lr: 5.1257e-05 gnorm: 0.97 [18:03:56< 3:56:43] +[titan] 2025-07-10 07:39:03,004 - root - INFO - step: 82080 loss: 16.2649 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.7685 +[titan] 2025-07-10 07:39:03,004 - root - INFO - lr: 5.1245e-05 gnorm: 0.98 [18:04:00< 3:56:39] +[titan] 2025-07-10 07:39:06,917 - root - INFO - step: 82085 loss: 16.0664 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.6148 +[titan] 2025-07-10 07:39:06,918 - root - INFO - lr: 5.1234e-05 gnorm: 1.02 [18:04:03< 3:56:35] +[titan] 2025-07-10 07:39:10,819 - root - INFO - step: 82090 loss: 16.0747 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.6212 +[titan] 2025-07-10 07:39:10,819 - root - INFO - lr: 5.1222e-05 gnorm: 0.98 [18:04:07< 3:56:31] +[titan] 2025-07-10 07:39:14,723 - root - INFO - step: 82095 loss: 16.5296 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.5349 global_avg_mtp_loss: 13.9947 +[titan] 2025-07-10 07:39:14,723 - root - INFO - lr: 5.1211e-05 gnorm: 0.99 [18:04:11< 3:56:27] +[titan] 2025-07-10 07:39:17,869 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:39:18,663 - root - INFO - step: 82100 loss: 16.6136 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.5743 global_avg_mtp_loss: 14.0393 +[titan] 2025-07-10 07:39:18,664 - root - INFO - lr: 5.1199e-05 gnorm: 1.02 [18:04:15< 3:56:23] +[titan] 2025-07-10 07:39:22,584 - root - INFO - step: 82105 loss: 16.2422 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7689 +[titan] 2025-07-10 07:39:22,585 - root - INFO - lr: 5.1188e-05 gnorm: 1.01 [18:04:19< 3:56:19] +[titan] 2025-07-10 07:39:26,501 - root - INFO - step: 82110 loss: 16.1883 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7179 +[titan] 2025-07-10 07:39:26,501 - root - INFO - lr: 5.1176e-05 gnorm: 0.95 [18:04:23< 3:56:15] +[titan] 2025-07-10 07:39:30,423 - root - INFO - step: 82115 loss: 16.5423 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5363 global_avg_mtp_loss: 14.0061 +[titan] 2025-07-10 07:39:30,423 - root - INFO - lr: 5.1165e-05 gnorm: 1.02 [18:04:27< 3:56:11] +[titan] 2025-07-10 07:39:34,332 - root - INFO - step: 82120 loss: 16.0083 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.5770 +[titan] 2025-07-10 07:39:34,332 - root - INFO - lr: 5.1153e-05 gnorm: 1.00 [18:04:31< 3:56:08] +[titan] 2025-07-10 07:39:38,257 - root - INFO - step: 82125 loss: 16.2852 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7901 +[titan] 2025-07-10 07:39:38,257 - root - INFO - lr: 5.1142e-05 gnorm: 1.03 [18:04:35< 3:56:04] +[titan] 2025-07-10 07:39:42,200 - root - INFO - step: 82130 loss: 16.2489 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7631 +[titan] 2025-07-10 07:39:42,200 - root - INFO - lr: 5.1130e-05 gnorm: 0.99 [18:04:39< 3:56:00] +[titan] 2025-07-10 07:39:46,130 - root - INFO - step: 82135 loss: 16.2545 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7742 +[titan] 2025-07-10 07:39:46,130 - root - INFO - lr: 5.1119e-05 gnorm: 0.98 [18:04:43< 3:55:56] +[titan] 2025-07-10 07:39:50,096 - root - INFO - step: 82140 loss: 16.2361 memory: 44.58GiB(31.99%) tps: 82,614 tflops: 285.11 mfu: 28.83% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7641 +[titan] 2025-07-10 07:39:50,097 - root - INFO - lr: 5.1107e-05 gnorm: 0.94 [18:04:47< 3:55:52] +[titan] 2025-07-10 07:39:54,078 - root - INFO - step: 82145 loss: 16.2805 memory: 44.58GiB(31.99%) tps: 82,302 tflops: 284.04 mfu: 28.72% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.7973 +[titan] 2025-07-10 07:39:54,078 - root - INFO - lr: 5.1096e-05 gnorm: 0.95 [18:04:51< 3:55:48] +[titan] 2025-07-10 07:39:57,225 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:39:58,015 - root - INFO - step: 82150 loss: 16.2413 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.7559 +[titan] 2025-07-10 07:39:58,016 - root - INFO - lr: 5.1084e-05 gnorm: 0.95 [18:04:55< 3:55:44] +[titan] 2025-07-10 07:40:01,918 - root - INFO - step: 82155 loss: 16.5147 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9764 +[titan] 2025-07-10 07:40:01,918 - root - INFO - lr: 5.1073e-05 gnorm: 0.97 [18:04:58< 3:55:40] +[titan] 2025-07-10 07:40:05,866 - root - INFO - step: 82160 loss: 16.1271 memory: 44.58GiB(31.99%) tps: 82,999 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 07:40:05,867 - root - INFO - lr: 5.1061e-05 gnorm: 0.99 [18:05:02< 3:55:36] +[titan] 2025-07-10 07:40:09,789 - root - INFO - step: 82165 loss: 16.3095 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8197 +[titan] 2025-07-10 07:40:09,789 - root - INFO - lr: 5.1050e-05 gnorm: 0.95 [18:05:06< 3:55:32] +[titan] 2025-07-10 07:40:13,708 - root - INFO - step: 82170 loss: 16.0976 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6488 +[titan] 2025-07-10 07:40:13,708 - root - INFO - lr: 5.1038e-05 gnorm: 1.00 [18:05:10< 3:55:28] +[titan] 2025-07-10 07:40:17,616 - root - INFO - step: 82175 loss: 16.3572 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8658 +[titan] 2025-07-10 07:40:17,616 - root - INFO - lr: 5.1027e-05 gnorm: 0.95 [18:05:14< 3:55:24] +[titan] 2025-07-10 07:40:21,551 - root - INFO - step: 82180 loss: 16.2587 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7711 +[titan] 2025-07-10 07:40:21,552 - root - INFO - lr: 5.1015e-05 gnorm: 0.96 [18:05:18< 3:55:20] +[titan] 2025-07-10 07:40:25,521 - root - INFO - step: 82185 loss: 16.1945 memory: 44.58GiB(31.99%) tps: 82,552 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7250 +[titan] 2025-07-10 07:40:25,522 - root - INFO - lr: 5.1004e-05 gnorm: 0.97 [18:05:22< 3:55:16] +[titan] 2025-07-10 07:40:29,455 - root - INFO - step: 82190 loss: 16.2666 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 07:40:29,455 - root - INFO - lr: 5.0992e-05 gnorm: 0.97 [18:05:26< 3:55:12] +[titan] 2025-07-10 07:40:33,411 - root - INFO - step: 82195 loss: 16.4070 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8994 +[titan] 2025-07-10 07:40:33,411 - root - INFO - lr: 5.0981e-05 gnorm: 1.08 [18:05:30< 3:55:08] +[titan] 2025-07-10 07:40:36,551 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:40:37,343 - root - INFO - step: 82200 loss: 16.2781 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.7779 +[titan] 2025-07-10 07:40:37,343 - root - INFO - lr: 5.0969e-05 gnorm: 0.94 [18:05:34< 3:55:04] +[titan] 2025-07-10 07:40:41,258 - root - INFO - step: 82205 loss: 15.9661 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4290 global_avg_mtp_loss: 13.5371 +[titan] 2025-07-10 07:40:41,258 - root - INFO - lr: 5.0958e-05 gnorm: 0.99 [18:05:38< 3:55:00] +[titan] 2025-07-10 07:40:45,178 - root - INFO - step: 82210 loss: 15.9060 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4205 global_avg_mtp_loss: 13.4855 +[titan] 2025-07-10 07:40:45,179 - root - INFO - lr: 5.0947e-05 gnorm: 0.99 [18:05:42< 3:54:56] +[titan] 2025-07-10 07:40:49,118 - root - INFO - step: 82215 loss: 16.2753 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7896 +[titan] 2025-07-10 07:40:49,118 - root - INFO - lr: 5.0935e-05 gnorm: 0.95 [18:05:46< 3:54:52] +[titan] 2025-07-10 07:40:53,029 - root - INFO - step: 82220 loss: 16.0670 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4388 global_avg_mtp_loss: 13.6281 +[titan] 2025-07-10 07:40:53,030 - root - INFO - lr: 5.0924e-05 gnorm: 0.98 [18:05:50< 3:54:48] +[titan] 2025-07-10 07:40:56,955 - root - INFO - step: 82225 loss: 16.1875 memory: 44.58GiB(31.99%) tps: 83,471 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.7203 +[titan] 2025-07-10 07:40:56,956 - root - INFO - lr: 5.0912e-05 gnorm: 0.98 [18:05:54< 3:54:44] +[titan] 2025-07-10 07:41:00,899 - root - INFO - step: 82230 loss: 16.2253 memory: 44.58GiB(31.99%) tps: 83,096 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7499 +[titan] 2025-07-10 07:41:00,900 - root - INFO - lr: 5.0901e-05 gnorm: 1.01 [18:05:57< 3:54:40] +[titan] 2025-07-10 07:41:04,819 - root - INFO - step: 82235 loss: 16.0949 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6362 +[titan] 2025-07-10 07:41:04,820 - root - INFO - lr: 5.0889e-05 gnorm: 1.01 [18:06:01< 3:54:36] +[titan] 2025-07-10 07:41:08,757 - root - INFO - step: 82240 loss: 16.2437 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7509 +[titan] 2025-07-10 07:41:08,757 - root - INFO - lr: 5.0878e-05 gnorm: 1.05 [18:06:05< 3:54:32] +[titan] 2025-07-10 07:41:12,686 - root - INFO - step: 82245 loss: 16.2474 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7664 +[titan] 2025-07-10 07:41:12,686 - root - INFO - lr: 5.0866e-05 gnorm: 1.01 [18:06:09< 3:54:28] +[titan] 2025-07-10 07:41:15,799 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:41:16,589 - root - INFO - step: 82250 loss: 16.1323 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6734 +[titan] 2025-07-10 07:41:16,589 - root - INFO - lr: 5.0855e-05 gnorm: 0.97 [18:06:13< 3:54:24] +[titan] 2025-07-10 07:41:20,490 - root - INFO - step: 82255 loss: 16.2506 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7731 +[titan] 2025-07-10 07:41:20,490 - root - INFO - lr: 5.0843e-05 gnorm: 0.99 [18:06:17< 3:54:20] +[titan] 2025-07-10 07:41:24,418 - root - INFO - step: 82260 loss: 16.3920 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8892 +[titan] 2025-07-10 07:41:24,419 - root - INFO - lr: 5.0832e-05 gnorm: 1.00 [18:06:21< 3:54:16] +[titan] 2025-07-10 07:41:28,348 - root - INFO - step: 82265 loss: 16.1147 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6558 +[titan] 2025-07-10 07:41:28,349 - root - INFO - lr: 5.0821e-05 gnorm: 0.95 [18:06:25< 3:54:12] +[titan] 2025-07-10 07:41:32,246 - root - INFO - step: 82270 loss: 16.0463 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.6001 +[titan] 2025-07-10 07:41:32,247 - root - INFO - lr: 5.0809e-05 gnorm: 1.00 [18:06:29< 3:54:08] +[titan] 2025-07-10 07:41:36,184 - root - INFO - step: 82275 loss: 16.3219 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8280 +[titan] 2025-07-10 07:41:36,184 - root - INFO - lr: 5.0798e-05 gnorm: 0.98 [18:06:33< 3:54:04] +[titan] 2025-07-10 07:41:40,110 - root - INFO - step: 82280 loss: 15.9955 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.5574 +[titan] 2025-07-10 07:41:40,110 - root - INFO - lr: 5.0786e-05 gnorm: 1.03 [18:06:37< 3:54:01] +[titan] 2025-07-10 07:41:44,018 - root - INFO - step: 82285 loss: 16.4203 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.9060 +[titan] 2025-07-10 07:41:44,018 - root - INFO - lr: 5.0775e-05 gnorm: 0.96 [18:06:41< 3:53:57] +[titan] 2025-07-10 07:41:47,931 - root - INFO - step: 82290 loss: 16.2452 memory: 44.58GiB(31.99%) tps: 83,746 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7644 +[titan] 2025-07-10 07:41:47,931 - root - INFO - lr: 5.0764e-05 gnorm: 0.95 [18:06:44< 3:53:53] +[titan] 2025-07-10 07:41:51,861 - root - INFO - step: 82295 loss: 16.3708 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 07:41:51,862 - root - INFO - lr: 5.0752e-05 gnorm: 0.94 [18:06:48< 3:53:49] +[titan] 2025-07-10 07:41:54,983 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:41:55,772 - root - INFO - step: 82300 loss: 16.3897 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5172 global_avg_mtp_loss: 13.8725 +[titan] 2025-07-10 07:41:55,772 - root - INFO - lr: 5.0741e-05 gnorm: 1.03 [18:06:52< 3:53:45] +[titan] 2025-07-10 07:41:59,743 - root - INFO - step: 82305 loss: 16.2573 memory: 44.58GiB(31.99%) tps: 82,519 tflops: 284.79 mfu: 28.80% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7759 +[titan] 2025-07-10 07:41:59,743 - root - INFO - lr: 5.0729e-05 gnorm: 0.99 [18:06:56< 3:53:41] +[titan] 2025-07-10 07:42:03,677 - root - INFO - step: 82310 loss: 16.1760 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.7057 +[titan] 2025-07-10 07:42:03,677 - root - INFO - lr: 5.0718e-05 gnorm: 0.96 [18:07:00< 3:53:37] +[titan] 2025-07-10 07:42:07,605 - root - INFO - step: 82315 loss: 16.2146 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7261 +[titan] 2025-07-10 07:42:07,605 - root - INFO - lr: 5.0706e-05 gnorm: 0.95 [18:07:04< 3:53:33] +[titan] 2025-07-10 07:42:11,545 - root - INFO - step: 82320 loss: 16.2770 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7903 +[titan] 2025-07-10 07:42:11,546 - root - INFO - lr: 5.0695e-05 gnorm: 0.94 [18:07:08< 3:53:29] +[titan] 2025-07-10 07:42:15,473 - root - INFO - step: 82325 loss: 16.1925 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4559 global_avg_mtp_loss: 13.7366 +[titan] 2025-07-10 07:42:15,473 - root - INFO - lr: 5.0684e-05 gnorm: 0.95 [18:07:12< 3:53:25] +[titan] 2025-07-10 07:42:19,410 - root - INFO - step: 82330 loss: 16.1705 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.6923 +[titan] 2025-07-10 07:42:19,411 - root - INFO - lr: 5.0672e-05 gnorm: 1.09 [18:07:16< 3:53:21] +[titan] 2025-07-10 07:42:23,369 - root - INFO - step: 82335 loss: 16.0859 memory: 44.58GiB(31.99%) tps: 82,786 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.4438 global_avg_mtp_loss: 13.6421 +[titan] 2025-07-10 07:42:23,369 - root - INFO - lr: 5.0661e-05 gnorm: 1.25 [18:07:20< 3:53:17] +[titan] 2025-07-10 07:42:27,351 - root - INFO - step: 82340 loss: 16.2305 memory: 44.58GiB(31.99%) tps: 82,302 tflops: 284.04 mfu: 28.72% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.7394 +[titan] 2025-07-10 07:42:27,351 - root - INFO - lr: 5.0650e-05 gnorm: 0.93 [18:07:24< 3:53:13] +[titan] 2025-07-10 07:42:31,263 - root - INFO - step: 82345 loss: 16.0295 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4378 global_avg_mtp_loss: 13.5918 +[titan] 2025-07-10 07:42:31,263 - root - INFO - lr: 5.0638e-05 gnorm: 0.98 [18:07:28< 3:53:09] +[titan] 2025-07-10 07:42:34,411 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:42:35,202 - root - INFO - step: 82350 loss: 15.9464 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4241 global_avg_mtp_loss: 13.5223 +[titan] 2025-07-10 07:42:35,202 - root - INFO - lr: 5.0627e-05 gnorm: 1.02 [18:07:32< 3:53:05] +[titan] 2025-07-10 07:42:39,137 - root - INFO - step: 82355 loss: 16.2483 memory: 44.58GiB(31.99%) tps: 83,272 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7684 +[titan] 2025-07-10 07:42:39,137 - root - INFO - lr: 5.0615e-05 gnorm: 1.01 [18:07:36< 3:53:01] +[titan] 2025-07-10 07:42:43,080 - root - INFO - step: 82360 loss: 16.2309 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7451 +[titan] 2025-07-10 07:42:43,081 - root - INFO - lr: 5.0604e-05 gnorm: 0.96 [18:07:40< 3:52:57] +[titan] 2025-07-10 07:42:46,996 - root - INFO - step: 82365 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.9128 +[titan] 2025-07-10 07:42:46,996 - root - INFO - lr: 5.0593e-05 gnorm: 1.00 [18:07:44< 3:52:53] +[titan] 2025-07-10 07:42:50,898 - root - INFO - step: 82370 loss: 16.1396 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6721 +[titan] 2025-07-10 07:42:50,898 - root - INFO - lr: 5.0581e-05 gnorm: 0.97 [18:07:47< 3:52:49] +[titan] 2025-07-10 07:42:54,800 - root - INFO - step: 82375 loss: 16.3717 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8729 +[titan] 2025-07-10 07:42:54,800 - root - INFO - lr: 5.0570e-05 gnorm: 0.99 [18:07:51< 3:52:45] +[titan] 2025-07-10 07:42:58,712 - root - INFO - step: 82380 loss: 16.1624 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.6957 +[titan] 2025-07-10 07:42:58,712 - root - INFO - lr: 5.0559e-05 gnorm: 0.99 [18:07:55< 3:52:41] +[titan] 2025-07-10 07:43:02,639 - root - INFO - step: 82385 loss: 16.2611 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7849 +[titan] 2025-07-10 07:43:02,639 - root - INFO - lr: 5.0547e-05 gnorm: 1.02 [18:07:59< 3:52:37] +[titan] 2025-07-10 07:43:06,543 - root - INFO - step: 82390 loss: 16.4240 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5128 global_avg_mtp_loss: 13.9113 +[titan] 2025-07-10 07:43:06,544 - root - INFO - lr: 5.0536e-05 gnorm: 1.01 [18:08:03< 3:52:33] +[titan] 2025-07-10 07:43:10,503 - root - INFO - step: 82395 loss: 16.0526 memory: 44.58GiB(31.99%) tps: 82,768 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.4407 global_avg_mtp_loss: 13.6118 +[titan] 2025-07-10 07:43:10,503 - root - INFO - lr: 5.0524e-05 gnorm: 1.01 [18:08:07< 3:52:29] +[titan] 2025-07-10 07:43:13,620 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:43:14,415 - root - INFO - step: 82400 loss: 16.1388 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.6708 +[titan] 2025-07-10 07:43:14,415 - root - INFO - lr: 5.0513e-05 gnorm: 0.96 [18:08:11< 3:52:25] +[titan] 2025-07-10 07:43:18,311 - root - INFO - step: 82405 loss: 16.2182 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7376 +[titan] 2025-07-10 07:43:18,312 - root - INFO - lr: 5.0502e-05 gnorm: 0.96 [18:08:15< 3:52:21] +[titan] 2025-07-10 07:43:22,248 - root - INFO - step: 82410 loss: 16.3310 memory: 44.58GiB(31.99%) tps: 83,246 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8332 +[titan] 2025-07-10 07:43:22,248 - root - INFO - lr: 5.0490e-05 gnorm: 0.95 [18:08:19< 3:52:17] +[titan] 2025-07-10 07:43:26,190 - root - INFO - step: 82415 loss: 16.2376 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7657 +[titan] 2025-07-10 07:43:26,190 - root - INFO - lr: 5.0479e-05 gnorm: 0.99 [18:08:23< 3:52:13] +[titan] 2025-07-10 07:43:30,118 - root - INFO - step: 82420 loss: 16.2950 memory: 44.58GiB(31.99%) tps: 83,429 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4893 global_avg_mtp_loss: 13.8057 +[titan] 2025-07-10 07:43:30,118 - root - INFO - lr: 5.0468e-05 gnorm: 0.92 [18:08:27< 3:52:09] +[titan] 2025-07-10 07:43:34,016 - root - INFO - step: 82425 loss: 16.3757 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8651 +[titan] 2025-07-10 07:43:34,017 - root - INFO - lr: 5.0456e-05 gnorm: 1.02 [18:08:31< 3:52:05] +[titan] 2025-07-10 07:43:38,003 - root - INFO - step: 82430 loss: 16.4794 memory: 44.58GiB(31.99%) tps: 82,211 tflops: 283.72 mfu: 28.69% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9562 +[titan] 2025-07-10 07:43:38,003 - root - INFO - lr: 5.0445e-05 gnorm: 1.09 [18:08:35< 3:52:01] +[titan] 2025-07-10 07:43:39,721 - root - INFO - Dumping profiler traces at step 82432 +[titan] 2025-07-10 07:43:39,753 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:43:42,113 - root - INFO - step: 82435 loss: 16.4698 memory: 44.58GiB(31.99%) tps: 79,729 tflops: 275.16 mfu: 27.82% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.9525 +[titan] 2025-07-10 07:43:42,113 - root - INFO - lr: 5.0434e-05 gnorm: 0.96 [18:08:39< 3:51:58] +[titan] 2025-07-10 07:43:46,016 - root - INFO - step: 82440 loss: 16.2286 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7478 +[titan] 2025-07-10 07:43:46,016 - root - INFO - lr: 5.0422e-05 gnorm: 0.97 [18:08:43< 3:51:54] +[titan] 2025-07-10 07:43:49,918 - root - INFO - step: 82445 loss: 16.3465 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8501 +[titan] 2025-07-10 07:43:49,918 - root - INFO - lr: 5.0411e-05 gnorm: 0.99 [18:08:46< 3:51:50] +[titan] 2025-07-10 07:43:53,042 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:43:53,841 - root - INFO - step: 82450 loss: 16.2867 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7995 +[titan] 2025-07-10 07:43:53,841 - root - INFO - lr: 5.0400e-05 gnorm: 0.94 [18:08:50< 3:51:46] +[titan] 2025-07-10 07:43:57,781 - root - INFO - step: 82455 loss: 16.3964 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8920 +[titan] 2025-07-10 07:43:57,781 - root - INFO - lr: 5.0388e-05 gnorm: 0.99 [18:08:54< 3:51:42] +[titan] 2025-07-10 07:44:01,690 - root - INFO - step: 82460 loss: 16.1108 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4555 global_avg_mtp_loss: 13.6553 +[titan] 2025-07-10 07:44:01,690 - root - INFO - lr: 5.0377e-05 gnorm: 0.97 [18:08:58< 3:51:38] +[titan] 2025-07-10 07:44:05,599 - root - INFO - step: 82465 loss: 16.0130 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.5692 +[titan] 2025-07-10 07:44:05,600 - root - INFO - lr: 5.0366e-05 gnorm: 0.96 [18:09:02< 3:51:34] +[titan] 2025-07-10 07:44:09,521 - root - INFO - step: 82470 loss: 16.4256 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.9116 +[titan] 2025-07-10 07:44:09,522 - root - INFO - lr: 5.0354e-05 gnorm: 0.95 [18:09:06< 3:51:30] +[titan] 2025-07-10 07:44:13,438 - root - INFO - step: 82475 loss: 16.2312 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7560 +[titan] 2025-07-10 07:44:13,438 - root - INFO - lr: 5.0343e-05 gnorm: 0.97 [18:09:10< 3:51:26] +[titan] 2025-07-10 07:44:17,408 - root - INFO - step: 82480 loss: 16.1012 memory: 44.58GiB(31.99%) tps: 82,536 tflops: 284.84 mfu: 28.80% global_avg_ntp_loss: 2.4558 global_avg_mtp_loss: 13.6454 +[titan] 2025-07-10 07:44:17,408 - root - INFO - lr: 5.0332e-05 gnorm: 1.01 [18:09:14< 3:51:22] +[titan] 2025-07-10 07:44:21,351 - root - INFO - step: 82485 loss: 16.2235 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7515 +[titan] 2025-07-10 07:44:21,351 - root - INFO - lr: 5.0321e-05 gnorm: 1.02 [18:09:18< 3:51:18] +[titan] 2025-07-10 07:44:25,272 - root - INFO - step: 82490 loss: 16.3474 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8454 +[titan] 2025-07-10 07:44:25,273 - root - INFO - lr: 5.0309e-05 gnorm: 1.09 [18:09:22< 3:51:14] +[titan] 2025-07-10 07:44:29,223 - root - INFO - step: 82495 loss: 16.2303 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7519 +[titan] 2025-07-10 07:44:29,223 - root - INFO - lr: 5.0298e-05 gnorm: 0.99 [18:09:26< 3:51:10] +[titan] 2025-07-10 07:44:32,384 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:44:33,171 - root - INFO - step: 82500 loss: 16.3342 memory: 44.58GiB(31.99%) tps: 83,006 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 07:44:33,171 - root - INFO - lr: 5.0287e-05 gnorm: 1.01 [18:09:30< 3:51:06] +[titan] 2025-07-10 07:44:37,074 - root - INFO - step: 82505 loss: 16.3653 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 07:44:37,074 - root - INFO - lr: 5.0275e-05 gnorm: 0.97 [18:09:34< 3:51:02] +[titan] 2025-07-10 07:44:40,998 - root - INFO - step: 82510 loss: 16.1909 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4663 global_avg_mtp_loss: 13.7245 +[titan] 2025-07-10 07:44:40,998 - root - INFO - lr: 5.0264e-05 gnorm: 0.99 [18:09:38< 3:50:58] +[titan] 2025-07-10 07:44:44,900 - root - INFO - step: 82515 loss: 16.4222 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.9065 +[titan] 2025-07-10 07:44:44,900 - root - INFO - lr: 5.0253e-05 gnorm: 0.96 [18:09:41< 3:50:54] +[titan] 2025-07-10 07:44:48,801 - root - INFO - step: 82520 loss: 16.3588 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8627 +[titan] 2025-07-10 07:44:48,802 - root - INFO - lr: 5.0242e-05 gnorm: 0.95 [18:09:45< 3:50:50] +[titan] 2025-07-10 07:44:52,697 - root - INFO - step: 82525 loss: 16.4920 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.5404 global_avg_mtp_loss: 13.9516 +[titan] 2025-07-10 07:44:52,697 - root - INFO - lr: 5.0230e-05 gnorm: 1.00 [18:09:49< 3:50:46] +[titan] 2025-07-10 07:44:56,637 - root - INFO - step: 82530 loss: 16.1902 memory: 44.58GiB(31.99%) tps: 83,183 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7155 +[titan] 2025-07-10 07:44:56,637 - root - INFO - lr: 5.0219e-05 gnorm: 1.01 [18:09:53< 3:50:42] +[titan] 2025-07-10 07:45:00,558 - root - INFO - step: 82535 loss: 16.2508 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.7581 +[titan] 2025-07-10 07:45:00,558 - root - INFO - lr: 5.0208e-05 gnorm: 1.00 [18:09:57< 3:50:38] +[titan] 2025-07-10 07:45:04,480 - root - INFO - step: 82540 loss: 16.1904 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7147 +[titan] 2025-07-10 07:45:04,480 - root - INFO - lr: 5.0196e-05 gnorm: 1.02 [18:10:01< 3:50:34] +[titan] 2025-07-10 07:45:08,394 - root - INFO - step: 82545 loss: 15.9758 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5362 +[titan] 2025-07-10 07:45:08,395 - root - INFO - lr: 5.0185e-05 gnorm: 1.01 [18:10:05< 3:50:30] +[titan] 2025-07-10 07:45:11,506 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:45:12,298 - root - INFO - step: 82550 loss: 16.2857 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.7950 +[titan] 2025-07-10 07:45:12,299 - root - INFO - lr: 5.0174e-05 gnorm: 1.00 [18:10:09< 3:50:26] +[titan] 2025-07-10 07:45:16,238 - root - INFO - step: 82555 loss: 16.0425 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.6023 +[titan] 2025-07-10 07:45:16,238 - root - INFO - lr: 5.0163e-05 gnorm: 1.02 [18:10:13< 3:50:22] +[titan] 2025-07-10 07:45:20,152 - root - INFO - step: 82560 loss: 16.1829 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7109 +[titan] 2025-07-10 07:45:20,152 - root - INFO - lr: 5.0151e-05 gnorm: 1.02 [18:10:17< 3:50:18] +[titan] 2025-07-10 07:45:24,065 - root - INFO - step: 82565 loss: 15.9798 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4453 global_avg_mtp_loss: 13.5346 +[titan] 2025-07-10 07:45:24,065 - root - INFO - lr: 5.0140e-05 gnorm: 1.03 [18:10:21< 3:50:14] +[titan] 2025-07-10 07:45:27,967 - root - INFO - step: 82570 loss: 16.1251 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6713 +[titan] 2025-07-10 07:45:27,968 - root - INFO - lr: 5.0129e-05 gnorm: 1.07 [18:10:24< 3:50:10] +[titan] 2025-07-10 07:45:31,892 - root - INFO - step: 82575 loss: 16.1463 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 07:45:31,893 - root - INFO - lr: 5.0118e-05 gnorm: 0.97 [18:10:28< 3:50:06] +[titan] 2025-07-10 07:45:35,841 - root - INFO - step: 82580 loss: 16.0359 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.97% global_avg_ntp_loss: 2.4337 global_avg_mtp_loss: 13.6022 +[titan] 2025-07-10 07:45:35,841 - root - INFO - lr: 5.0106e-05 gnorm: 1.04 [18:10:32< 3:50:02] +[titan] 2025-07-10 07:45:39,775 - root - INFO - step: 82585 loss: 16.3720 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8607 +[titan] 2025-07-10 07:45:39,775 - root - INFO - lr: 5.0095e-05 gnorm: 1.01 [18:10:36< 3:49:58] +[titan] 2025-07-10 07:45:43,731 - root - INFO - step: 82590 loss: 16.2003 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7251 +[titan] 2025-07-10 07:45:43,731 - root - INFO - lr: 5.0084e-05 gnorm: 0.95 [18:10:40< 3:49:54] +[titan] 2025-07-10 07:45:47,679 - root - INFO - step: 82595 loss: 16.4309 memory: 44.58GiB(31.99%) tps: 83,013 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5223 global_avg_mtp_loss: 13.9087 +[titan] 2025-07-10 07:45:47,679 - root - INFO - lr: 5.0073e-05 gnorm: 0.97 [18:10:44< 3:49:50] +[titan] 2025-07-10 07:45:50,835 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:45:51,635 - root - INFO - step: 82600 loss: 16.1127 memory: 44.58GiB(31.99%) tps: 82,829 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6523 +[titan] 2025-07-10 07:45:51,636 - root - INFO - lr: 5.0061e-05 gnorm: 0.93 [18:10:48< 3:49:47] +[titan] 2025-07-10 07:45:55,549 - root - INFO - step: 82605 loss: 16.2228 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7475 +[titan] 2025-07-10 07:45:55,549 - root - INFO - lr: 5.0050e-05 gnorm: 0.95 [18:10:52< 3:49:43] +[titan] 2025-07-10 07:45:59,496 - root - INFO - step: 82610 loss: 16.5284 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5412 global_avg_mtp_loss: 13.9873 +[titan] 2025-07-10 07:45:59,496 - root - INFO - lr: 5.0039e-05 gnorm: 0.96 [18:10:56< 3:49:39] +[titan] 2025-07-10 07:46:03,431 - root - INFO - step: 82615 loss: 16.3661 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5103 global_avg_mtp_loss: 13.8558 +[titan] 2025-07-10 07:46:03,431 - root - INFO - lr: 5.0028e-05 gnorm: 0.96 [18:11:00< 3:49:35] +[titan] 2025-07-10 07:46:07,333 - root - INFO - step: 82620 loss: 16.2008 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.7201 +[titan] 2025-07-10 07:46:07,333 - root - INFO - lr: 5.0016e-05 gnorm: 1.00 [18:11:04< 3:49:31] +[titan] 2025-07-10 07:46:11,247 - root - INFO - step: 82625 loss: 16.2820 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.7752 +[titan] 2025-07-10 07:46:11,247 - root - INFO - lr: 5.0005e-05 gnorm: 0.97 [18:11:08< 3:49:27] +[titan] 2025-07-10 07:46:15,155 - root - INFO - step: 82630 loss: 16.4966 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5352 global_avg_mtp_loss: 13.9613 +[titan] 2025-07-10 07:46:15,156 - root - INFO - lr: 4.9994e-05 gnorm: 0.96 [18:11:12< 3:49:23] +[titan] 2025-07-10 07:46:19,072 - root - INFO - step: 82635 loss: 16.2694 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7886 +[titan] 2025-07-10 07:46:19,073 - root - INFO - lr: 4.9983e-05 gnorm: 0.95 [18:11:16< 3:49:19] +[titan] 2025-07-10 07:46:22,994 - root - INFO - step: 82640 loss: 16.3246 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8294 +[titan] 2025-07-10 07:46:22,995 - root - INFO - lr: 4.9972e-05 gnorm: 0.96 [18:11:20< 3:49:15] +[titan] 2025-07-10 07:46:26,925 - root - INFO - step: 82645 loss: 16.3794 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8699 +[titan] 2025-07-10 07:46:26,925 - root - INFO - lr: 4.9960e-05 gnorm: 0.95 [18:11:23< 3:49:11] +[titan] 2025-07-10 07:46:30,061 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:46:30,854 - root - INFO - step: 82650 loss: 16.3974 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8911 +[titan] 2025-07-10 07:46:30,854 - root - INFO - lr: 4.9949e-05 gnorm: 1.00 [18:11:27< 3:49:07] +[titan] 2025-07-10 07:46:34,767 - root - INFO - step: 82655 loss: 16.2934 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.8057 +[titan] 2025-07-10 07:46:34,767 - root - INFO - lr: 4.9938e-05 gnorm: 0.96 [18:11:31< 3:49:03] +[titan] 2025-07-10 07:46:38,685 - root - INFO - step: 82660 loss: 16.2923 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7993 +[titan] 2025-07-10 07:46:38,686 - root - INFO - lr: 4.9927e-05 gnorm: 1.02 [18:11:35< 3:48:59] +[titan] 2025-07-10 07:46:42,601 - root - INFO - step: 82665 loss: 16.1753 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.7123 +[titan] 2025-07-10 07:46:42,601 - root - INFO - lr: 4.9916e-05 gnorm: 1.03 [18:11:39< 3:48:55] +[titan] 2025-07-10 07:46:46,535 - root - INFO - step: 82670 loss: 16.4691 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5198 global_avg_mtp_loss: 13.9494 +[titan] 2025-07-10 07:46:46,535 - root - INFO - lr: 4.9904e-05 gnorm: 0.98 [18:11:43< 3:48:51] +[titan] 2025-07-10 07:46:50,443 - root - INFO - step: 82675 loss: 16.5230 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9856 +[titan] 2025-07-10 07:46:50,444 - root - INFO - lr: 4.9893e-05 gnorm: 0.99 [18:11:47< 3:48:47] +[titan] 2025-07-10 07:46:54,365 - root - INFO - step: 82680 loss: 16.2250 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7512 +[titan] 2025-07-10 07:46:54,365 - root - INFO - lr: 4.9882e-05 gnorm: 0.99 [18:11:51< 3:48:43] +[titan] 2025-07-10 07:46:58,325 - root - INFO - step: 82685 loss: 16.0674 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.4471 global_avg_mtp_loss: 13.6203 +[titan] 2025-07-10 07:46:58,326 - root - INFO - lr: 4.9871e-05 gnorm: 0.94 [18:11:55< 3:48:39] +[titan] 2025-07-10 07:47:02,235 - root - INFO - step: 82690 loss: 16.0913 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4413 global_avg_mtp_loss: 13.6500 +[titan] 2025-07-10 07:47:02,235 - root - INFO - lr: 4.9860e-05 gnorm: 0.96 [18:11:59< 3:48:35] +[titan] 2025-07-10 07:47:06,184 - root - INFO - step: 82695 loss: 16.2794 memory: 44.58GiB(31.99%) tps: 82,981 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 07:47:06,185 - root - INFO - lr: 4.9848e-05 gnorm: 1.01 [18:12:03< 3:48:31] +[titan] 2025-07-10 07:47:09,308 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:47:10,108 - root - INFO - step: 82700 loss: 15.9627 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4364 global_avg_mtp_loss: 13.5263 +[titan] 2025-07-10 07:47:10,109 - root - INFO - lr: 4.9837e-05 gnorm: 0.98 [18:12:07< 3:48:27] +[titan] 2025-07-10 07:47:14,038 - root - INFO - step: 82705 loss: 16.3653 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8579 +[titan] 2025-07-10 07:47:14,039 - root - INFO - lr: 4.9826e-05 gnorm: 0.98 [18:12:11< 3:48:23] +[titan] 2025-07-10 07:47:17,976 - root - INFO - step: 82710 loss: 16.0640 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4525 global_avg_mtp_loss: 13.6114 +[titan] 2025-07-10 07:47:17,976 - root - INFO - lr: 4.9815e-05 gnorm: 1.00 [18:12:14< 3:48:19] +[titan] 2025-07-10 07:47:21,909 - root - INFO - step: 82715 loss: 16.1273 memory: 44.58GiB(31.99%) tps: 83,321 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6659 +[titan] 2025-07-10 07:47:21,910 - root - INFO - lr: 4.9804e-05 gnorm: 0.98 [18:12:18< 3:48:15] +[titan] 2025-07-10 07:47:25,812 - root - INFO - step: 82720 loss: 16.0758 memory: 44.58GiB(31.99%) tps: 83,979 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.6285 +[titan] 2025-07-10 07:47:25,812 - root - INFO - lr: 4.9793e-05 gnorm: 0.99 [18:12:22< 3:48:11] +[titan] 2025-07-10 07:47:29,710 - root - INFO - step: 82725 loss: 16.2023 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.7198 +[titan] 2025-07-10 07:47:29,710 - root - INFO - lr: 4.9781e-05 gnorm: 1.07 [18:12:26< 3:48:07] +[titan] 2025-07-10 07:47:33,643 - root - INFO - step: 82730 loss: 16.3688 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8710 +[titan] 2025-07-10 07:47:33,643 - root - INFO - lr: 4.9770e-05 gnorm: 0.99 [18:12:30< 3:48:03] +[titan] 2025-07-10 07:47:37,551 - root - INFO - step: 82735 loss: 16.2237 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7435 +[titan] 2025-07-10 07:47:37,551 - root - INFO - lr: 4.9759e-05 gnorm: 1.02 [18:12:34< 3:47:59] +[titan] 2025-07-10 07:47:41,459 - root - INFO - step: 82740 loss: 16.2998 memory: 44.58GiB(31.99%) tps: 83,850 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4941 global_avg_mtp_loss: 13.8057 +[titan] 2025-07-10 07:47:41,460 - root - INFO - lr: 4.9748e-05 gnorm: 0.99 [18:12:38< 3:47:55] +[titan] 2025-07-10 07:47:45,392 - root - INFO - step: 82745 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7372 +[titan] 2025-07-10 07:47:45,392 - root - INFO - lr: 4.9737e-05 gnorm: 0.96 [18:12:42< 3:47:51] +[titan] 2025-07-10 07:47:48,506 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:47:49,297 - root - INFO - step: 82750 loss: 16.1553 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.6828 +[titan] 2025-07-10 07:47:49,297 - root - INFO - lr: 4.9726e-05 gnorm: 1.01 [18:12:46< 3:47:47] +[titan] 2025-07-10 07:47:53,194 - root - INFO - step: 82755 loss: 16.2092 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4730 global_avg_mtp_loss: 13.7362 +[titan] 2025-07-10 07:47:53,194 - root - INFO - lr: 4.9714e-05 gnorm: 0.93 [18:12:50< 3:47:43] +[titan] 2025-07-10 07:47:57,098 - root - INFO - step: 82760 loss: 16.2509 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7776 +[titan] 2025-07-10 07:47:57,098 - root - INFO - lr: 4.9703e-05 gnorm: 0.98 [18:12:54< 3:47:39] +[titan] 2025-07-10 07:48:01,019 - root - INFO - step: 82765 loss: 16.0602 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.6167 +[titan] 2025-07-10 07:48:01,019 - root - INFO - lr: 4.9692e-05 gnorm: 0.94 [18:12:58< 3:47:35] +[titan] 2025-07-10 07:48:04,944 - root - INFO - step: 82770 loss: 16.7273 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5683 global_avg_mtp_loss: 14.1590 +[titan] 2025-07-10 07:48:04,944 - root - INFO - lr: 4.9681e-05 gnorm: 0.94 [18:13:01< 3:47:32] +[titan] 2025-07-10 07:48:08,863 - root - INFO - step: 82775 loss: 16.4311 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.9185 +[titan] 2025-07-10 07:48:08,864 - root - INFO - lr: 4.9670e-05 gnorm: 0.99 [18:13:05< 3:47:28] +[titan] 2025-07-10 07:48:12,785 - root - INFO - step: 82780 loss: 16.3505 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.8552 +[titan] 2025-07-10 07:48:12,785 - root - INFO - lr: 4.9659e-05 gnorm: 0.98 [18:13:09< 3:47:24] +[titan] 2025-07-10 07:48:16,702 - root - INFO - step: 82785 loss: 16.3499 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8477 +[titan] 2025-07-10 07:48:16,702 - root - INFO - lr: 4.9648e-05 gnorm: 0.94 [18:13:13< 3:47:20] +[titan] 2025-07-10 07:48:20,623 - root - INFO - step: 82790 loss: 16.2726 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.7822 +[titan] 2025-07-10 07:48:20,623 - root - INFO - lr: 4.9637e-05 gnorm: 0.98 [18:13:17< 3:47:16] +[titan] 2025-07-10 07:48:24,574 - root - INFO - step: 82795 loss: 16.2591 memory: 44.58GiB(31.99%) tps: 82,948 tflops: 286.27 mfu: 28.94% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7719 +[titan] 2025-07-10 07:48:24,574 - root - INFO - lr: 4.9625e-05 gnorm: 1.07 [18:13:21< 3:47:12] +[titan] 2025-07-10 07:48:27,714 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:48:28,512 - root - INFO - step: 82800 loss: 16.2473 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7582 +[titan] 2025-07-10 07:48:28,512 - root - INFO - lr: 4.9614e-05 gnorm: 0.95 [18:13:25< 3:47:08] +[titan] 2025-07-10 07:48:32,458 - root - INFO - step: 82805 loss: 16.2585 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 07:48:32,458 - root - INFO - lr: 4.9603e-05 gnorm: 0.95 [18:13:29< 3:47:04] +[titan] 2025-07-10 07:48:36,365 - root - INFO - step: 82810 loss: 16.4722 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5321 global_avg_mtp_loss: 13.9402 +[titan] 2025-07-10 07:48:36,365 - root - INFO - lr: 4.9592e-05 gnorm: 1.05 [18:13:33< 3:47:00] +[titan] 2025-07-10 07:48:40,262 - root - INFO - step: 82815 loss: 16.2739 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7883 +[titan] 2025-07-10 07:48:40,263 - root - INFO - lr: 4.9581e-05 gnorm: 0.97 [18:13:37< 3:46:56] +[titan] 2025-07-10 07:48:44,190 - root - INFO - step: 82820 loss: 15.8675 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4246 global_avg_mtp_loss: 13.4429 +[titan] 2025-07-10 07:48:44,191 - root - INFO - lr: 4.9570e-05 gnorm: 1.04 [18:13:41< 3:46:52] +[titan] 2025-07-10 07:48:48,123 - root - INFO - step: 82825 loss: 16.1132 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.6503 +[titan] 2025-07-10 07:48:48,123 - root - INFO - lr: 4.9559e-05 gnorm: 1.04 [18:13:45< 3:46:48] +[titan] 2025-07-10 07:48:52,029 - root - INFO - step: 82830 loss: 16.6282 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5660 global_avg_mtp_loss: 14.0622 +[titan] 2025-07-10 07:48:52,030 - root - INFO - lr: 4.9548e-05 gnorm: 1.09 [18:13:49< 3:46:44] +[titan] 2025-07-10 07:48:55,949 - root - INFO - step: 82835 loss: 16.3256 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 07:48:55,949 - root - INFO - lr: 4.9537e-05 gnorm: 0.95 [18:13:52< 3:46:40] +[titan] 2025-07-10 07:48:59,871 - root - INFO - step: 82840 loss: 16.4035 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8866 +[titan] 2025-07-10 07:48:59,872 - root - INFO - lr: 4.9525e-05 gnorm: 0.99 [18:13:56< 3:46:36] +[titan] 2025-07-10 07:49:03,816 - root - INFO - step: 82845 loss: 16.2624 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7750 +[titan] 2025-07-10 07:49:03,816 - root - INFO - lr: 4.9514e-05 gnorm: 1.00 [18:14:00< 3:46:32] +[titan] 2025-07-10 07:49:06,937 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:49:07,727 - root - INFO - step: 82850 loss: 16.2974 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.7999 +[titan] 2025-07-10 07:49:07,727 - root - INFO - lr: 4.9503e-05 gnorm: 1.03 [18:14:04< 3:46:28] +[titan] 2025-07-10 07:49:11,651 - root - INFO - step: 82855 loss: 16.2503 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.7582 +[titan] 2025-07-10 07:49:11,652 - root - INFO - lr: 4.9492e-05 gnorm: 1.05 [18:14:08< 3:46:24] +[titan] 2025-07-10 07:49:15,604 - root - INFO - step: 82860 loss: 16.3490 memory: 44.58GiB(31.99%) tps: 82,902 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8478 +[titan] 2025-07-10 07:49:15,605 - root - INFO - lr: 4.9481e-05 gnorm: 0.97 [18:14:12< 3:46:20] +[titan] 2025-07-10 07:49:19,518 - root - INFO - step: 82865 loss: 16.3488 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.8520 +[titan] 2025-07-10 07:49:19,519 - root - INFO - lr: 4.9470e-05 gnorm: 1.00 [18:14:16< 3:46:16] +[titan] 2025-07-10 07:49:23,442 - root - INFO - step: 82870 loss: 16.1903 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7243 +[titan] 2025-07-10 07:49:23,442 - root - INFO - lr: 4.9459e-05 gnorm: 1.01 [18:14:20< 3:46:12] +[titan] 2025-07-10 07:49:27,345 - root - INFO - step: 82875 loss: 16.4964 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-10 07:49:27,345 - root - INFO - lr: 4.9448e-05 gnorm: 0.92 [18:14:24< 3:46:08] +[titan] 2025-07-10 07:49:31,249 - root - INFO - step: 82880 loss: 15.9716 memory: 44.58GiB(31.99%) tps: 83,939 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4254 global_avg_mtp_loss: 13.5463 +[titan] 2025-07-10 07:49:31,250 - root - INFO - lr: 4.9437e-05 gnorm: 0.94 [18:14:28< 3:46:04] +[titan] 2025-07-10 07:49:35,146 - root - INFO - step: 82885 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 84,097 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7426 +[titan] 2025-07-10 07:49:35,146 - root - INFO - lr: 4.9426e-05 gnorm: 0.98 [18:14:32< 3:46:00] +[titan] 2025-07-10 07:49:39,048 - root - INFO - step: 82890 loss: 16.2624 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4835 global_avg_mtp_loss: 13.7789 +[titan] 2025-07-10 07:49:39,048 - root - INFO - lr: 4.9415e-05 gnorm: 0.95 [18:14:36< 3:45:56] +[titan] 2025-07-10 07:49:42,991 - root - INFO - step: 82895 loss: 16.0918 memory: 44.58GiB(31.99%) tps: 83,106 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6333 +[titan] 2025-07-10 07:49:42,991 - root - INFO - lr: 4.9404e-05 gnorm: 0.96 [18:14:39< 3:45:52] +[titan] 2025-07-10 07:49:46,114 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:49:46,903 - root - INFO - step: 82900 loss: 16.0828 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6260 +[titan] 2025-07-10 07:49:46,903 - root - INFO - lr: 4.9392e-05 gnorm: 0.95 [18:14:43< 3:45:48] +[titan] 2025-07-10 07:49:50,823 - root - INFO - step: 82905 loss: 16.2614 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.7647 +[titan] 2025-07-10 07:49:50,824 - root - INFO - lr: 4.9381e-05 gnorm: 0.97 [18:14:47< 3:45:44] +[titan] 2025-07-10 07:49:54,780 - root - INFO - step: 82910 loss: 16.1165 memory: 44.58GiB(31.99%) tps: 82,826 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6513 +[titan] 2025-07-10 07:49:54,780 - root - INFO - lr: 4.9370e-05 gnorm: 0.99 [18:14:51< 3:45:40] +[titan] 2025-07-10 07:49:58,694 - root - INFO - step: 82915 loss: 16.2984 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8022 +[titan] 2025-07-10 07:49:58,695 - root - INFO - lr: 4.9359e-05 gnorm: 1.02 [18:14:55< 3:45:36] +[titan] 2025-07-10 07:50:02,635 - root - INFO - step: 82920 loss: 15.9173 memory: 44.58GiB(31.99%) tps: 83,159 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4256 global_avg_mtp_loss: 13.4918 +[titan] 2025-07-10 07:50:02,635 - root - INFO - lr: 4.9348e-05 gnorm: 0.94 [18:14:59< 3:45:32] +[titan] 2025-07-10 07:50:06,541 - root - INFO - step: 82925 loss: 15.9913 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4403 global_avg_mtp_loss: 13.5510 +[titan] 2025-07-10 07:50:06,542 - root - INFO - lr: 4.9337e-05 gnorm: 0.98 [18:15:03< 3:45:28] +[titan] 2025-07-10 07:50:10,479 - root - INFO - step: 82930 loss: 16.1383 memory: 44.58GiB(31.99%) tps: 83,229 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4536 global_avg_mtp_loss: 13.6847 +[titan] 2025-07-10 07:50:10,479 - root - INFO - lr: 4.9326e-05 gnorm: 0.99 [18:15:07< 3:45:24] +[titan] 2025-07-10 07:50:14,436 - root - INFO - step: 82935 loss: 16.2977 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.8106 +[titan] 2025-07-10 07:50:14,437 - root - INFO - lr: 4.9315e-05 gnorm: 0.99 [18:15:11< 3:45:21] +[titan] 2025-07-10 07:50:18,332 - root - INFO - step: 82940 loss: 16.2041 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7293 +[titan] 2025-07-10 07:50:18,333 - root - INFO - lr: 4.9304e-05 gnorm: 1.00 [18:15:15< 3:45:17] +[titan] 2025-07-10 07:50:21,676 - root - INFO - Dumping profiler traces at step 82944 +[titan] 2025-07-10 07:50:21,708 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 07:50:22,505 - root - INFO - step: 82945 loss: 16.2511 memory: 44.58GiB(31.99%) tps: 78,542 tflops: 271.06 mfu: 27.41% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.7845 +[titan] 2025-07-10 07:50:22,505 - root - INFO - lr: 4.9293e-05 gnorm: 1.04 [18:15:19< 3:45:13] +[titan] 2025-07-10 07:50:25,618 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:50:26,405 - root - INFO - step: 82950 loss: 16.2086 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7239 +[titan] 2025-07-10 07:50:26,405 - root - INFO - lr: 4.9282e-05 gnorm: 1.08 [18:15:23< 3:45:09] +[titan] 2025-07-10 07:50:30,306 - root - INFO - step: 82955 loss: 16.2402 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7533 +[titan] 2025-07-10 07:50:30,307 - root - INFO - lr: 4.9271e-05 gnorm: 0.95 [18:15:27< 3:45:05] +[titan] 2025-07-10 07:50:34,226 - root - INFO - step: 82960 loss: 16.4088 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8946 +[titan] 2025-07-10 07:50:34,226 - root - INFO - lr: 4.9260e-05 gnorm: 1.05 [18:15:31< 3:45:01] +[titan] 2025-07-10 07:50:38,133 - root - INFO - step: 82965 loss: 16.1356 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.6821 +[titan] 2025-07-10 07:50:38,134 - root - INFO - lr: 4.9249e-05 gnorm: 1.00 [18:15:35< 3:44:57] +[titan] 2025-07-10 07:50:42,054 - root - INFO - step: 82970 loss: 16.0919 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.6276 +[titan] 2025-07-10 07:50:42,054 - root - INFO - lr: 4.9238e-05 gnorm: 1.03 [18:15:39< 3:44:53] +[titan] 2025-07-10 07:50:45,987 - root - INFO - step: 82975 loss: 15.8951 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4295 global_avg_mtp_loss: 13.4656 +[titan] 2025-07-10 07:50:45,988 - root - INFO - lr: 4.9227e-05 gnorm: 0.98 [18:15:42< 3:44:49] +[titan] 2025-07-10 07:50:49,910 - root - INFO - step: 82980 loss: 16.4087 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5289 global_avg_mtp_loss: 13.8797 +[titan] 2025-07-10 07:50:49,910 - root - INFO - lr: 4.9216e-05 gnorm: 1.08 [18:15:46< 3:44:45] +[titan] 2025-07-10 07:50:53,912 - root - INFO - step: 82985 loss: 16.0738 memory: 44.58GiB(31.99%) tps: 81,881 tflops: 282.59 mfu: 28.57% global_avg_ntp_loss: 2.4460 global_avg_mtp_loss: 13.6278 +[titan] 2025-07-10 07:50:53,912 - root - INFO - lr: 4.9205e-05 gnorm: 1.03 [18:15:50< 3:44:41] +[titan] 2025-07-10 07:50:57,834 - root - INFO - step: 82990 loss: 15.9049 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4194 global_avg_mtp_loss: 13.4855 +[titan] 2025-07-10 07:50:57,835 - root - INFO - lr: 4.9194e-05 gnorm: 0.94 [18:15:54< 3:44:37] +[titan] 2025-07-10 07:51:01,789 - root - INFO - step: 82995 loss: 16.1319 memory: 44.58GiB(31.99%) tps: 82,868 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.4558 global_avg_mtp_loss: 13.6761 +[titan] 2025-07-10 07:51:01,789 - root - INFO - lr: 4.9183e-05 gnorm: 0.96 [18:15:58< 3:44:33] +[titan] 2025-07-10 07:51:04,917 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:51:05,728 - root - INFO - step: 83000 loss: 16.0928 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.6328 +[titan] 2025-07-10 07:51:05,728 - root - INFO - lr: 4.9172e-05 gnorm: 0.96 [18:16:02< 3:44:29] +[titan] 2025-07-10 07:51:09,646 - root - INFO - step: 83005 loss: 15.9600 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4350 global_avg_mtp_loss: 13.5250 +[titan] 2025-07-10 07:51:09,646 - root - INFO - lr: 4.9161e-05 gnorm: 0.97 [18:16:06< 3:44:25] +[titan] 2025-07-10 07:51:13,574 - root - INFO - step: 83010 loss: 16.1863 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7155 +[titan] 2025-07-10 07:51:13,574 - root - INFO - lr: 4.9150e-05 gnorm: 0.98 [18:16:10< 3:44:21] +[titan] 2025-07-10 07:51:17,487 - root - INFO - step: 83015 loss: 16.3922 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8874 +[titan] 2025-07-10 07:51:17,488 - root - INFO - lr: 4.9139e-05 gnorm: 0.98 [18:16:14< 3:44:17] +[titan] 2025-07-10 07:51:21,434 - root - INFO - step: 83020 loss: 16.1110 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.6409 +[titan] 2025-07-10 07:51:21,434 - root - INFO - lr: 4.9128e-05 gnorm: 1.02 [18:16:18< 3:44:13] +[titan] 2025-07-10 07:51:25,366 - root - INFO - step: 83025 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8193 +[titan] 2025-07-10 07:51:25,366 - root - INFO - lr: 4.9117e-05 gnorm: 0.96 [18:16:22< 3:44:09] +[titan] 2025-07-10 07:51:29,318 - root - INFO - step: 83030 loss: 16.2428 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7746 +[titan] 2025-07-10 07:51:29,318 - root - INFO - lr: 4.9106e-05 gnorm: 1.05 [18:16:26< 3:44:05] +[titan] 2025-07-10 07:51:33,237 - root - INFO - step: 83035 loss: 16.1907 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 07:51:33,238 - root - INFO - lr: 4.9095e-05 gnorm: 0.98 [18:16:30< 3:44:01] +[titan] 2025-07-10 07:51:37,147 - root - INFO - step: 83040 loss: 16.3766 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8612 +[titan] 2025-07-10 07:51:37,148 - root - INFO - lr: 4.9084e-05 gnorm: 1.01 [18:16:34< 3:43:57] +[titan] 2025-07-10 07:51:41,081 - root - INFO - step: 83045 loss: 16.4967 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5272 global_avg_mtp_loss: 13.9695 +[titan] 2025-07-10 07:51:41,081 - root - INFO - lr: 4.9073e-05 gnorm: 1.02 [18:16:38< 3:43:53] +[titan] 2025-07-10 07:51:44,196 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:51:44,984 - root - INFO - step: 83050 loss: 16.2551 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 07:51:44,984 - root - INFO - lr: 4.9062e-05 gnorm: 1.05 [18:16:41< 3:43:49] +[titan] 2025-07-10 07:51:48,888 - root - INFO - step: 83055 loss: 16.2169 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7460 +[titan] 2025-07-10 07:51:48,888 - root - INFO - lr: 4.9051e-05 gnorm: 0.98 [18:16:45< 3:43:45] +[titan] 2025-07-10 07:51:52,819 - root - INFO - step: 83060 loss: 16.0398 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.5861 +[titan] 2025-07-10 07:51:52,819 - root - INFO - lr: 4.9040e-05 gnorm: 1.05 [18:16:49< 3:43:41] +[titan] 2025-07-10 07:51:56,726 - root - INFO - step: 83065 loss: 16.2917 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.7992 +[titan] 2025-07-10 07:51:56,726 - root - INFO - lr: 4.9029e-05 gnorm: 0.98 [18:16:53< 3:43:37] +[titan] 2025-07-10 07:52:00,666 - root - INFO - step: 83070 loss: 16.5293 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 13.9731 +[titan] 2025-07-10 07:52:00,666 - root - INFO - lr: 4.9018e-05 gnorm: 1.01 [18:16:57< 3:43:33] +[titan] 2025-07-10 07:52:04,632 - root - INFO - step: 83075 loss: 16.4518 memory: 44.58GiB(31.99%) tps: 82,624 tflops: 285.15 mfu: 28.83% global_avg_ntp_loss: 2.5265 global_avg_mtp_loss: 13.9253 +[titan] 2025-07-10 07:52:04,632 - root - INFO - lr: 4.9007e-05 gnorm: 0.99 [18:17:01< 3:43:29] +[titan] 2025-07-10 07:52:08,596 - root - INFO - step: 83080 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 82,680 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.5190 global_avg_mtp_loss: 13.9137 +[titan] 2025-07-10 07:52:08,596 - root - INFO - lr: 4.8996e-05 gnorm: 0.96 [18:17:05< 3:43:25] +[titan] 2025-07-10 07:52:12,523 - root - INFO - step: 83085 loss: 16.2559 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7705 +[titan] 2025-07-10 07:52:12,523 - root - INFO - lr: 4.8985e-05 gnorm: 1.00 [18:17:09< 3:43:22] +[titan] 2025-07-10 07:52:16,434 - root - INFO - step: 83090 loss: 16.2259 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7513 +[titan] 2025-07-10 07:52:16,434 - root - INFO - lr: 4.8974e-05 gnorm: 0.93 [18:17:13< 3:43:18] +[titan] 2025-07-10 07:52:20,350 - root - INFO - step: 83095 loss: 16.1853 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7199 +[titan] 2025-07-10 07:52:20,350 - root - INFO - lr: 4.8963e-05 gnorm: 0.95 [18:17:17< 3:43:14] +[titan] 2025-07-10 07:52:23,485 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:52:24,277 - root - INFO - step: 83100 loss: 16.2538 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7659 +[titan] 2025-07-10 07:52:24,277 - root - INFO - lr: 4.8952e-05 gnorm: 1.06 [18:17:21< 3:43:10] +[titan] 2025-07-10 07:52:28,189 - root - INFO - step: 83105 loss: 16.3838 memory: 44.58GiB(31.99%) tps: 83,773 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.8778 +[titan] 2025-07-10 07:52:28,189 - root - INFO - lr: 4.8941e-05 gnorm: 0.97 [18:17:25< 3:43:06] +[titan] 2025-07-10 07:52:32,094 - root - INFO - step: 83110 loss: 16.3145 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8153 +[titan] 2025-07-10 07:52:32,095 - root - INFO - lr: 4.8931e-05 gnorm: 0.98 [18:17:29< 3:43:02] +[titan] 2025-07-10 07:52:36,002 - root - INFO - step: 83115 loss: 15.9890 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.5439 +[titan] 2025-07-10 07:52:36,002 - root - INFO - lr: 4.8920e-05 gnorm: 0.96 [18:17:32< 3:42:58] +[titan] 2025-07-10 07:52:39,913 - root - INFO - step: 83120 loss: 16.1493 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.6782 +[titan] 2025-07-10 07:52:39,913 - root - INFO - lr: 4.8909e-05 gnorm: 0.95 [18:17:36< 3:42:54] +[titan] 2025-07-10 07:52:43,835 - root - INFO - step: 83125 loss: 16.4920 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.9673 +[titan] 2025-07-10 07:52:43,835 - root - INFO - lr: 4.8898e-05 gnorm: 1.00 [18:17:40< 3:42:50] +[titan] 2025-07-10 07:52:47,787 - root - INFO - step: 83130 loss: 16.1577 memory: 44.58GiB(31.99%) tps: 82,930 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.6846 +[titan] 2025-07-10 07:52:47,787 - root - INFO - lr: 4.8887e-05 gnorm: 1.00 [18:17:44< 3:42:46] +[titan] 2025-07-10 07:52:51,745 - root - INFO - step: 83135 loss: 16.1070 memory: 44.58GiB(31.99%) tps: 82,798 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6441 +[titan] 2025-07-10 07:52:51,745 - root - INFO - lr: 4.8876e-05 gnorm: 1.05 [18:17:48< 3:42:42] +[titan] 2025-07-10 07:52:55,643 - root - INFO - step: 83140 loss: 16.1605 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.6898 +[titan] 2025-07-10 07:52:55,643 - root - INFO - lr: 4.8865e-05 gnorm: 1.03 [18:17:52< 3:42:38] +[titan] 2025-07-10 07:52:59,561 - root - INFO - step: 83145 loss: 16.3722 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8711 +[titan] 2025-07-10 07:52:59,561 - root - INFO - lr: 4.8854e-05 gnorm: 0.98 [18:17:56< 3:42:34] +[titan] 2025-07-10 07:53:02,727 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:53:03,512 - root - INFO - step: 83150 loss: 16.3540 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8450 +[titan] 2025-07-10 07:53:03,512 - root - INFO - lr: 4.8843e-05 gnorm: 1.00 [18:18:00< 3:42:30] +[titan] 2025-07-10 07:53:07,418 - root - INFO - step: 83155 loss: 16.1846 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7104 +[titan] 2025-07-10 07:53:07,418 - root - INFO - lr: 4.8832e-05 gnorm: 1.02 [18:18:04< 3:42:26] +[titan] 2025-07-10 07:53:11,359 - root - INFO - step: 83160 loss: 16.2867 memory: 44.58GiB(31.99%) tps: 83,146 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.7876 +[titan] 2025-07-10 07:53:11,359 - root - INFO - lr: 4.8821e-05 gnorm: 1.02 [18:18:08< 3:42:22] +[titan] 2025-07-10 07:53:15,260 - root - INFO - step: 83165 loss: 16.2268 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.7590 +[titan] 2025-07-10 07:53:15,261 - root - INFO - lr: 4.8810e-05 gnorm: 0.97 [18:18:12< 3:42:18] +[titan] 2025-07-10 07:53:19,171 - root - INFO - step: 83170 loss: 16.2013 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.7303 +[titan] 2025-07-10 07:53:19,171 - root - INFO - lr: 4.8799e-05 gnorm: 0.96 [18:18:16< 3:42:14] +[titan] 2025-07-10 07:53:23,166 - root - INFO - step: 83175 loss: 16.1995 memory: 44.58GiB(31.99%) tps: 82,026 tflops: 283.09 mfu: 28.62% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7232 +[titan] 2025-07-10 07:53:23,166 - root - INFO - lr: 4.8789e-05 gnorm: 0.99 [18:18:20< 3:42:10] +[titan] 2025-07-10 07:53:27,093 - root - INFO - step: 83180 loss: 16.2837 memory: 44.58GiB(31.99%) tps: 83,455 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.7842 +[titan] 2025-07-10 07:53:27,093 - root - INFO - lr: 4.8778e-05 gnorm: 1.19 [18:18:24< 3:42:06] +[titan] 2025-07-10 07:53:31,022 - root - INFO - step: 83185 loss: 16.5299 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5280 global_avg_mtp_loss: 14.0019 +[titan] 2025-07-10 07:53:31,023 - root - INFO - lr: 4.8767e-05 gnorm: 1.03 [18:18:27< 3:42:02] +[titan] 2025-07-10 07:53:34,921 - root - INFO - step: 83190 loss: 16.2042 memory: 44.58GiB(31.99%) tps: 84,065 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.7362 +[titan] 2025-07-10 07:53:34,921 - root - INFO - lr: 4.8756e-05 gnorm: 1.02 [18:18:31< 3:41:58] +[titan] 2025-07-10 07:53:38,883 - root - INFO - step: 83195 loss: 16.1494 memory: 44.58GiB(31.99%) tps: 82,702 tflops: 285.42 mfu: 28.86% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6809 +[titan] 2025-07-10 07:53:38,883 - root - INFO - lr: 4.8745e-05 gnorm: 0.99 [18:18:35< 3:41:54] +[titan] 2025-07-10 07:53:42,004 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:53:42,799 - root - INFO - step: 83200 loss: 16.0800 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4506 global_avg_mtp_loss: 13.6294 +[titan] 2025-07-10 07:53:42,800 - root - INFO - lr: 4.8734e-05 gnorm: 1.09 [18:18:39< 3:41:50] +[titan] 2025-07-10 07:53:46,710 - root - INFO - step: 83205 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5149 global_avg_mtp_loss: 13.8953 +[titan] 2025-07-10 07:53:46,710 - root - INFO - lr: 4.8723e-05 gnorm: 1.08 [18:18:43< 3:41:46] +[titan] 2025-07-10 07:53:50,612 - root - INFO - step: 83210 loss: 16.0399 memory: 44.58GiB(31.99%) tps: 83,974 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.5847 +[titan] 2025-07-10 07:53:50,612 - root - INFO - lr: 4.8712e-05 gnorm: 0.98 [18:18:47< 3:41:42] +[titan] 2025-07-10 07:53:54,524 - root - INFO - step: 83215 loss: 16.2775 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7947 +[titan] 2025-07-10 07:53:54,525 - root - INFO - lr: 4.8701e-05 gnorm: 0.98 [18:18:51< 3:41:38] +[titan] 2025-07-10 07:53:58,453 - root - INFO - step: 83220 loss: 16.2458 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7712 +[titan] 2025-07-10 07:53:58,453 - root - INFO - lr: 4.8691e-05 gnorm: 0.94 [18:18:55< 3:41:34] +[titan] 2025-07-10 07:54:02,361 - root - INFO - step: 83225 loss: 15.9869 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.5422 +[titan] 2025-07-10 07:54:02,362 - root - INFO - lr: 4.8680e-05 gnorm: 1.00 [18:18:59< 3:41:30] +[titan] 2025-07-10 07:54:06,300 - root - INFO - step: 83230 loss: 16.1977 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 07:54:06,301 - root - INFO - lr: 4.8669e-05 gnorm: 0.96 [18:19:03< 3:41:26] +[titan] 2025-07-10 07:54:10,218 - root - INFO - step: 83235 loss: 16.2917 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.7931 +[titan] 2025-07-10 07:54:10,219 - root - INFO - lr: 4.8658e-05 gnorm: 0.99 [18:19:07< 3:41:22] +[titan] 2025-07-10 07:54:14,148 - root - INFO - step: 83240 loss: 16.2687 memory: 44.58GiB(31.99%) tps: 83,399 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7889 +[titan] 2025-07-10 07:54:14,148 - root - INFO - lr: 4.8647e-05 gnorm: 0.99 [18:19:11< 3:41:18] +[titan] 2025-07-10 07:54:18,052 - root - INFO - step: 83245 loss: 15.9944 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4261 global_avg_mtp_loss: 13.5683 +[titan] 2025-07-10 07:54:18,052 - root - INFO - lr: 4.8636e-05 gnorm: 0.97 [18:19:15< 3:41:14] +[titan] 2025-07-10 07:54:21,177 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:54:21,970 - root - INFO - step: 83250 loss: 16.3275 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.8415 +[titan] 2025-07-10 07:54:21,971 - root - INFO - lr: 4.8625e-05 gnorm: 0.96 [18:19:18< 3:41:11] +[titan] 2025-07-10 07:54:25,889 - root - INFO - step: 83255 loss: 16.6271 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.5400 global_avg_mtp_loss: 14.0871 +[titan] 2025-07-10 07:54:25,889 - root - INFO - lr: 4.8615e-05 gnorm: 0.99 [18:19:22< 3:41:07] +[titan] 2025-07-10 07:54:29,811 - root - INFO - step: 83260 loss: 16.3169 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.8243 +[titan] 2025-07-10 07:54:29,811 - root - INFO - lr: 4.8604e-05 gnorm: 0.96 [18:19:26< 3:41:03] +[titan] 2025-07-10 07:54:33,722 - root - INFO - step: 83265 loss: 16.1183 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.6544 +[titan] 2025-07-10 07:54:33,722 - root - INFO - lr: 4.8593e-05 gnorm: 0.97 [18:19:30< 3:40:59] +[titan] 2025-07-10 07:54:37,644 - root - INFO - step: 83270 loss: 16.2906 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.7934 +[titan] 2025-07-10 07:54:37,645 - root - INFO - lr: 4.8582e-05 gnorm: 0.94 [18:19:34< 3:40:55] +[titan] 2025-07-10 07:54:41,546 - root - INFO - step: 83275 loss: 16.1238 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.6737 +[titan] 2025-07-10 07:54:41,547 - root - INFO - lr: 4.8571e-05 gnorm: 0.93 [18:19:38< 3:40:51] +[titan] 2025-07-10 07:54:45,443 - root - INFO - step: 83280 loss: 16.2661 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7843 +[titan] 2025-07-10 07:54:45,443 - root - INFO - lr: 4.8560e-05 gnorm: 1.01 [18:19:42< 3:40:47] +[titan] 2025-07-10 07:54:49,339 - root - INFO - step: 83285 loss: 16.0142 memory: 44.58GiB(31.99%) tps: 84,113 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4431 global_avg_mtp_loss: 13.5711 +[titan] 2025-07-10 07:54:49,339 - root - INFO - lr: 4.8549e-05 gnorm: 0.96 [18:19:46< 3:40:43] +[titan] 2025-07-10 07:54:53,239 - root - INFO - step: 83290 loss: 15.9763 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4541 global_avg_mtp_loss: 13.5222 +[titan] 2025-07-10 07:54:53,240 - root - INFO - lr: 4.8539e-05 gnorm: 0.95 [18:19:50< 3:40:39] +[titan] 2025-07-10 07:54:57,143 - root - INFO - step: 83295 loss: 16.2199 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7408 +[titan] 2025-07-10 07:54:57,143 - root - INFO - lr: 4.8528e-05 gnorm: 1.02 [18:19:54< 3:40:35] +[titan] 2025-07-10 07:55:00,311 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:55:01,097 - root - INFO - step: 83300 loss: 16.2770 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7876 +[titan] 2025-07-10 07:55:01,097 - root - INFO - lr: 4.8517e-05 gnorm: 0.98 [18:19:58< 3:40:31] +[titan] 2025-07-10 07:55:05,012 - root - INFO - step: 83305 loss: 16.3048 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8059 +[titan] 2025-07-10 07:55:05,012 - root - INFO - lr: 4.8506e-05 gnorm: 0.99 [18:20:01< 3:40:27] +[titan] 2025-07-10 07:55:08,936 - root - INFO - step: 83310 loss: 16.3550 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.8420 +[titan] 2025-07-10 07:55:08,936 - root - INFO - lr: 4.8495e-05 gnorm: 1.01 [18:20:05< 3:40:23] +[titan] 2025-07-10 07:55:12,864 - root - INFO - step: 83315 loss: 16.1363 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.6768 +[titan] 2025-07-10 07:55:12,864 - root - INFO - lr: 4.8484e-05 gnorm: 0.97 [18:20:09< 3:40:19] +[titan] 2025-07-10 07:55:16,828 - root - INFO - step: 83320 loss: 16.3107 memory: 44.58GiB(31.99%) tps: 82,670 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.5018 global_avg_mtp_loss: 13.8090 +[titan] 2025-07-10 07:55:16,828 - root - INFO - lr: 4.8474e-05 gnorm: 0.97 [18:20:13< 3:40:15] +[titan] 2025-07-10 07:55:20,761 - root - INFO - step: 83325 loss: 16.3675 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.8565 +[titan] 2025-07-10 07:55:20,761 - root - INFO - lr: 4.8463e-05 gnorm: 1.04 [18:20:17< 3:40:11] +[titan] 2025-07-10 07:55:24,651 - root - INFO - step: 83330 loss: 16.4787 memory: 44.58GiB(31.99%) tps: 84,242 tflops: 290.73 mfu: 29.40% global_avg_ntp_loss: 2.5204 global_avg_mtp_loss: 13.9583 +[titan] 2025-07-10 07:55:24,651 - root - INFO - lr: 4.8452e-05 gnorm: 0.95 [18:20:21< 3:40:07] +[titan] 2025-07-10 07:55:28,568 - root - INFO - step: 83335 loss: 16.1671 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.7000 +[titan] 2025-07-10 07:55:28,568 - root - INFO - lr: 4.8441e-05 gnorm: 0.96 [18:20:25< 3:40:03] +[titan] 2025-07-10 07:55:32,519 - root - INFO - step: 83340 loss: 16.1482 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6830 +[titan] 2025-07-10 07:55:32,519 - root - INFO - lr: 4.8430e-05 gnorm: 0.96 [18:20:29< 3:39:59] +[titan] 2025-07-10 07:55:36,453 - root - INFO - step: 83345 loss: 16.4087 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.5092 global_avg_mtp_loss: 13.8995 +[titan] 2025-07-10 07:55:36,454 - root - INFO - lr: 4.8420e-05 gnorm: 0.99 [18:20:33< 3:39:55] +[titan] 2025-07-10 07:55:39,581 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:55:40,367 - root - INFO - step: 83350 loss: 16.1052 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.6332 +[titan] 2025-07-10 07:55:40,367 - root - INFO - lr: 4.8409e-05 gnorm: 0.99 [18:20:37< 3:39:51] +[titan] 2025-07-10 07:55:44,296 - root - INFO - step: 83355 loss: 16.0861 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4505 global_avg_mtp_loss: 13.6356 +[titan] 2025-07-10 07:55:44,297 - root - INFO - lr: 4.8398e-05 gnorm: 0.97 [18:20:41< 3:39:47] +[titan] 2025-07-10 07:55:48,211 - root - INFO - step: 83360 loss: 16.2273 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 07:55:48,211 - root - INFO - lr: 4.8387e-05 gnorm: 1.02 [18:20:45< 3:39:43] +[titan] 2025-07-10 07:55:52,144 - root - INFO - step: 83365 loss: 16.1523 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.6790 +[titan] 2025-07-10 07:55:52,144 - root - INFO - lr: 4.8376e-05 gnorm: 0.96 [18:20:49< 3:39:39] +[titan] 2025-07-10 07:55:56,067 - root - INFO - step: 83370 loss: 16.0901 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.6334 +[titan] 2025-07-10 07:55:56,068 - root - INFO - lr: 4.8366e-05 gnorm: 1.00 [18:20:53< 3:39:35] +[titan] 2025-07-10 07:56:00,034 - root - INFO - step: 83375 loss: 16.0308 memory: 44.58GiB(31.99%) tps: 82,630 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.4346 global_avg_mtp_loss: 13.5962 +[titan] 2025-07-10 07:56:00,034 - root - INFO - lr: 4.8355e-05 gnorm: 1.01 [18:20:56< 3:39:31] +[titan] 2025-07-10 07:56:03,947 - root - INFO - step: 83380 loss: 15.9877 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.5427 +[titan] 2025-07-10 07:56:03,947 - root - INFO - lr: 4.8344e-05 gnorm: 0.98 [18:21:00< 3:39:27] +[titan] 2025-07-10 07:56:07,866 - root - INFO - step: 83385 loss: 16.3785 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8762 +[titan] 2025-07-10 07:56:07,866 - root - INFO - lr: 4.8333e-05 gnorm: 0.98 [18:21:04< 3:39:23] +[titan] 2025-07-10 07:56:11,817 - root - INFO - step: 83390 loss: 16.0228 memory: 44.58GiB(31.99%) tps: 82,928 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.5778 +[titan] 2025-07-10 07:56:11,818 - root - INFO - lr: 4.8323e-05 gnorm: 0.99 [18:21:08< 3:39:19] +[titan] 2025-07-10 07:56:15,727 - root - INFO - step: 83395 loss: 16.7316 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5896 global_avg_mtp_loss: 14.1420 +[titan] 2025-07-10 07:56:15,728 - root - INFO - lr: 4.8312e-05 gnorm: 1.02 [18:21:12< 3:39:15] +[titan] 2025-07-10 07:56:18,866 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:56:19,654 - root - INFO - step: 83400 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7931 +[titan] 2025-07-10 07:56:19,654 - root - INFO - lr: 4.8301e-05 gnorm: 1.02 [18:21:16< 3:39:11] +[titan] 2025-07-10 07:56:23,569 - root - INFO - step: 83405 loss: 16.1105 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 07:56:23,570 - root - INFO - lr: 4.8290e-05 gnorm: 0.99 [18:21:20< 3:39:07] +[titan] 2025-07-10 07:56:27,472 - root - INFO - step: 83410 loss: 16.2881 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.7840 +[titan] 2025-07-10 07:56:27,472 - root - INFO - lr: 4.8279e-05 gnorm: 0.99 [18:21:24< 3:39:03] +[titan] 2025-07-10 07:56:31,384 - root - INFO - step: 83415 loss: 16.1059 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4539 global_avg_mtp_loss: 13.6520 +[titan] 2025-07-10 07:56:31,384 - root - INFO - lr: 4.8269e-05 gnorm: 0.97 [18:21:28< 3:39:00] +[titan] 2025-07-10 07:56:35,331 - root - INFO - step: 83420 loss: 16.0965 memory: 44.58GiB(31.99%) tps: 83,039 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.4555 global_avg_mtp_loss: 13.6410 +[titan] 2025-07-10 07:56:35,331 - root - INFO - lr: 4.8258e-05 gnorm: 0.98 [18:21:32< 3:38:56] +[titan] 2025-07-10 07:56:39,251 - root - INFO - step: 83425 loss: 16.2723 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.7774 +[titan] 2025-07-10 07:56:39,251 - root - INFO - lr: 4.8247e-05 gnorm: 0.99 [18:21:36< 3:38:52] +[titan] 2025-07-10 07:56:43,180 - root - INFO - step: 83430 loss: 16.2992 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4905 global_avg_mtp_loss: 13.8087 +[titan] 2025-07-10 07:56:43,180 - root - INFO - lr: 4.8236e-05 gnorm: 0.98 [18:21:40< 3:38:48] +[titan] 2025-07-10 07:56:47,119 - root - INFO - step: 83435 loss: 16.1486 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4683 global_avg_mtp_loss: 13.6803 +[titan] 2025-07-10 07:56:47,119 - root - INFO - lr: 4.8226e-05 gnorm: 0.99 [18:21:44< 3:38:44] +[titan] 2025-07-10 07:56:51,034 - root - INFO - step: 83440 loss: 16.2702 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7889 +[titan] 2025-07-10 07:56:51,034 - root - INFO - lr: 4.8215e-05 gnorm: 0.95 [18:21:47< 3:38:40] +[titan] 2025-07-10 07:56:54,963 - root - INFO - step: 83445 loss: 16.3616 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8591 +[titan] 2025-07-10 07:56:54,964 - root - INFO - lr: 4.8204e-05 gnorm: 0.98 [18:21:51< 3:38:36] +[titan] 2025-07-10 07:56:58,110 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:56:58,902 - root - INFO - step: 83450 loss: 15.9136 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4064 global_avg_mtp_loss: 13.5073 +[titan] 2025-07-10 07:56:58,902 - root - INFO - lr: 4.8193e-05 gnorm: 1.03 [18:21:55< 3:38:32] +[titan] 2025-07-10 07:57:02,910 - root - INFO - step: 83455 loss: 16.3945 memory: 44.58GiB(31.99%) tps: 81,764 tflops: 282.18 mfu: 28.53% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8876 +[titan] 2025-07-10 07:57:02,911 - root - INFO - lr: 4.8183e-05 gnorm: 0.93 [18:21:59< 3:38:28] +[titan] 2025-07-10 07:57:03,856 - root - INFO - Dumping profiler traces at step 83456 +[titan] 2025-07-10 07:57:03,898 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 07:57:07,054 - root - INFO - step: 83460 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 79,092 tflops: 272.96 mfu: 27.60% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 07:57:07,054 - root - INFO - lr: 4.8172e-05 gnorm: 0.96 [18:22:03< 3:38:24] +[titan] 2025-07-10 07:57:10,991 - root - INFO - step: 83465 loss: 16.2821 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7930 +[titan] 2025-07-10 07:57:10,991 - root - INFO - lr: 4.8161e-05 gnorm: 1.03 [18:22:07< 3:38:20] +[titan] 2025-07-10 07:57:14,913 - root - INFO - step: 83470 loss: 16.2739 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7930 +[titan] 2025-07-10 07:57:14,913 - root - INFO - lr: 4.8151e-05 gnorm: 0.95 [18:22:11< 3:38:16] +[titan] 2025-07-10 07:57:18,818 - root - INFO - step: 83475 loss: 16.1701 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4607 global_avg_mtp_loss: 13.7094 +[titan] 2025-07-10 07:57:18,818 - root - INFO - lr: 4.8140e-05 gnorm: 0.95 [18:22:15< 3:38:12] +[titan] 2025-07-10 07:57:22,708 - root - INFO - step: 83480 loss: 16.1900 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.69 mfu: 29.39% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.7333 +[titan] 2025-07-10 07:57:22,709 - root - INFO - lr: 4.8129e-05 gnorm: 0.97 [18:22:19< 3:38:08] +[titan] 2025-07-10 07:57:26,632 - root - INFO - step: 83485 loss: 16.3894 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8749 +[titan] 2025-07-10 07:57:26,632 - root - INFO - lr: 4.8118e-05 gnorm: 0.96 [18:22:23< 3:38:04] +[titan] 2025-07-10 07:57:30,547 - root - INFO - step: 83490 loss: 16.0150 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4486 global_avg_mtp_loss: 13.5664 +[titan] 2025-07-10 07:57:30,547 - root - INFO - lr: 4.8108e-05 gnorm: 0.98 [18:22:27< 3:38:00] +[titan] 2025-07-10 07:57:34,482 - root - INFO - step: 83495 loss: 15.9838 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4434 global_avg_mtp_loss: 13.5404 +[titan] 2025-07-10 07:57:34,482 - root - INFO - lr: 4.8097e-05 gnorm: 1.00 [18:22:31< 3:37:56] +[titan] 2025-07-10 07:57:37,593 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:57:38,399 - root - INFO - step: 83500 loss: 16.3700 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8697 +[titan] 2025-07-10 07:57:38,399 - root - INFO - lr: 4.8086e-05 gnorm: 0.99 [18:22:35< 3:37:52] +[titan] 2025-07-10 07:57:42,307 - root - INFO - step: 83505 loss: 16.3116 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 07:57:42,308 - root - INFO - lr: 4.8076e-05 gnorm: 0.98 [18:22:39< 3:37:48] +[titan] 2025-07-10 07:57:46,223 - root - INFO - step: 83510 loss: 15.9851 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4370 global_avg_mtp_loss: 13.5481 +[titan] 2025-07-10 07:57:46,223 - root - INFO - lr: 4.8065e-05 gnorm: 0.93 [18:22:43< 3:37:44] +[titan] 2025-07-10 07:57:50,129 - root - INFO - step: 83515 loss: 16.4593 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.5350 global_avg_mtp_loss: 13.9243 +[titan] 2025-07-10 07:57:50,129 - root - INFO - lr: 4.8054e-05 gnorm: 0.97 [18:22:47< 3:37:40] +[titan] 2025-07-10 07:57:54,033 - root - INFO - step: 83520 loss: 16.1697 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.7063 +[titan] 2025-07-10 07:57:54,034 - root - INFO - lr: 4.8043e-05 gnorm: 0.99 [18:22:50< 3:37:36] +[titan] 2025-07-10 07:57:57,973 - root - INFO - step: 83525 loss: 16.4106 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.8905 +[titan] 2025-07-10 07:57:57,973 - root - INFO - lr: 4.8033e-05 gnorm: 1.03 [18:22:54< 3:37:32] +[titan] 2025-07-10 07:58:01,871 - root - INFO - step: 83530 loss: 16.3565 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8512 +[titan] 2025-07-10 07:58:01,871 - root - INFO - lr: 4.8022e-05 gnorm: 0.94 [18:22:58< 3:37:28] +[titan] 2025-07-10 07:58:05,782 - root - INFO - step: 83535 loss: 16.1656 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.6935 +[titan] 2025-07-10 07:58:05,782 - root - INFO - lr: 4.8011e-05 gnorm: 1.00 [18:23:02< 3:37:24] +[titan] 2025-07-10 07:58:09,699 - root - INFO - step: 83540 loss: 16.3177 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8264 +[titan] 2025-07-10 07:58:09,700 - root - INFO - lr: 4.8001e-05 gnorm: 0.96 [18:23:06< 3:37:20] +[titan] 2025-07-10 07:58:13,648 - root - INFO - step: 83545 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 82,995 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7683 +[titan] 2025-07-10 07:58:13,648 - root - INFO - lr: 4.7990e-05 gnorm: 1.05 [18:23:10< 3:37:16] +[titan] 2025-07-10 07:58:16,793 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:58:17,586 - root - INFO - step: 83550 loss: 16.3055 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8105 +[titan] 2025-07-10 07:58:17,586 - root - INFO - lr: 4.7979e-05 gnorm: 0.99 [18:23:14< 3:37:12] +[titan] 2025-07-10 07:58:21,486 - root - INFO - step: 83555 loss: 16.4621 memory: 44.58GiB(31.99%) tps: 84,025 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.9318 +[titan] 2025-07-10 07:58:21,487 - root - INFO - lr: 4.7969e-05 gnorm: 0.99 [18:23:18< 3:37:08] +[titan] 2025-07-10 07:58:25,400 - root - INFO - step: 83560 loss: 16.0133 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5731 +[titan] 2025-07-10 07:58:25,400 - root - INFO - lr: 4.7958e-05 gnorm: 1.06 [18:23:22< 3:37:04] +[titan] 2025-07-10 07:58:29,320 - root - INFO - step: 83565 loss: 15.7994 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4388 global_avg_mtp_loss: 13.3606 +[titan] 2025-07-10 07:58:29,321 - root - INFO - lr: 4.7947e-05 gnorm: 1.13 [18:23:26< 3:37:00] +[titan] 2025-07-10 07:58:33,272 - root - INFO - step: 83570 loss: 16.1074 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.4521 global_avg_mtp_loss: 13.6553 +[titan] 2025-07-10 07:58:33,272 - root - INFO - lr: 4.7937e-05 gnorm: 1.00 [18:23:30< 3:36:57] +[titan] 2025-07-10 07:58:37,181 - root - INFO - step: 83575 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 83,830 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8677 +[titan] 2025-07-10 07:58:37,182 - root - INFO - lr: 4.7926e-05 gnorm: 0.96 [18:23:34< 3:36:53] +[titan] 2025-07-10 07:58:41,097 - root - INFO - step: 83580 loss: 16.2872 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.8007 +[titan] 2025-07-10 07:58:41,097 - root - INFO - lr: 4.7915e-05 gnorm: 1.00 [18:23:38< 3:36:49] +[titan] 2025-07-10 07:58:45,003 - root - INFO - step: 83585 loss: 15.9066 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4198 global_avg_mtp_loss: 13.4868 +[titan] 2025-07-10 07:58:45,004 - root - INFO - lr: 4.7905e-05 gnorm: 1.00 [18:23:41< 3:36:45] +[titan] 2025-07-10 07:58:48,952 - root - INFO - step: 83590 loss: 16.2396 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.7668 +[titan] 2025-07-10 07:58:48,953 - root - INFO - lr: 4.7894e-05 gnorm: 1.02 [18:23:45< 3:36:41] +[titan] 2025-07-10 07:58:52,879 - root - INFO - step: 83595 loss: 16.2852 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 07:58:52,879 - root - INFO - lr: 4.7883e-05 gnorm: 1.01 [18:23:49< 3:36:37] +[titan] 2025-07-10 07:58:56,001 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:58:56,805 - root - INFO - step: 83600 loss: 16.1699 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.6955 +[titan] 2025-07-10 07:58:56,806 - root - INFO - lr: 4.7873e-05 gnorm: 0.99 [18:23:53< 3:36:33] +[titan] 2025-07-10 07:59:00,760 - root - INFO - step: 83605 loss: 16.3986 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8901 +[titan] 2025-07-10 07:59:00,760 - root - INFO - lr: 4.7862e-05 gnorm: 1.08 [18:23:57< 3:36:29] +[titan] 2025-07-10 07:59:04,691 - root - INFO - step: 83610 loss: 16.0784 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4529 global_avg_mtp_loss: 13.6255 +[titan] 2025-07-10 07:59:04,691 - root - INFO - lr: 4.7851e-05 gnorm: 0.98 [18:24:01< 3:36:25] +[titan] 2025-07-10 07:59:08,602 - root - INFO - step: 83615 loss: 16.4191 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5121 global_avg_mtp_loss: 13.9070 +[titan] 2025-07-10 07:59:08,602 - root - INFO - lr: 4.7841e-05 gnorm: 0.97 [18:24:05< 3:36:21] +[titan] 2025-07-10 07:59:12,518 - root - INFO - step: 83620 loss: 16.1736 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7008 +[titan] 2025-07-10 07:59:12,518 - root - INFO - lr: 4.7830e-05 gnorm: 1.00 [18:24:09< 3:36:17] +[titan] 2025-07-10 07:59:16,450 - root - INFO - step: 83625 loss: 15.9863 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4335 global_avg_mtp_loss: 13.5528 +[titan] 2025-07-10 07:59:16,451 - root - INFO - lr: 4.7819e-05 gnorm: 0.96 [18:24:13< 3:36:13] +[titan] 2025-07-10 07:59:20,394 - root - INFO - step: 83630 loss: 16.1578 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.6901 +[titan] 2025-07-10 07:59:20,394 - root - INFO - lr: 4.7809e-05 gnorm: 0.99 [18:24:17< 3:36:09] +[titan] 2025-07-10 07:59:24,346 - root - INFO - step: 83635 loss: 16.2206 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7389 +[titan] 2025-07-10 07:59:24,346 - root - INFO - lr: 4.7798e-05 gnorm: 1.00 [18:24:21< 3:36:05] +[titan] 2025-07-10 07:59:28,267 - root - INFO - step: 83640 loss: 16.2770 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7949 +[titan] 2025-07-10 07:59:28,268 - root - INFO - lr: 4.7787e-05 gnorm: 1.03 [18:24:25< 3:36:01] +[titan] 2025-07-10 07:59:32,193 - root - INFO - step: 83645 loss: 16.4029 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.9044 +[titan] 2025-07-10 07:59:32,193 - root - INFO - lr: 4.7777e-05 gnorm: 1.00 [18:24:29< 3:35:57] +[titan] 2025-07-10 07:59:35,324 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 07:59:36,111 - root - INFO - step: 83650 loss: 16.1628 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.6927 +[titan] 2025-07-10 07:59:36,111 - root - INFO - lr: 4.7766e-05 gnorm: 1.02 [18:24:33< 3:35:53] +[titan] 2025-07-10 07:59:40,025 - root - INFO - step: 83655 loss: 16.2528 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7738 +[titan] 2025-07-10 07:59:40,026 - root - INFO - lr: 4.7756e-05 gnorm: 0.99 [18:24:36< 3:35:49] +[titan] 2025-07-10 07:59:43,960 - root - INFO - step: 83660 loss: 16.3278 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8238 +[titan] 2025-07-10 07:59:43,960 - root - INFO - lr: 4.7745e-05 gnorm: 0.99 [18:24:40< 3:35:45] +[titan] 2025-07-10 07:59:47,869 - root - INFO - step: 83665 loss: 16.3663 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 07:59:47,870 - root - INFO - lr: 4.7734e-05 gnorm: 1.01 [18:24:44< 3:35:41] +[titan] 2025-07-10 07:59:51,800 - root - INFO - step: 83670 loss: 16.3246 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4944 global_avg_mtp_loss: 13.8303 +[titan] 2025-07-10 07:59:51,801 - root - INFO - lr: 4.7724e-05 gnorm: 0.97 [18:24:48< 3:35:37] +[titan] 2025-07-10 07:59:55,711 - root - INFO - step: 83675 loss: 16.1696 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4654 global_avg_mtp_loss: 13.7042 +[titan] 2025-07-10 07:59:55,712 - root - INFO - lr: 4.7713e-05 gnorm: 1.01 [18:24:52< 3:35:33] +[titan] 2025-07-10 07:59:59,627 - root - INFO - step: 83680 loss: 16.2788 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7969 +[titan] 2025-07-10 07:59:59,627 - root - INFO - lr: 4.7703e-05 gnorm: 0.98 [18:24:56< 3:35:29] +[titan] 2025-07-10 08:00:03,543 - root - INFO - step: 83685 loss: 16.2961 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 08:00:03,543 - root - INFO - lr: 4.7692e-05 gnorm: 1.00 [18:25:00< 3:35:25] +[titan] 2025-07-10 08:00:07,433 - root - INFO - step: 83690 loss: 16.2454 memory: 44.58GiB(31.99%) tps: 84,237 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.7511 +[titan] 2025-07-10 08:00:07,433 - root - INFO - lr: 4.7681e-05 gnorm: 0.98 [18:25:04< 3:35:21] +[titan] 2025-07-10 08:00:11,376 - root - INFO - step: 83695 loss: 16.3350 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.5025 global_avg_mtp_loss: 13.8325 +[titan] 2025-07-10 08:00:11,376 - root - INFO - lr: 4.7671e-05 gnorm: 0.97 [18:25:08< 3:35:17] +[titan] 2025-07-10 08:00:14,540 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:00:15,328 - root - INFO - step: 83700 loss: 16.5272 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 14.0013 +[titan] 2025-07-10 08:00:15,328 - root - INFO - lr: 4.7660e-05 gnorm: 0.99 [18:25:12< 3:35:13] +[titan] 2025-07-10 08:00:19,281 - root - INFO - step: 83705 loss: 16.5400 memory: 44.58GiB(31.99%) tps: 82,890 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.5332 global_avg_mtp_loss: 14.0068 +[titan] 2025-07-10 08:00:19,281 - root - INFO - lr: 4.7650e-05 gnorm: 0.99 [18:25:16< 3:35:09] +[titan] 2025-07-10 08:00:23,191 - root - INFO - step: 83710 loss: 16.3147 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8213 +[titan] 2025-07-10 08:00:23,191 - root - INFO - lr: 4.7639e-05 gnorm: 0.98 [18:25:20< 3:35:05] +[titan] 2025-07-10 08:00:27,112 - root - INFO - step: 83715 loss: 16.2477 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 08:00:27,113 - root - INFO - lr: 4.7628e-05 gnorm: 0.97 [18:25:24< 3:35:01] +[titan] 2025-07-10 08:00:31,033 - root - INFO - step: 83720 loss: 16.1988 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7302 +[titan] 2025-07-10 08:00:31,034 - root - INFO - lr: 4.7618e-05 gnorm: 0.95 [18:25:27< 3:34:57] +[titan] 2025-07-10 08:00:34,932 - root - INFO - step: 83725 loss: 15.8346 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4052 global_avg_mtp_loss: 13.4294 +[titan] 2025-07-10 08:00:34,932 - root - INFO - lr: 4.7607e-05 gnorm: 1.06 [18:25:31< 3:34:54] +[titan] 2025-07-10 08:00:38,860 - root - INFO - step: 83730 loss: 16.3240 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 08:00:38,860 - root - INFO - lr: 4.7597e-05 gnorm: 0.94 [18:25:35< 3:34:50] +[titan] 2025-07-10 08:00:42,823 - root - INFO - step: 83735 loss: 16.2003 memory: 44.58GiB(31.99%) tps: 82,692 tflops: 285.39 mfu: 28.86% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-10 08:00:42,823 - root - INFO - lr: 4.7586e-05 gnorm: 1.09 [18:25:39< 3:34:46] +[titan] 2025-07-10 08:00:46,756 - root - INFO - step: 83740 loss: 16.0080 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4340 global_avg_mtp_loss: 13.5740 +[titan] 2025-07-10 08:00:46,757 - root - INFO - lr: 4.7576e-05 gnorm: 0.98 [18:25:43< 3:34:42] +[titan] 2025-07-10 08:00:50,703 - root - INFO - step: 83745 loss: 16.2505 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7634 +[titan] 2025-07-10 08:00:50,703 - root - INFO - lr: 4.7565e-05 gnorm: 0.97 [18:25:47< 3:34:38] +[titan] 2025-07-10 08:00:53,842 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:00:54,631 - root - INFO - step: 83750 loss: 15.7958 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4013 global_avg_mtp_loss: 13.3945 +[titan] 2025-07-10 08:00:54,632 - root - INFO - lr: 4.7554e-05 gnorm: 1.06 [18:25:51< 3:34:34] +[titan] 2025-07-10 08:00:58,551 - root - INFO - step: 83755 loss: 16.0530 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.6129 +[titan] 2025-07-10 08:00:58,551 - root - INFO - lr: 4.7544e-05 gnorm: 1.00 [18:25:55< 3:34:30] +[titan] 2025-07-10 08:01:02,491 - root - INFO - step: 83760 loss: 16.0187 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.4560 global_avg_mtp_loss: 13.5627 +[titan] 2025-07-10 08:01:02,491 - root - INFO - lr: 4.7533e-05 gnorm: 1.03 [18:25:59< 3:34:26] +[titan] 2025-07-10 08:01:06,406 - root - INFO - step: 83765 loss: 16.1993 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7214 +[titan] 2025-07-10 08:01:06,406 - root - INFO - lr: 4.7523e-05 gnorm: 0.94 [18:26:03< 3:34:22] +[titan] 2025-07-10 08:01:10,314 - root - INFO - step: 83770 loss: 15.9259 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4220 global_avg_mtp_loss: 13.5039 +[titan] 2025-07-10 08:01:10,314 - root - INFO - lr: 4.7512e-05 gnorm: 0.99 [18:26:07< 3:34:18] +[titan] 2025-07-10 08:01:14,262 - root - INFO - step: 83775 loss: 16.3549 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8408 +[titan] 2025-07-10 08:01:14,263 - root - INFO - lr: 4.7502e-05 gnorm: 0.96 [18:26:11< 3:34:14] +[titan] 2025-07-10 08:01:18,219 - root - INFO - step: 83780 loss: 16.3257 memory: 44.58GiB(31.99%) tps: 82,819 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5505 global_avg_mtp_loss: 13.7752 +[titan] 2025-07-10 08:01:18,219 - root - INFO - lr: 4.7491e-05 gnorm: 1.13 [18:26:15< 3:34:10] +[titan] 2025-07-10 08:01:22,127 - root - INFO - step: 83785 loss: 16.2932 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.7940 +[titan] 2025-07-10 08:01:22,127 - root - INFO - lr: 4.7481e-05 gnorm: 1.02 [18:26:19< 3:34:06] +[titan] 2025-07-10 08:01:26,024 - root - INFO - step: 83790 loss: 16.0703 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6134 +[titan] 2025-07-10 08:01:26,024 - root - INFO - lr: 4.7470e-05 gnorm: 1.00 [18:26:22< 3:34:02] +[titan] 2025-07-10 08:01:29,923 - root - INFO - step: 83795 loss: 16.2543 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7707 +[titan] 2025-07-10 08:01:29,923 - root - INFO - lr: 4.7459e-05 gnorm: 1.01 [18:26:26< 3:33:58] +[titan] 2025-07-10 08:01:33,049 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:01:33,840 - root - INFO - step: 83800 loss: 15.6123 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.3780 global_avg_mtp_loss: 13.2343 +[titan] 2025-07-10 08:01:33,841 - root - INFO - lr: 4.7449e-05 gnorm: 1.02 [18:26:30< 3:33:54] +[titan] 2025-07-10 08:01:37,774 - root - INFO - step: 83805 loss: 15.9752 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.5306 +[titan] 2025-07-10 08:01:37,774 - root - INFO - lr: 4.7438e-05 gnorm: 1.00 [18:26:34< 3:33:50] +[titan] 2025-07-10 08:01:41,698 - root - INFO - step: 83810 loss: 16.2566 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7702 +[titan] 2025-07-10 08:01:41,698 - root - INFO - lr: 4.7428e-05 gnorm: 0.97 [18:26:38< 3:33:46] +[titan] 2025-07-10 08:01:45,733 - root - INFO - step: 83815 loss: 16.0453 memory: 44.58GiB(31.99%) tps: 81,218 tflops: 280.30 mfu: 28.34% global_avg_ntp_loss: 2.4352 global_avg_mtp_loss: 13.6101 +[titan] 2025-07-10 08:01:45,733 - root - INFO - lr: 4.7417e-05 gnorm: 0.98 [18:26:42< 3:33:42] +[titan] 2025-07-10 08:01:49,644 - root - INFO - step: 83820 loss: 16.1981 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7174 +[titan] 2025-07-10 08:01:49,645 - root - INFO - lr: 4.7407e-05 gnorm: 0.96 [18:26:46< 3:33:38] +[titan] 2025-07-10 08:01:53,570 - root - INFO - step: 83825 loss: 16.2403 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7674 +[titan] 2025-07-10 08:01:53,571 - root - INFO - lr: 4.7396e-05 gnorm: 0.94 [18:26:50< 3:33:34] +[titan] 2025-07-10 08:01:57,493 - root - INFO - step: 83830 loss: 16.3126 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.8312 +[titan] 2025-07-10 08:01:57,493 - root - INFO - lr: 4.7386e-05 gnorm: 0.95 [18:26:54< 3:33:30] +[titan] 2025-07-10 08:02:01,399 - root - INFO - step: 83835 loss: 16.2890 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.7991 +[titan] 2025-07-10 08:02:01,399 - root - INFO - lr: 4.7375e-05 gnorm: 1.00 [18:26:58< 3:33:26] +[titan] 2025-07-10 08:02:05,310 - root - INFO - step: 83840 loss: 16.3839 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 08:02:05,310 - root - INFO - lr: 4.7365e-05 gnorm: 0.96 [18:27:02< 3:33:22] +[titan] 2025-07-10 08:02:09,221 - root - INFO - step: 83845 loss: 16.1183 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4531 global_avg_mtp_loss: 13.6652 +[titan] 2025-07-10 08:02:09,221 - root - INFO - lr: 4.7354e-05 gnorm: 0.98 [18:27:06< 3:33:18] +[titan] 2025-07-10 08:02:12,341 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:02:13,128 - root - INFO - step: 83850 loss: 16.0864 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 08:02:13,128 - root - INFO - lr: 4.7344e-05 gnorm: 0.97 [18:27:10< 3:33:14] +[titan] 2025-07-10 08:02:17,026 - root - INFO - step: 83855 loss: 16.0983 memory: 44.58GiB(31.99%) tps: 84,075 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6436 +[titan] 2025-07-10 08:02:17,026 - root - INFO - lr: 4.7333e-05 gnorm: 1.01 [18:27:13< 3:33:10] +[titan] 2025-07-10 08:02:20,948 - root - INFO - step: 83860 loss: 16.0869 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6255 +[titan] 2025-07-10 08:02:20,949 - root - INFO - lr: 4.7323e-05 gnorm: 0.99 [18:27:17< 3:33:06] +[titan] 2025-07-10 08:02:24,871 - root - INFO - step: 83865 loss: 16.1017 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6452 +[titan] 2025-07-10 08:02:24,871 - root - INFO - lr: 4.7312e-05 gnorm: 0.96 [18:27:21< 3:33:02] +[titan] 2025-07-10 08:02:28,782 - root - INFO - step: 83870 loss: 16.1879 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.7261 +[titan] 2025-07-10 08:02:28,782 - root - INFO - lr: 4.7302e-05 gnorm: 0.96 [18:27:25< 3:32:58] +[titan] 2025-07-10 08:02:32,696 - root - INFO - step: 83875 loss: 16.1975 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.7225 +[titan] 2025-07-10 08:02:32,696 - root - INFO - lr: 4.7291e-05 gnorm: 0.95 [18:27:29< 3:32:54] +[titan] 2025-07-10 08:02:36,649 - root - INFO - step: 83880 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 82,901 tflops: 286.11 mfu: 28.93% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8128 +[titan] 2025-07-10 08:02:36,649 - root - INFO - lr: 4.7281e-05 gnorm: 0.99 [18:27:33< 3:32:50] +[titan] 2025-07-10 08:02:40,545 - root - INFO - step: 83885 loss: 16.0121 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.5685 +[titan] 2025-07-10 08:02:40,545 - root - INFO - lr: 4.7270e-05 gnorm: 1.02 [18:27:37< 3:32:47] +[titan] 2025-07-10 08:02:44,457 - root - INFO - step: 83890 loss: 16.3344 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 08:02:44,458 - root - INFO - lr: 4.7260e-05 gnorm: 1.03 [18:27:41< 3:32:43] +[titan] 2025-07-10 08:02:48,399 - root - INFO - step: 83895 loss: 16.2027 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.7301 +[titan] 2025-07-10 08:02:48,399 - root - INFO - lr: 4.7249e-05 gnorm: 1.02 [18:27:45< 3:32:39] +[titan] 2025-07-10 08:02:51,512 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:02:52,311 - root - INFO - step: 83900 loss: 16.3954 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8906 +[titan] 2025-07-10 08:02:52,312 - root - INFO - lr: 4.7239e-05 gnorm: 0.99 [18:27:49< 3:32:35] +[titan] 2025-07-10 08:02:56,242 - root - INFO - step: 83905 loss: 16.0827 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4470 global_avg_mtp_loss: 13.6357 +[titan] 2025-07-10 08:02:56,242 - root - INFO - lr: 4.7228e-05 gnorm: 0.98 [18:27:53< 3:32:31] +[titan] 2025-07-10 08:03:00,168 - root - INFO - step: 83910 loss: 15.9377 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4183 global_avg_mtp_loss: 13.5194 +[titan] 2025-07-10 08:03:00,168 - root - INFO - lr: 4.7218e-05 gnorm: 1.01 [18:27:57< 3:32:27] +[titan] 2025-07-10 08:03:04,077 - root - INFO - step: 83915 loss: 16.2506 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7758 +[titan] 2025-07-10 08:03:04,077 - root - INFO - lr: 4.7207e-05 gnorm: 0.95 [18:28:00< 3:32:23] +[titan] 2025-07-10 08:03:07,972 - root - INFO - step: 83920 loss: 16.2695 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7921 +[titan] 2025-07-10 08:03:07,972 - root - INFO - lr: 4.7197e-05 gnorm: 0.96 [18:28:04< 3:32:19] +[titan] 2025-07-10 08:03:11,926 - root - INFO - step: 83925 loss: 16.2539 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.7748 +[titan] 2025-07-10 08:03:11,927 - root - INFO - lr: 4.7187e-05 gnorm: 0.96 [18:28:08< 3:32:15] +[titan] 2025-07-10 08:03:15,865 - root - INFO - step: 83930 loss: 16.0032 memory: 44.58GiB(31.99%) tps: 83,213 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4406 global_avg_mtp_loss: 13.5626 +[titan] 2025-07-10 08:03:15,865 - root - INFO - lr: 4.7176e-05 gnorm: 0.96 [18:28:12< 3:32:11] +[titan] 2025-07-10 08:03:19,815 - root - INFO - step: 83935 loss: 16.0967 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.4562 global_avg_mtp_loss: 13.6406 +[titan] 2025-07-10 08:03:19,815 - root - INFO - lr: 4.7166e-05 gnorm: 1.01 [18:28:16< 3:32:07] +[titan] 2025-07-10 08:03:23,758 - root - INFO - step: 83940 loss: 16.4941 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9669 +[titan] 2025-07-10 08:03:23,758 - root - INFO - lr: 4.7155e-05 gnorm: 1.01 [18:28:20< 3:32:03] +[titan] 2025-07-10 08:03:27,702 - root - INFO - step: 83945 loss: 16.4178 memory: 44.58GiB(31.99%) tps: 83,088 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5072 global_avg_mtp_loss: 13.9107 +[titan] 2025-07-10 08:03:27,702 - root - INFO - lr: 4.7145e-05 gnorm: 0.98 [18:28:24< 3:31:59] +[titan] 2025-07-10 08:03:30,859 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:03:31,650 - root - INFO - step: 83950 loss: 16.2147 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7383 +[titan] 2025-07-10 08:03:31,650 - root - INFO - lr: 4.7134e-05 gnorm: 0.99 [18:28:28< 3:31:55] +[titan] 2025-07-10 08:03:35,573 - root - INFO - step: 83955 loss: 16.2685 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7799 +[titan] 2025-07-10 08:03:35,574 - root - INFO - lr: 4.7124e-05 gnorm: 0.97 [18:28:32< 3:31:51] +[titan] 2025-07-10 08:03:39,477 - root - INFO - step: 83960 loss: 16.3582 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8522 +[titan] 2025-07-10 08:03:39,477 - root - INFO - lr: 4.7113e-05 gnorm: 0.99 [18:28:36< 3:31:47] +[titan] 2025-07-10 08:03:43,574 - root - INFO - step: 83965 loss: 15.9914 memory: 44.58GiB(31.99%) tps: 79,997 tflops: 276.08 mfu: 27.92% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.5465 +[titan] 2025-07-10 08:03:43,574 - root - INFO - lr: 4.7103e-05 gnorm: 1.01 [18:28:40< 3:31:43] +[titan] 2025-07-10 08:03:46,089 - root - INFO - Dumping profiler traces at step 83968 +[titan] 2025-07-10 08:03:46,122 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:03:47,700 - root - INFO - step: 83970 loss: 16.2156 memory: 44.58GiB(31.99%) tps: 79,417 tflops: 274.08 mfu: 27.71% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.7239 +[titan] 2025-07-10 08:03:47,700 - root - INFO - lr: 4.7093e-05 gnorm: 0.98 [18:28:44< 3:31:39] +[titan] 2025-07-10 08:03:51,690 - root - INFO - step: 83975 loss: 16.2270 memory: 44.58GiB(31.99%) tps: 82,134 tflops: 283.46 mfu: 28.66% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7509 +[titan] 2025-07-10 08:03:51,690 - root - INFO - lr: 4.7082e-05 gnorm: 0.97 [18:28:48< 3:31:35] +[titan] 2025-07-10 08:03:55,612 - root - INFO - step: 83980 loss: 16.1829 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7120 +[titan] 2025-07-10 08:03:55,612 - root - INFO - lr: 4.7072e-05 gnorm: 0.97 [18:28:52< 3:31:31] +[titan] 2025-07-10 08:03:59,543 - root - INFO - step: 83985 loss: 16.3587 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5036 global_avg_mtp_loss: 13.8551 +[titan] 2025-07-10 08:03:59,543 - root - INFO - lr: 4.7061e-05 gnorm: 0.96 [18:28:56< 3:31:27] +[titan] 2025-07-10 08:04:03,453 - root - INFO - step: 83990 loss: 16.1882 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.7287 +[titan] 2025-07-10 08:04:03,453 - root - INFO - lr: 4.7051e-05 gnorm: 0.98 [18:29:00< 3:31:23] +[titan] 2025-07-10 08:04:07,385 - root - INFO - step: 83995 loss: 16.3259 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.8286 +[titan] 2025-07-10 08:04:07,385 - root - INFO - lr: 4.7040e-05 gnorm: 0.95 [18:29:04< 3:31:19] +[titan] 2025-07-10 08:04:10,517 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:04:11,307 - root - INFO - step: 84000 loss: 16.3831 memory: 44.58GiB(31.99%) tps: 83,555 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8889 +[titan] 2025-07-10 08:04:11,308 - root - INFO - lr: 4.7030e-05 gnorm: 0.97 [18:29:08< 3:31:15] +[titan] 2025-07-10 08:04:15,217 - root - INFO - step: 84005 loss: 16.4613 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.9449 +[titan] 2025-07-10 08:04:15,217 - root - INFO - lr: 4.7020e-05 gnorm: 1.00 [18:29:12< 3:31:11] +[titan] 2025-07-10 08:04:19,150 - root - INFO - step: 84010 loss: 16.3335 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4905 global_avg_mtp_loss: 13.8431 +[titan] 2025-07-10 08:04:19,151 - root - INFO - lr: 4.7009e-05 gnorm: 1.02 [18:29:16< 3:31:07] +[titan] 2025-07-10 08:04:23,066 - root - INFO - step: 84015 loss: 16.3075 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.8294 +[titan] 2025-07-10 08:04:23,066 - root - INFO - lr: 4.6999e-05 gnorm: 0.96 [18:29:19< 3:31:03] +[titan] 2025-07-10 08:04:27,014 - root - INFO - step: 84020 loss: 16.0728 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.6096 +[titan] 2025-07-10 08:04:27,014 - root - INFO - lr: 4.6988e-05 gnorm: 1.04 [18:29:23< 3:30:59] +[titan] 2025-07-10 08:04:30,979 - root - INFO - step: 84025 loss: 16.2820 memory: 44.58GiB(31.99%) tps: 82,639 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7948 +[titan] 2025-07-10 08:04:30,980 - root - INFO - lr: 4.6978e-05 gnorm: 0.93 [18:29:27< 3:30:56] +[titan] 2025-07-10 08:04:34,899 - root - INFO - step: 84030 loss: 16.2575 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7819 +[titan] 2025-07-10 08:04:34,900 - root - INFO - lr: 4.6968e-05 gnorm: 0.96 [18:29:31< 3:30:52] +[titan] 2025-07-10 08:04:38,810 - root - INFO - step: 84035 loss: 16.3461 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5055 global_avg_mtp_loss: 13.8406 +[titan] 2025-07-10 08:04:38,810 - root - INFO - lr: 4.6957e-05 gnorm: 1.01 [18:29:35< 3:30:48] +[titan] 2025-07-10 08:04:42,745 - root - INFO - step: 84040 loss: 16.3676 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.5043 global_avg_mtp_loss: 13.8634 +[titan] 2025-07-10 08:04:42,745 - root - INFO - lr: 4.6947e-05 gnorm: 0.95 [18:29:39< 3:30:44] +[titan] 2025-07-10 08:04:46,688 - root - INFO - step: 84045 loss: 15.9977 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4229 global_avg_mtp_loss: 13.5747 +[titan] 2025-07-10 08:04:46,688 - root - INFO - lr: 4.6936e-05 gnorm: 1.04 [18:29:43< 3:30:40] +[titan] 2025-07-10 08:04:49,821 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:04:50,620 - root - INFO - step: 84050 loss: 15.8872 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4377 global_avg_mtp_loss: 13.4494 +[titan] 2025-07-10 08:04:50,620 - root - INFO - lr: 4.6926e-05 gnorm: 1.00 [18:29:47< 3:30:36] +[titan] 2025-07-10 08:04:54,561 - root - INFO - step: 84055 loss: 16.3263 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 08:04:54,561 - root - INFO - lr: 4.6916e-05 gnorm: 1.05 [18:29:51< 3:30:32] +[titan] 2025-07-10 08:04:58,475 - root - INFO - step: 84060 loss: 16.0946 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6408 +[titan] 2025-07-10 08:04:58,476 - root - INFO - lr: 4.6905e-05 gnorm: 0.97 [18:29:55< 3:30:28] +[titan] 2025-07-10 08:05:02,400 - root - INFO - step: 84065 loss: 16.1235 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6594 +[titan] 2025-07-10 08:05:02,400 - root - INFO - lr: 4.6895e-05 gnorm: 0.96 [18:29:59< 3:30:24] +[titan] 2025-07-10 08:05:06,319 - root - INFO - step: 84070 loss: 16.3187 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5001 global_avg_mtp_loss: 13.8185 +[titan] 2025-07-10 08:05:06,319 - root - INFO - lr: 4.6884e-05 gnorm: 0.96 [18:30:03< 3:30:20] +[titan] 2025-07-10 08:05:10,274 - root - INFO - step: 84075 loss: 16.1095 memory: 44.58GiB(31.99%) tps: 82,854 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.4483 global_avg_mtp_loss: 13.6611 +[titan] 2025-07-10 08:05:10,275 - root - INFO - lr: 4.6874e-05 gnorm: 0.97 [18:30:07< 3:30:16] +[titan] 2025-07-10 08:05:14,193 - root - INFO - step: 84080 loss: 16.0110 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4310 global_avg_mtp_loss: 13.5800 +[titan] 2025-07-10 08:05:14,194 - root - INFO - lr: 4.6864e-05 gnorm: 1.01 [18:30:11< 3:30:12] +[titan] 2025-07-10 08:05:18,144 - root - INFO - step: 84085 loss: 16.0870 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4359 global_avg_mtp_loss: 13.6511 +[titan] 2025-07-10 08:05:18,145 - root - INFO - lr: 4.6853e-05 gnorm: 0.99 [18:30:15< 3:30:08] +[titan] 2025-07-10 08:05:22,130 - root - INFO - step: 84090 loss: 16.3702 memory: 44.58GiB(31.99%) tps: 82,223 tflops: 283.76 mfu: 28.69% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.8644 +[titan] 2025-07-10 08:05:22,130 - root - INFO - lr: 4.6843e-05 gnorm: 0.97 [18:30:18< 3:30:04] +[titan] 2025-07-10 08:05:26,071 - root - INFO - step: 84095 loss: 16.3349 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-10 08:05:26,071 - root - INFO - lr: 4.6833e-05 gnorm: 1.03 [18:30:22< 3:30:00] +[titan] 2025-07-10 08:05:29,240 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:05:30,030 - root - INFO - step: 84100 loss: 16.1818 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.7105 +[titan] 2025-07-10 08:05:30,030 - root - INFO - lr: 4.6822e-05 gnorm: 0.97 [18:30:26< 3:29:56] +[titan] 2025-07-10 08:05:33,977 - root - INFO - step: 84105 loss: 16.3568 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5047 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 08:05:33,977 - root - INFO - lr: 4.6812e-05 gnorm: 0.97 [18:30:30< 3:29:52] +[titan] 2025-07-10 08:05:37,898 - root - INFO - step: 84110 loss: 16.3172 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8212 +[titan] 2025-07-10 08:05:37,898 - root - INFO - lr: 4.6802e-05 gnorm: 1.00 [18:30:34< 3:29:48] +[titan] 2025-07-10 08:05:41,828 - root - INFO - step: 84115 loss: 15.9209 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4232 global_avg_mtp_loss: 13.4977 +[titan] 2025-07-10 08:05:41,828 - root - INFO - lr: 4.6791e-05 gnorm: 1.00 [18:30:38< 3:29:44] +[titan] 2025-07-10 08:05:45,728 - root - INFO - step: 84120 loss: 16.2078 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7345 +[titan] 2025-07-10 08:05:45,728 - root - INFO - lr: 4.6781e-05 gnorm: 0.99 [18:30:42< 3:29:40] +[titan] 2025-07-10 08:05:49,649 - root - INFO - step: 84125 loss: 16.2126 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.7447 +[titan] 2025-07-10 08:05:49,649 - root - INFO - lr: 4.6771e-05 gnorm: 0.97 [18:30:46< 3:29:36] +[titan] 2025-07-10 08:05:53,589 - root - INFO - step: 84130 loss: 16.2778 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7957 +[titan] 2025-07-10 08:05:53,589 - root - INFO - lr: 4.6760e-05 gnorm: 1.03 [18:30:50< 3:29:32] +[titan] 2025-07-10 08:05:57,497 - root - INFO - step: 84135 loss: 16.0422 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4515 global_avg_mtp_loss: 13.5907 +[titan] 2025-07-10 08:05:57,497 - root - INFO - lr: 4.6750e-05 gnorm: 1.01 [18:30:54< 3:29:28] +[titan] 2025-07-10 08:06:01,428 - root - INFO - step: 84140 loss: 15.9081 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4322 global_avg_mtp_loss: 13.4759 +[titan] 2025-07-10 08:06:01,428 - root - INFO - lr: 4.6740e-05 gnorm: 1.08 [18:30:58< 3:29:24] +[titan] 2025-07-10 08:06:05,352 - root - INFO - step: 84145 loss: 16.1211 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6613 +[titan] 2025-07-10 08:06:05,353 - root - INFO - lr: 4.6729e-05 gnorm: 1.00 [18:31:02< 3:29:20] +[titan] 2025-07-10 08:06:08,470 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:06:09,257 - root - INFO - step: 84150 loss: 16.3777 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.8701 +[titan] 2025-07-10 08:06:09,257 - root - INFO - lr: 4.6719e-05 gnorm: 1.02 [18:31:06< 3:29:16] +[titan] 2025-07-10 08:06:13,183 - root - INFO - step: 84155 loss: 16.2458 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7690 +[titan] 2025-07-10 08:06:13,184 - root - INFO - lr: 4.6709e-05 gnorm: 1.01 [18:31:10< 3:29:12] +[titan] 2025-07-10 08:06:17,079 - root - INFO - step: 84160 loss: 16.3422 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 08:06:17,080 - root - INFO - lr: 4.6698e-05 gnorm: 0.99 [18:31:13< 3:29:08] +[titan] 2025-07-10 08:06:20,990 - root - INFO - step: 84165 loss: 16.0328 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4443 global_avg_mtp_loss: 13.5885 +[titan] 2025-07-10 08:06:20,990 - root - INFO - lr: 4.6688e-05 gnorm: 1.03 [18:31:17< 3:29:04] +[titan] 2025-07-10 08:06:24,912 - root - INFO - step: 84170 loss: 16.3693 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.8508 +[titan] 2025-07-10 08:06:24,912 - root - INFO - lr: 4.6678e-05 gnorm: 1.01 [18:31:21< 3:29:00] +[titan] 2025-07-10 08:06:28,879 - root - INFO - step: 84175 loss: 16.2169 memory: 44.58GiB(31.99%) tps: 82,605 tflops: 285.08 mfu: 28.83% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7356 +[titan] 2025-07-10 08:06:28,879 - root - INFO - lr: 4.6667e-05 gnorm: 0.94 [18:31:25< 3:28:56] +[titan] 2025-07-10 08:06:32,802 - root - INFO - step: 84180 loss: 16.1096 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6511 +[titan] 2025-07-10 08:06:32,802 - root - INFO - lr: 4.6657e-05 gnorm: 0.97 [18:31:29< 3:28:53] +[titan] 2025-07-10 08:06:36,705 - root - INFO - step: 84185 loss: 16.2787 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.7888 +[titan] 2025-07-10 08:06:36,706 - root - INFO - lr: 4.6647e-05 gnorm: 0.98 [18:31:33< 3:28:49] +[titan] 2025-07-10 08:06:40,619 - root - INFO - step: 84190 loss: 16.1354 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4599 global_avg_mtp_loss: 13.6755 +[titan] 2025-07-10 08:06:40,620 - root - INFO - lr: 4.6636e-05 gnorm: 0.99 [18:31:37< 3:28:45] +[titan] 2025-07-10 08:06:44,568 - root - INFO - step: 84195 loss: 16.0842 memory: 44.58GiB(31.99%) tps: 83,002 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6212 +[titan] 2025-07-10 08:06:44,568 - root - INFO - lr: 4.6626e-05 gnorm: 0.98 [18:31:41< 3:28:41] +[titan] 2025-07-10 08:06:47,724 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:06:48,510 - root - INFO - step: 84200 loss: 16.4131 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.9017 +[titan] 2025-07-10 08:06:48,511 - root - INFO - lr: 4.6616e-05 gnorm: 0.98 [18:31:45< 3:28:37] +[titan] 2025-07-10 08:06:52,439 - root - INFO - step: 84205 loss: 16.0856 memory: 44.58GiB(31.99%) tps: 83,418 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6240 +[titan] 2025-07-10 08:06:52,439 - root - INFO - lr: 4.6606e-05 gnorm: 0.98 [18:31:49< 3:28:33] +[titan] 2025-07-10 08:06:56,372 - root - INFO - step: 84210 loss: 16.7450 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5775 global_avg_mtp_loss: 14.1675 +[titan] 2025-07-10 08:06:56,372 - root - INFO - lr: 4.6595e-05 gnorm: 1.00 [18:31:53< 3:28:29] +[titan] 2025-07-10 08:07:00,317 - root - INFO - step: 84215 loss: 16.2212 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7505 +[titan] 2025-07-10 08:07:00,317 - root - INFO - lr: 4.6585e-05 gnorm: 0.99 [18:31:57< 3:28:25] +[titan] 2025-07-10 08:07:04,220 - root - INFO - step: 84220 loss: 16.2572 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.7732 +[titan] 2025-07-10 08:07:04,220 - root - INFO - lr: 4.6575e-05 gnorm: 0.97 [18:32:01< 3:28:21] +[titan] 2025-07-10 08:07:08,177 - root - INFO - step: 84225 loss: 16.1634 memory: 44.58GiB(31.99%) tps: 82,827 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6970 +[titan] 2025-07-10 08:07:08,177 - root - INFO - lr: 4.6564e-05 gnorm: 0.97 [18:32:05< 3:28:17] +[titan] 2025-07-10 08:07:12,080 - root - INFO - step: 84230 loss: 16.1548 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.6792 +[titan] 2025-07-10 08:07:12,080 - root - INFO - lr: 4.6554e-05 gnorm: 0.99 [18:32:08< 3:28:13] +[titan] 2025-07-10 08:07:16,016 - root - INFO - step: 84235 loss: 16.3281 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8319 +[titan] 2025-07-10 08:07:16,017 - root - INFO - lr: 4.6544e-05 gnorm: 0.98 [18:32:12< 3:28:09] +[titan] 2025-07-10 08:07:19,916 - root - INFO - step: 84240 loss: 16.4585 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.5276 global_avg_mtp_loss: 13.9309 +[titan] 2025-07-10 08:07:19,916 - root - INFO - lr: 4.6534e-05 gnorm: 0.97 [18:32:16< 3:28:05] +[titan] 2025-07-10 08:07:23,833 - root - INFO - step: 84245 loss: 16.2538 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7739 +[titan] 2025-07-10 08:07:23,833 - root - INFO - lr: 4.6523e-05 gnorm: 1.00 [18:32:20< 3:28:01] +[titan] 2025-07-10 08:07:26,945 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:07:27,739 - root - INFO - step: 84250 loss: 16.3206 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8202 +[titan] 2025-07-10 08:07:27,739 - root - INFO - lr: 4.6513e-05 gnorm: 1.04 [18:32:24< 3:27:57] +[titan] 2025-07-10 08:07:31,685 - root - INFO - step: 84255 loss: 16.4969 memory: 44.58GiB(31.99%) tps: 83,043 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.5399 global_avg_mtp_loss: 13.9570 +[titan] 2025-07-10 08:07:31,686 - root - INFO - lr: 4.6503e-05 gnorm: 1.02 [18:32:28< 3:27:53] +[titan] 2025-07-10 08:07:35,593 - root - INFO - step: 84260 loss: 16.1296 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.6579 +[titan] 2025-07-10 08:07:35,593 - root - INFO - lr: 4.6492e-05 gnorm: 0.99 [18:32:32< 3:27:49] +[titan] 2025-07-10 08:07:39,519 - root - INFO - step: 84265 loss: 16.0999 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4544 global_avg_mtp_loss: 13.6456 +[titan] 2025-07-10 08:07:39,520 - root - INFO - lr: 4.6482e-05 gnorm: 1.01 [18:32:36< 3:27:45] +[titan] 2025-07-10 08:07:43,454 - root - INFO - step: 84270 loss: 15.9923 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4448 global_avg_mtp_loss: 13.5476 +[titan] 2025-07-10 08:07:43,454 - root - INFO - lr: 4.6472e-05 gnorm: 1.00 [18:32:40< 3:27:41] +[titan] 2025-07-10 08:07:47,369 - root - INFO - step: 84275 loss: 16.1307 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6789 +[titan] 2025-07-10 08:07:47,369 - root - INFO - lr: 4.6462e-05 gnorm: 0.98 [18:32:44< 3:27:37] +[titan] 2025-07-10 08:07:51,326 - root - INFO - step: 84280 loss: 16.1073 memory: 44.58GiB(31.99%) tps: 82,809 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6457 +[titan] 2025-07-10 08:07:51,326 - root - INFO - lr: 4.6451e-05 gnorm: 0.97 [18:32:48< 3:27:33] +[titan] 2025-07-10 08:07:55,233 - root - INFO - step: 84285 loss: 16.4944 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 13.9696 +[titan] 2025-07-10 08:07:55,233 - root - INFO - lr: 4.6441e-05 gnorm: 1.00 [18:32:52< 3:27:29] +[titan] 2025-07-10 08:07:59,161 - root - INFO - step: 84290 loss: 15.8778 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4151 global_avg_mtp_loss: 13.4627 +[titan] 2025-07-10 08:07:59,162 - root - INFO - lr: 4.6431e-05 gnorm: 1.02 [18:32:56< 3:27:25] +[titan] 2025-07-10 08:08:03,060 - root - INFO - step: 84295 loss: 16.3742 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.8722 +[titan] 2025-07-10 08:08:03,060 - root - INFO - lr: 4.6421e-05 gnorm: 0.97 [18:32:59< 3:27:21] +[titan] 2025-07-10 08:08:06,214 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:08:07,003 - root - INFO - step: 84300 loss: 15.9966 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4332 global_avg_mtp_loss: 13.5634 +[titan] 2025-07-10 08:08:07,004 - root - INFO - lr: 4.6411e-05 gnorm: 1.00 [18:33:03< 3:27:17] +[titan] 2025-07-10 08:08:10,937 - root - INFO - step: 84305 loss: 16.0252 memory: 44.58GiB(31.99%) tps: 83,321 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.5736 +[titan] 2025-07-10 08:08:10,937 - root - INFO - lr: 4.6400e-05 gnorm: 1.01 [18:33:07< 3:27:13] +[titan] 2025-07-10 08:08:14,873 - root - INFO - step: 84310 loss: 16.0926 memory: 44.58GiB(31.99%) tps: 83,260 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.6464 +[titan] 2025-07-10 08:08:14,873 - root - INFO - lr: 4.6390e-05 gnorm: 1.01 [18:33:11< 3:27:09] +[titan] 2025-07-10 08:08:18,796 - root - INFO - step: 84315 loss: 15.9898 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4311 global_avg_mtp_loss: 13.5587 +[titan] 2025-07-10 08:08:18,796 - root - INFO - lr: 4.6380e-05 gnorm: 1.02 [18:33:15< 3:27:05] +[titan] 2025-07-10 08:08:22,699 - root - INFO - step: 84320 loss: 16.2123 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7306 +[titan] 2025-07-10 08:08:22,699 - root - INFO - lr: 4.6370e-05 gnorm: 1.04 [18:33:19< 3:27:01] +[titan] 2025-07-10 08:08:26,607 - root - INFO - step: 84325 loss: 16.1195 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.6585 +[titan] 2025-07-10 08:08:26,607 - root - INFO - lr: 4.6359e-05 gnorm: 0.98 [18:33:23< 3:26:57] +[titan] 2025-07-10 08:08:30,517 - root - INFO - step: 84330 loss: 16.1032 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6489 +[titan] 2025-07-10 08:08:30,517 - root - INFO - lr: 4.6349e-05 gnorm: 0.96 [18:33:27< 3:26:53] +[titan] 2025-07-10 08:08:34,439 - root - INFO - step: 84335 loss: 16.2655 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7861 +[titan] 2025-07-10 08:08:34,439 - root - INFO - lr: 4.6339e-05 gnorm: 0.96 [18:33:31< 3:26:50] +[titan] 2025-07-10 08:08:38,356 - root - INFO - step: 84340 loss: 16.1506 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 08:08:38,357 - root - INFO - lr: 4.6329e-05 gnorm: 0.97 [18:33:35< 3:26:46] +[titan] 2025-07-10 08:08:42,268 - root - INFO - step: 84345 loss: 16.0497 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4398 global_avg_mtp_loss: 13.6100 +[titan] 2025-07-10 08:08:42,268 - root - INFO - lr: 4.6319e-05 gnorm: 0.98 [18:33:39< 3:26:42] +[titan] 2025-07-10 08:08:45,397 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:08:46,187 - root - INFO - step: 84350 loss: 16.3989 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8994 +[titan] 2025-07-10 08:08:46,188 - root - INFO - lr: 4.6308e-05 gnorm: 0.99 [18:33:43< 3:26:38] +[titan] 2025-07-10 08:08:50,091 - root - INFO - step: 84355 loss: 16.1529 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.6918 +[titan] 2025-07-10 08:08:50,091 - root - INFO - lr: 4.6298e-05 gnorm: 0.98 [18:33:46< 3:26:34] +[titan] 2025-07-10 08:08:54,000 - root - INFO - step: 84360 loss: 16.3288 memory: 44.58GiB(31.99%) tps: 83,842 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.8382 +[titan] 2025-07-10 08:08:54,000 - root - INFO - lr: 4.6288e-05 gnorm: 0.99 [18:33:50< 3:26:30] +[titan] 2025-07-10 08:08:57,939 - root - INFO - step: 84365 loss: 16.2719 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7910 +[titan] 2025-07-10 08:08:57,939 - root - INFO - lr: 4.6278e-05 gnorm: 0.97 [18:33:54< 3:26:26] +[titan] 2025-07-10 08:09:01,855 - root - INFO - step: 84370 loss: 16.1386 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 08:09:01,855 - root - INFO - lr: 4.6268e-05 gnorm: 1.00 [18:33:58< 3:26:22] +[titan] 2025-07-10 08:09:05,798 - root - INFO - step: 84375 loss: 16.1617 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6999 +[titan] 2025-07-10 08:09:05,798 - root - INFO - lr: 4.6257e-05 gnorm: 0.97 [18:34:02< 3:26:18] +[titan] 2025-07-10 08:09:09,755 - root - INFO - step: 84380 loss: 16.4878 memory: 44.58GiB(31.99%) tps: 82,818 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9577 +[titan] 2025-07-10 08:09:09,755 - root - INFO - lr: 4.6247e-05 gnorm: 1.02 [18:34:06< 3:26:14] +[titan] 2025-07-10 08:09:13,667 - root - INFO - step: 84385 loss: 16.3049 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8020 +[titan] 2025-07-10 08:09:13,667 - root - INFO - lr: 4.6237e-05 gnorm: 0.97 [18:34:10< 3:26:10] +[titan] 2025-07-10 08:09:17,625 - root - INFO - step: 84390 loss: 16.1279 memory: 44.58GiB(31.99%) tps: 82,795 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.6590 +[titan] 2025-07-10 08:09:17,626 - root - INFO - lr: 4.6227e-05 gnorm: 0.95 [18:34:14< 3:26:06] +[titan] 2025-07-10 08:09:21,561 - root - INFO - step: 84395 loss: 16.5997 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5540 global_avg_mtp_loss: 14.0457 +[titan] 2025-07-10 08:09:21,561 - root - INFO - lr: 4.6217e-05 gnorm: 0.99 [18:34:18< 3:26:02] +[titan] 2025-07-10 08:09:24,706 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:09:25,508 - root - INFO - step: 84400 loss: 16.1828 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7170 +[titan] 2025-07-10 08:09:25,508 - root - INFO - lr: 4.6206e-05 gnorm: 0.97 [18:34:22< 3:25:58] +[titan] 2025-07-10 08:09:29,418 - root - INFO - step: 84405 loss: 15.9849 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4444 global_avg_mtp_loss: 13.5405 +[titan] 2025-07-10 08:09:29,419 - root - INFO - lr: 4.6196e-05 gnorm: 0.97 [18:34:26< 3:25:54] +[titan] 2025-07-10 08:09:33,340 - root - INFO - step: 84410 loss: 16.3254 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4948 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 08:09:33,341 - root - INFO - lr: 4.6186e-05 gnorm: 0.99 [18:34:30< 3:25:50] +[titan] 2025-07-10 08:09:37,266 - root - INFO - step: 84415 loss: 16.1890 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.7169 +[titan] 2025-07-10 08:09:37,266 - root - INFO - lr: 4.6176e-05 gnorm: 1.02 [18:34:34< 3:25:46] +[titan] 2025-07-10 08:09:41,181 - root - INFO - step: 84420 loss: 16.3798 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.8654 +[titan] 2025-07-10 08:09:41,181 - root - INFO - lr: 4.6166e-05 gnorm: 1.02 [18:34:38< 3:25:42] +[titan] 2025-07-10 08:09:45,092 - root - INFO - step: 84425 loss: 16.0073 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4339 global_avg_mtp_loss: 13.5734 +[titan] 2025-07-10 08:09:45,092 - root - INFO - lr: 4.6156e-05 gnorm: 0.95 [18:34:41< 3:25:38] +[titan] 2025-07-10 08:09:49,004 - root - INFO - step: 84430 loss: 16.2841 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7968 +[titan] 2025-07-10 08:09:49,004 - root - INFO - lr: 4.6145e-05 gnorm: 0.96 [18:34:45< 3:25:34] +[titan] 2025-07-10 08:09:52,906 - root - INFO - step: 84435 loss: 16.2446 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7577 +[titan] 2025-07-10 08:09:52,906 - root - INFO - lr: 4.6135e-05 gnorm: 0.98 [18:34:49< 3:25:30] +[titan] 2025-07-10 08:09:56,849 - root - INFO - step: 84440 loss: 16.2038 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4827 global_avg_mtp_loss: 13.7211 +[titan] 2025-07-10 08:09:56,849 - root - INFO - lr: 4.6125e-05 gnorm: 0.97 [18:34:53< 3:25:26] +[titan] 2025-07-10 08:10:00,759 - root - INFO - step: 84445 loss: 16.2954 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8042 +[titan] 2025-07-10 08:10:00,759 - root - INFO - lr: 4.6115e-05 gnorm: 0.99 [18:34:57< 3:25:22] +[titan] 2025-07-10 08:10:03,941 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:10:04,742 - root - INFO - step: 84450 loss: 16.4365 memory: 44.58GiB(31.99%) tps: 82,266 tflops: 283.91 mfu: 28.71% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9171 +[titan] 2025-07-10 08:10:04,743 - root - INFO - lr: 4.6105e-05 gnorm: 0.96 [18:35:01< 3:25:18] +[titan] 2025-07-10 08:10:08,655 - root - INFO - step: 84455 loss: 16.0967 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6282 +[titan] 2025-07-10 08:10:08,655 - root - INFO - lr: 4.6095e-05 gnorm: 1.09 [18:35:05< 3:25:14] +[titan] 2025-07-10 08:10:12,578 - root - INFO - step: 84460 loss: 16.0541 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.6146 +[titan] 2025-07-10 08:10:12,578 - root - INFO - lr: 4.6085e-05 gnorm: 0.98 [18:35:09< 3:25:10] +[titan] 2025-07-10 08:10:16,481 - root - INFO - step: 84465 loss: 16.1139 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.6435 +[titan] 2025-07-10 08:10:16,481 - root - INFO - lr: 4.6074e-05 gnorm: 0.95 [18:35:13< 3:25:06] +[titan] 2025-07-10 08:10:20,381 - root - INFO - step: 84470 loss: 16.3327 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.8399 +[titan] 2025-07-10 08:10:20,382 - root - INFO - lr: 4.6064e-05 gnorm: 0.97 [18:35:17< 3:25:02] +[titan] 2025-07-10 08:10:24,286 - root - INFO - step: 84475 loss: 16.1358 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6700 +[titan] 2025-07-10 08:10:24,286 - root - INFO - lr: 4.6054e-05 gnorm: 1.03 [18:35:21< 3:24:58] +[titan] 2025-07-10 08:10:28,315 - root - INFO - step: 84480 loss: 16.4606 memory: 44.58GiB(31.99%) tps: 81,347 tflops: 280.74 mfu: 28.39% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.9490 +[titan] 2025-07-10 08:10:28,315 - root - INFO - lr: 4.6044e-05 gnorm: 0.97 [18:35:25< 3:24:54] +[titan] 2025-07-10 08:10:28,472 - root - INFO - Dumping profiler traces at step 84480 +[titan] 2025-07-10 08:10:28,508 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 08:10:32,439 - root - INFO - step: 84485 loss: 16.3274 memory: 44.58GiB(31.99%) tps: 79,466 tflops: 274.25 mfu: 27.73% global_avg_ntp_loss: 2.4886 global_avg_mtp_loss: 13.8389 +[titan] 2025-07-10 08:10:32,439 - root - INFO - lr: 4.6034e-05 gnorm: 1.02 [18:35:29< 3:24:51] +[titan] 2025-07-10 08:10:36,347 - root - INFO - step: 84490 loss: 16.2093 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7229 +[titan] 2025-07-10 08:10:36,347 - root - INFO - lr: 4.6024e-05 gnorm: 1.01 [18:35:33< 3:24:47] +[titan] 2025-07-10 08:10:40,241 - root - INFO - step: 84495 loss: 16.2602 memory: 44.58GiB(31.99%) tps: 84,170 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7822 +[titan] 2025-07-10 08:10:40,241 - root - INFO - lr: 4.6014e-05 gnorm: 1.01 [18:35:37< 3:24:43] +[titan] 2025-07-10 08:10:43,363 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:10:44,148 - root - INFO - step: 84500 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8501 +[titan] 2025-07-10 08:10:44,149 - root - INFO - lr: 4.6003e-05 gnorm: 1.00 [18:35:40< 3:24:39] +[titan] 2025-07-10 08:10:48,052 - root - INFO - step: 84505 loss: 16.2983 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.8165 +[titan] 2025-07-10 08:10:48,052 - root - INFO - lr: 4.5993e-05 gnorm: 0.99 [18:35:44< 3:24:35] +[titan] 2025-07-10 08:10:52,010 - root - INFO - step: 84510 loss: 16.3896 memory: 44.58GiB(31.99%) tps: 82,791 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8796 +[titan] 2025-07-10 08:10:52,010 - root - INFO - lr: 4.5983e-05 gnorm: 1.00 [18:35:48< 3:24:31] +[titan] 2025-07-10 08:10:55,948 - root - INFO - step: 84515 loss: 16.0510 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.5922 +[titan] 2025-07-10 08:10:55,949 - root - INFO - lr: 4.5973e-05 gnorm: 1.04 [18:35:52< 3:24:27] +[titan] 2025-07-10 08:10:59,889 - root - INFO - step: 84520 loss: 16.1792 memory: 44.58GiB(31.99%) tps: 83,164 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7057 +[titan] 2025-07-10 08:10:59,889 - root - INFO - lr: 4.5963e-05 gnorm: 1.00 [18:35:56< 3:24:23] +[titan] 2025-07-10 08:11:03,801 - root - INFO - step: 84525 loss: 16.5004 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5327 global_avg_mtp_loss: 13.9677 +[titan] 2025-07-10 08:11:03,801 - root - INFO - lr: 4.5953e-05 gnorm: 1.04 [18:36:00< 3:24:19] +[titan] 2025-07-10 08:11:07,707 - root - INFO - step: 84530 loss: 15.9469 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.5090 +[titan] 2025-07-10 08:11:07,707 - root - INFO - lr: 4.5943e-05 gnorm: 1.11 [18:36:04< 3:24:15] +[titan] 2025-07-10 08:11:11,615 - root - INFO - step: 84535 loss: 16.1349 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6872 +[titan] 2025-07-10 08:11:11,615 - root - INFO - lr: 4.5933e-05 gnorm: 0.95 [18:36:08< 3:24:11] +[titan] 2025-07-10 08:11:15,516 - root - INFO - step: 84540 loss: 16.0487 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4449 global_avg_mtp_loss: 13.6037 +[titan] 2025-07-10 08:11:15,516 - root - INFO - lr: 4.5923e-05 gnorm: 1.00 [18:36:12< 3:24:07] +[titan] 2025-07-10 08:11:19,444 - root - INFO - step: 84545 loss: 16.3427 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8383 +[titan] 2025-07-10 08:11:19,444 - root - INFO - lr: 4.5913e-05 gnorm: 1.03 [18:36:16< 3:24:03] +[titan] 2025-07-10 08:11:22,567 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:11:23,354 - root - INFO - step: 84550 loss: 16.1230 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6595 +[titan] 2025-07-10 08:11:23,354 - root - INFO - lr: 4.5902e-05 gnorm: 0.99 [18:36:20< 3:23:59] +[titan] 2025-07-10 08:11:27,297 - root - INFO - step: 84555 loss: 16.4551 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5203 global_avg_mtp_loss: 13.9349 +[titan] 2025-07-10 08:11:27,297 - root - INFO - lr: 4.5892e-05 gnorm: 1.08 [18:36:24< 3:23:55] +[titan] 2025-07-10 08:11:31,203 - root - INFO - step: 84560 loss: 15.9720 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4343 global_avg_mtp_loss: 13.5378 +[titan] 2025-07-10 08:11:31,204 - root - INFO - lr: 4.5882e-05 gnorm: 1.01 [18:36:28< 3:23:51] +[titan] 2025-07-10 08:11:35,122 - root - INFO - step: 84565 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.8019 +[titan] 2025-07-10 08:11:35,122 - root - INFO - lr: 4.5872e-05 gnorm: 0.99 [18:36:31< 3:23:47] +[titan] 2025-07-10 08:11:39,060 - root - INFO - step: 84570 loss: 16.2621 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7798 +[titan] 2025-07-10 08:11:39,060 - root - INFO - lr: 4.5862e-05 gnorm: 0.94 [18:36:35< 3:23:43] +[titan] 2025-07-10 08:11:42,986 - root - INFO - step: 84575 loss: 16.0781 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.4510 global_avg_mtp_loss: 13.6271 +[titan] 2025-07-10 08:11:42,986 - root - INFO - lr: 4.5852e-05 gnorm: 0.96 [18:36:39< 3:23:39] +[titan] 2025-07-10 08:11:46,889 - root - INFO - step: 84580 loss: 16.1537 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6964 +[titan] 2025-07-10 08:11:46,889 - root - INFO - lr: 4.5842e-05 gnorm: 0.98 [18:36:43< 3:23:35] +[titan] 2025-07-10 08:11:50,785 - root - INFO - step: 84585 loss: 16.1872 memory: 44.58GiB(31.99%) tps: 84,100 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7050 +[titan] 2025-07-10 08:11:50,786 - root - INFO - lr: 4.5832e-05 gnorm: 1.01 [18:36:47< 3:23:31] +[titan] 2025-07-10 08:11:54,701 - root - INFO - step: 84590 loss: 16.4566 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5340 global_avg_mtp_loss: 13.9226 +[titan] 2025-07-10 08:11:54,701 - root - INFO - lr: 4.5822e-05 gnorm: 1.01 [18:36:51< 3:23:27] +[titan] 2025-07-10 08:11:58,669 - root - INFO - step: 84595 loss: 16.0981 memory: 44.58GiB(31.99%) tps: 82,588 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.4491 global_avg_mtp_loss: 13.6490 +[titan] 2025-07-10 08:11:58,669 - root - INFO - lr: 4.5812e-05 gnorm: 0.92 [18:36:55< 3:23:23] +[titan] 2025-07-10 08:12:01,810 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:12:02,627 - root - INFO - step: 84600 loss: 16.4000 memory: 44.58GiB(31.99%) tps: 82,800 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8848 +[titan] 2025-07-10 08:12:02,627 - root - INFO - lr: 4.5802e-05 gnorm: 0.99 [18:36:59< 3:23:19] +[titan] 2025-07-10 08:12:06,537 - root - INFO - step: 84605 loss: 16.0813 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6335 +[titan] 2025-07-10 08:12:06,538 - root - INFO - lr: 4.5792e-05 gnorm: 0.97 [18:37:03< 3:23:15] +[titan] 2025-07-10 08:12:10,431 - root - INFO - step: 84610 loss: 16.2550 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7723 +[titan] 2025-07-10 08:12:10,431 - root - INFO - lr: 4.5782e-05 gnorm: 0.93 [18:37:07< 3:23:11] +[titan] 2025-07-10 08:12:14,332 - root - INFO - step: 84615 loss: 16.1813 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.6985 +[titan] 2025-07-10 08:12:14,333 - root - INFO - lr: 4.5772e-05 gnorm: 1.01 [18:37:11< 3:23:07] +[titan] 2025-07-10 08:12:18,226 - root - INFO - step: 84620 loss: 16.1770 memory: 44.58GiB(31.99%) tps: 84,166 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.7119 +[titan] 2025-07-10 08:12:18,226 - root - INFO - lr: 4.5762e-05 gnorm: 1.02 [18:37:15< 3:23:03] +[titan] 2025-07-10 08:12:22,129 - root - INFO - step: 84625 loss: 15.6948 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.3901 global_avg_mtp_loss: 13.3046 +[titan] 2025-07-10 08:12:22,129 - root - INFO - lr: 4.5752e-05 gnorm: 1.02 [18:37:18< 3:22:59] +[titan] 2025-07-10 08:12:26,025 - root - INFO - step: 84630 loss: 16.3921 memory: 44.58GiB(31.99%) tps: 84,102 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.5063 global_avg_mtp_loss: 13.8858 +[titan] 2025-07-10 08:12:26,026 - root - INFO - lr: 4.5741e-05 gnorm: 1.04 [18:37:22< 3:22:55] +[titan] 2025-07-10 08:12:29,941 - root - INFO - step: 84635 loss: 16.1195 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6677 +[titan] 2025-07-10 08:12:29,942 - root - INFO - lr: 4.5731e-05 gnorm: 1.04 [18:37:26< 3:22:51] +[titan] 2025-07-10 08:12:33,872 - root - INFO - step: 84640 loss: 16.2570 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7788 +[titan] 2025-07-10 08:12:33,872 - root - INFO - lr: 4.5721e-05 gnorm: 0.99 [18:37:30< 3:22:47] +[titan] 2025-07-10 08:12:37,783 - root - INFO - step: 84645 loss: 16.1038 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.6454 +[titan] 2025-07-10 08:12:37,783 - root - INFO - lr: 4.5711e-05 gnorm: 1.02 [18:37:34< 3:22:44] +[titan] 2025-07-10 08:12:40,903 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:12:41,696 - root - INFO - step: 84650 loss: 16.4956 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.9771 +[titan] 2025-07-10 08:12:41,696 - root - INFO - lr: 4.5701e-05 gnorm: 0.99 [18:37:38< 3:22:40] +[titan] 2025-07-10 08:12:45,625 - root - INFO - step: 84655 loss: 16.2594 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 08:12:45,625 - root - INFO - lr: 4.5691e-05 gnorm: 0.99 [18:37:42< 3:22:36] +[titan] 2025-07-10 08:12:49,559 - root - INFO - step: 84660 loss: 16.2234 memory: 44.58GiB(31.99%) tps: 83,307 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7450 +[titan] 2025-07-10 08:12:49,559 - root - INFO - lr: 4.5681e-05 gnorm: 0.95 [18:37:46< 3:22:32] +[titan] 2025-07-10 08:12:53,462 - root - INFO - step: 84665 loss: 16.2199 memory: 44.58GiB(31.99%) tps: 83,963 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7349 +[titan] 2025-07-10 08:12:53,462 - root - INFO - lr: 4.5671e-05 gnorm: 1.00 [18:37:50< 3:22:28] +[titan] 2025-07-10 08:12:57,392 - root - INFO - step: 84670 loss: 16.2113 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7274 +[titan] 2025-07-10 08:12:57,392 - root - INFO - lr: 4.5661e-05 gnorm: 0.97 [18:37:54< 3:22:24] +[titan] 2025-07-10 08:13:01,304 - root - INFO - step: 84675 loss: 16.1752 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4706 global_avg_mtp_loss: 13.7046 +[titan] 2025-07-10 08:13:01,305 - root - INFO - lr: 4.5651e-05 gnorm: 0.94 [18:37:58< 3:22:20] +[titan] 2025-07-10 08:13:05,222 - root - INFO - step: 84680 loss: 16.3462 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8403 +[titan] 2025-07-10 08:13:05,222 - root - INFO - lr: 4.5641e-05 gnorm: 1.00 [18:38:02< 3:22:16] +[titan] 2025-07-10 08:13:09,168 - root - INFO - step: 84685 loss: 16.0617 memory: 44.58GiB(31.99%) tps: 83,055 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.6128 +[titan] 2025-07-10 08:13:09,168 - root - INFO - lr: 4.5631e-05 gnorm: 1.02 [18:38:05< 3:22:12] +[titan] 2025-07-10 08:13:13,106 - root - INFO - step: 84690 loss: 16.3961 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.5189 global_avg_mtp_loss: 13.8773 +[titan] 2025-07-10 08:13:13,106 - root - INFO - lr: 4.5621e-05 gnorm: 1.06 [18:38:09< 3:22:08] +[titan] 2025-07-10 08:13:17,044 - root - INFO - step: 84695 loss: 16.3809 memory: 44.58GiB(31.99%) tps: 83,211 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8870 +[titan] 2025-07-10 08:13:17,045 - root - INFO - lr: 4.5611e-05 gnorm: 0.97 [18:38:13< 3:22:04] +[titan] 2025-07-10 08:13:20,174 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:13:20,973 - root - INFO - step: 84700 loss: 16.0809 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6290 +[titan] 2025-07-10 08:13:20,973 - root - INFO - lr: 4.5601e-05 gnorm: 1.04 [18:38:17< 3:22:00] +[titan] 2025-07-10 08:13:24,897 - root - INFO - step: 84705 loss: 16.3568 memory: 44.58GiB(31.99%) tps: 83,520 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.8483 +[titan] 2025-07-10 08:13:24,897 - root - INFO - lr: 4.5591e-05 gnorm: 1.00 [18:38:21< 3:21:56] +[titan] 2025-07-10 08:13:28,849 - root - INFO - step: 84710 loss: 15.9939 memory: 44.58GiB(31.99%) tps: 82,924 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.4323 global_avg_mtp_loss: 13.5616 +[titan] 2025-07-10 08:13:28,849 - root - INFO - lr: 4.5581e-05 gnorm: 1.01 [18:38:25< 3:21:52] +[titan] 2025-07-10 08:13:32,774 - root - INFO - step: 84715 loss: 16.2279 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 08:13:32,774 - root - INFO - lr: 4.5571e-05 gnorm: 1.00 [18:38:29< 3:21:48] +[titan] 2025-07-10 08:13:36,678 - root - INFO - step: 84720 loss: 15.9469 memory: 44.58GiB(31.99%) tps: 83,952 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.4246 global_avg_mtp_loss: 13.5223 +[titan] 2025-07-10 08:13:36,678 - root - INFO - lr: 4.5561e-05 gnorm: 1.02 [18:38:33< 3:21:44] +[titan] 2025-07-10 08:13:40,621 - root - INFO - step: 84725 loss: 15.9608 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4239 global_avg_mtp_loss: 13.5368 +[titan] 2025-07-10 08:13:40,622 - root - INFO - lr: 4.5551e-05 gnorm: 0.99 [18:38:37< 3:21:40] +[titan] 2025-07-10 08:13:44,522 - root - INFO - step: 84730 loss: 16.3402 memory: 44.58GiB(31.99%) tps: 84,019 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8422 +[titan] 2025-07-10 08:13:44,522 - root - INFO - lr: 4.5541e-05 gnorm: 0.97 [18:38:41< 3:21:36] +[titan] 2025-07-10 08:13:48,438 - root - INFO - step: 84735 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7933 +[titan] 2025-07-10 08:13:48,438 - root - INFO - lr: 4.5531e-05 gnorm: 0.97 [18:38:45< 3:21:32] +[titan] 2025-07-10 08:13:52,379 - root - INFO - step: 84740 loss: 15.9296 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.4214 global_avg_mtp_loss: 13.5081 +[titan] 2025-07-10 08:13:52,379 - root - INFO - lr: 4.5521e-05 gnorm: 0.99 [18:38:49< 3:21:28] +[titan] 2025-07-10 08:13:56,347 - root - INFO - step: 84745 loss: 16.0169 memory: 44.58GiB(31.99%) tps: 82,588 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.5849 +[titan] 2025-07-10 08:13:56,347 - root - INFO - lr: 4.5511e-05 gnorm: 0.97 [18:38:53< 3:21:24] +[titan] 2025-07-10 08:13:59,472 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:14:00,269 - root - INFO - step: 84750 loss: 15.6564 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.3829 global_avg_mtp_loss: 13.2736 +[titan] 2025-07-10 08:14:00,269 - root - INFO - lr: 4.5501e-05 gnorm: 1.00 [18:38:57< 3:21:20] +[titan] 2025-07-10 08:14:04,215 - root - INFO - step: 84755 loss: 15.9676 memory: 44.58GiB(31.99%) tps: 83,052 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.4339 global_avg_mtp_loss: 13.5337 +[titan] 2025-07-10 08:14:04,215 - root - INFO - lr: 4.5491e-05 gnorm: 1.02 [18:39:01< 3:21:16] +[titan] 2025-07-10 08:14:08,133 - root - INFO - step: 84760 loss: 15.9009 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4275 global_avg_mtp_loss: 13.4733 +[titan] 2025-07-10 08:14:08,134 - root - INFO - lr: 4.5482e-05 gnorm: 1.03 [18:39:04< 3:21:12] +[titan] 2025-07-10 08:14:12,028 - root - INFO - step: 84765 loss: 16.0706 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6164 +[titan] 2025-07-10 08:14:12,028 - root - INFO - lr: 4.5472e-05 gnorm: 1.04 [18:39:08< 3:21:08] +[titan] 2025-07-10 08:14:15,965 - root - INFO - step: 84770 loss: 16.1467 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6863 +[titan] 2025-07-10 08:14:15,965 - root - INFO - lr: 4.5462e-05 gnorm: 1.06 [18:39:12< 3:21:04] +[titan] 2025-07-10 08:14:19,867 - root - INFO - step: 84775 loss: 16.3288 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8272 +[titan] 2025-07-10 08:14:19,867 - root - INFO - lr: 4.5452e-05 gnorm: 1.01 [18:39:16< 3:21:00] +[titan] 2025-07-10 08:14:23,770 - root - INFO - step: 84780 loss: 16.2969 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.8053 +[titan] 2025-07-10 08:14:23,771 - root - INFO - lr: 4.5442e-05 gnorm: 0.98 [18:39:20< 3:20:56] +[titan] 2025-07-10 08:14:27,667 - root - INFO - step: 84785 loss: 16.1526 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6852 +[titan] 2025-07-10 08:14:27,668 - root - INFO - lr: 4.5432e-05 gnorm: 1.00 [18:39:24< 3:20:52] +[titan] 2025-07-10 08:14:31,619 - root - INFO - step: 84790 loss: 16.2397 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.4953 global_avg_mtp_loss: 13.7443 +[titan] 2025-07-10 08:14:31,620 - root - INFO - lr: 4.5422e-05 gnorm: 1.00 [18:39:28< 3:20:48] +[titan] 2025-07-10 08:14:35,524 - root - INFO - step: 84795 loss: 16.2398 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7674 +[titan] 2025-07-10 08:14:35,524 - root - INFO - lr: 4.5412e-05 gnorm: 0.97 [18:39:32< 3:20:44] +[titan] 2025-07-10 08:14:38,657 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:14:39,451 - root - INFO - step: 84800 loss: 16.2547 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.7481 +[titan] 2025-07-10 08:14:39,451 - root - INFO - lr: 4.5402e-05 gnorm: 0.97 [18:39:36< 3:20:41] +[titan] 2025-07-10 08:14:43,352 - root - INFO - step: 84805 loss: 16.1288 memory: 44.58GiB(31.99%) tps: 84,010 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4536 global_avg_mtp_loss: 13.6752 +[titan] 2025-07-10 08:14:43,352 - root - INFO - lr: 4.5392e-05 gnorm: 0.97 [18:39:40< 3:20:37] +[titan] 2025-07-10 08:14:47,313 - root - INFO - step: 84810 loss: 16.0804 memory: 44.58GiB(31.99%) tps: 82,742 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.6254 +[titan] 2025-07-10 08:14:47,313 - root - INFO - lr: 4.5382e-05 gnorm: 0.99 [18:39:44< 3:20:33] +[titan] 2025-07-10 08:14:51,243 - root - INFO - step: 84815 loss: 16.3106 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.8244 +[titan] 2025-07-10 08:14:51,243 - root - INFO - lr: 4.5372e-05 gnorm: 1.01 [18:39:48< 3:20:29] +[titan] 2025-07-10 08:14:55,147 - root - INFO - step: 84820 loss: 16.2638 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7781 +[titan] 2025-07-10 08:14:55,147 - root - INFO - lr: 4.5362e-05 gnorm: 0.98 [18:39:51< 3:20:25] +[titan] 2025-07-10 08:14:59,100 - root - INFO - step: 84825 loss: 16.3006 memory: 44.58GiB(31.99%) tps: 82,886 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8060 +[titan] 2025-07-10 08:14:59,101 - root - INFO - lr: 4.5352e-05 gnorm: 1.00 [18:39:55< 3:20:21] +[titan] 2025-07-10 08:15:03,021 - root - INFO - step: 84830 loss: 16.1712 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4625 global_avg_mtp_loss: 13.7088 +[titan] 2025-07-10 08:15:03,021 - root - INFO - lr: 4.5342e-05 gnorm: 0.96 [18:39:59< 3:20:17] +[titan] 2025-07-10 08:15:06,933 - root - INFO - step: 84835 loss: 16.4731 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5511 global_avg_mtp_loss: 13.9220 +[titan] 2025-07-10 08:15:06,934 - root - INFO - lr: 4.5332e-05 gnorm: 1.10 [18:40:03< 3:20:13] +[titan] 2025-07-10 08:15:10,829 - root - INFO - step: 84840 loss: 16.0656 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4560 global_avg_mtp_loss: 13.6095 +[titan] 2025-07-10 08:15:10,829 - root - INFO - lr: 4.5323e-05 gnorm: 0.96 [18:40:07< 3:20:09] +[titan] 2025-07-10 08:15:14,767 - root - INFO - step: 84845 loss: 16.2229 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7487 +[titan] 2025-07-10 08:15:14,768 - root - INFO - lr: 4.5313e-05 gnorm: 0.94 [18:40:11< 3:20:05] +[titan] 2025-07-10 08:15:17,919 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:15:18,716 - root - INFO - step: 84850 loss: 15.9893 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.4347 global_avg_mtp_loss: 13.5546 +[titan] 2025-07-10 08:15:18,717 - root - INFO - lr: 4.5303e-05 gnorm: 0.98 [18:40:15< 3:20:01] +[titan] 2025-07-10 08:15:22,617 - root - INFO - step: 84855 loss: 16.3033 memory: 44.58GiB(31.99%) tps: 84,008 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8063 +[titan] 2025-07-10 08:15:22,618 - root - INFO - lr: 4.5293e-05 gnorm: 1.05 [18:40:19< 3:19:57] +[titan] 2025-07-10 08:15:26,554 - root - INFO - step: 84860 loss: 16.1170 memory: 44.58GiB(31.99%) tps: 83,250 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.4576 global_avg_mtp_loss: 13.6594 +[titan] 2025-07-10 08:15:26,554 - root - INFO - lr: 4.5283e-05 gnorm: 0.97 [18:40:23< 3:19:53] +[titan] 2025-07-10 08:15:30,451 - root - INFO - step: 84865 loss: 16.2000 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.7396 +[titan] 2025-07-10 08:15:30,451 - root - INFO - lr: 4.5273e-05 gnorm: 0.97 [18:40:27< 3:19:49] +[titan] 2025-07-10 08:15:34,374 - root - INFO - step: 84870 loss: 16.0237 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4318 global_avg_mtp_loss: 13.5919 +[titan] 2025-07-10 08:15:34,375 - root - INFO - lr: 4.5263e-05 gnorm: 1.00 [18:40:31< 3:19:45] +[titan] 2025-07-10 08:15:38,305 - root - INFO - step: 84875 loss: 15.8742 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4268 global_avg_mtp_loss: 13.4474 +[titan] 2025-07-10 08:15:38,305 - root - INFO - lr: 4.5253e-05 gnorm: 1.00 [18:40:35< 3:19:41] +[titan] 2025-07-10 08:15:42,229 - root - INFO - step: 84880 loss: 16.0646 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4442 global_avg_mtp_loss: 13.6204 +[titan] 2025-07-10 08:15:42,230 - root - INFO - lr: 4.5243e-05 gnorm: 1.01 [18:40:39< 3:19:37] +[titan] 2025-07-10 08:15:46,191 - root - INFO - step: 84885 loss: 16.3906 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.5026 global_avg_mtp_loss: 13.8880 +[titan] 2025-07-10 08:15:46,191 - root - INFO - lr: 4.5233e-05 gnorm: 1.04 [18:40:42< 3:19:33] +[titan] 2025-07-10 08:15:50,094 - root - INFO - step: 84890 loss: 16.3167 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.8295 +[titan] 2025-07-10 08:15:50,094 - root - INFO - lr: 4.5224e-05 gnorm: 0.96 [18:40:46< 3:19:29] +[titan] 2025-07-10 08:15:54,012 - root - INFO - step: 84895 loss: 16.0024 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4355 global_avg_mtp_loss: 13.5669 +[titan] 2025-07-10 08:15:54,012 - root - INFO - lr: 4.5214e-05 gnorm: 0.96 [18:40:50< 3:19:25] +[titan] 2025-07-10 08:15:57,123 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:15:57,911 - root - INFO - step: 84900 loss: 16.1984 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7179 +[titan] 2025-07-10 08:15:57,911 - root - INFO - lr: 4.5204e-05 gnorm: 1.00 [18:40:54< 3:19:21] +[titan] 2025-07-10 08:16:01,872 - root - INFO - step: 84905 loss: 16.0893 memory: 44.58GiB(31.99%) tps: 82,729 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6373 +[titan] 2025-07-10 08:16:01,873 - root - INFO - lr: 4.5194e-05 gnorm: 0.99 [18:40:58< 3:19:17] +[titan] 2025-07-10 08:16:05,802 - root - INFO - step: 84910 loss: 16.1004 memory: 44.58GiB(31.99%) tps: 83,398 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4503 global_avg_mtp_loss: 13.6501 +[titan] 2025-07-10 08:16:05,802 - root - INFO - lr: 4.5184e-05 gnorm: 0.98 [18:41:02< 3:19:13] +[titan] 2025-07-10 08:16:09,724 - root - INFO - step: 84915 loss: 16.5276 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.5297 global_avg_mtp_loss: 13.9979 +[titan] 2025-07-10 08:16:09,725 - root - INFO - lr: 4.5174e-05 gnorm: 1.00 [18:41:06< 3:19:09] +[titan] 2025-07-10 08:16:13,636 - root - INFO - step: 84920 loss: 16.3197 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8262 +[titan] 2025-07-10 08:16:13,636 - root - INFO - lr: 4.5164e-05 gnorm: 0.98 [18:41:10< 3:19:05] +[titan] 2025-07-10 08:16:17,536 - root - INFO - step: 84925 loss: 16.0373 memory: 44.58GiB(31.99%) tps: 84,015 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.5934 +[titan] 2025-07-10 08:16:17,537 - root - INFO - lr: 4.5154e-05 gnorm: 1.06 [18:41:14< 3:19:01] +[titan] 2025-07-10 08:16:21,455 - root - INFO - step: 84930 loss: 15.9728 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4365 global_avg_mtp_loss: 13.5363 +[titan] 2025-07-10 08:16:21,455 - root - INFO - lr: 4.5145e-05 gnorm: 0.94 [18:41:18< 3:18:57] +[titan] 2025-07-10 08:16:25,368 - root - INFO - step: 84935 loss: 16.3765 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5035 global_avg_mtp_loss: 13.8730 +[titan] 2025-07-10 08:16:25,368 - root - INFO - lr: 4.5135e-05 gnorm: 1.02 [18:41:22< 3:18:53] +[titan] 2025-07-10 08:16:29,294 - root - INFO - step: 84940 loss: 16.2564 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.05 mfu: 29.12% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7737 +[titan] 2025-07-10 08:16:29,295 - root - INFO - lr: 4.5125e-05 gnorm: 1.02 [18:41:26< 3:18:49] +[titan] 2025-07-10 08:16:33,221 - root - INFO - step: 84945 loss: 16.0750 memory: 44.58GiB(31.99%) tps: 83,465 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4465 global_avg_mtp_loss: 13.6284 +[titan] 2025-07-10 08:16:33,221 - root - INFO - lr: 4.5115e-05 gnorm: 1.01 [18:41:30< 3:18:45] +[titan] 2025-07-10 08:16:36,355 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:16:37,141 - root - INFO - step: 84950 loss: 16.0761 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4559 global_avg_mtp_loss: 13.6202 +[titan] 2025-07-10 08:16:37,142 - root - INFO - lr: 4.5105e-05 gnorm: 0.95 [18:41:33< 3:18:41] +[titan] 2025-07-10 08:16:41,042 - root - INFO - step: 84955 loss: 16.2722 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.7807 +[titan] 2025-07-10 08:16:41,042 - root - INFO - lr: 4.5095e-05 gnorm: 0.95 [18:41:37< 3:18:38] +[titan] 2025-07-10 08:16:44,970 - root - INFO - step: 84960 loss: 15.9994 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.5618 +[titan] 2025-07-10 08:16:44,971 - root - INFO - lr: 4.5086e-05 gnorm: 0.94 [18:41:41< 3:18:34] +[titan] 2025-07-10 08:16:48,941 - root - INFO - step: 84965 loss: 16.3144 memory: 44.58GiB(31.99%) tps: 82,528 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 08:16:48,942 - root - INFO - lr: 4.5076e-05 gnorm: 1.01 [18:41:45< 3:18:30] +[titan] 2025-07-10 08:16:52,866 - root - INFO - step: 84970 loss: 16.4590 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5215 global_avg_mtp_loss: 13.9375 +[titan] 2025-07-10 08:16:52,866 - root - INFO - lr: 4.5066e-05 gnorm: 0.99 [18:41:49< 3:18:26] +[titan] 2025-07-10 08:16:56,795 - root - INFO - step: 84975 loss: 16.2544 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7637 +[titan] 2025-07-10 08:16:56,795 - root - INFO - lr: 4.5056e-05 gnorm: 0.97 [18:41:53< 3:18:22] +[titan] 2025-07-10 08:17:00,746 - root - INFO - step: 84980 loss: 15.9128 memory: 44.58GiB(31.99%) tps: 82,947 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.4875 +[titan] 2025-07-10 08:17:00,746 - root - INFO - lr: 4.5046e-05 gnorm: 1.00 [18:41:57< 3:18:18] +[titan] 2025-07-10 08:17:04,648 - root - INFO - step: 84985 loss: 16.2302 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7558 +[titan] 2025-07-10 08:17:04,649 - root - INFO - lr: 4.5036e-05 gnorm: 0.95 [18:42:01< 3:18:14] +[titan] 2025-07-10 08:17:08,675 - root - INFO - step: 84990 loss: 16.0445 memory: 44.58GiB(31.99%) tps: 81,383 tflops: 280.87 mfu: 28.40% global_avg_ntp_loss: 2.4542 global_avg_mtp_loss: 13.5903 +[titan] 2025-07-10 08:17:08,676 - root - INFO - lr: 4.5027e-05 gnorm: 0.96 [18:42:05< 3:18:10] +[titan] 2025-07-10 08:17:10,414 - root - INFO - Dumping profiler traces at step 84992 +[titan] 2025-07-10 08:17:10,447 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:17:12,802 - root - INFO - step: 84995 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 79,410 tflops: 274.06 mfu: 27.71% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.9058 +[titan] 2025-07-10 08:17:12,802 - root - INFO - lr: 4.5017e-05 gnorm: 0.97 [18:42:09< 3:18:06] +[titan] 2025-07-10 08:17:15,933 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:17:16,728 - root - INFO - step: 85000 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7825 +[titan] 2025-07-10 08:17:16,728 - root - INFO - lr: 4.5007e-05 gnorm: 1.01 [18:42:13< 3:18:02] +[titan] 2025-07-10 08:17:20,647 - root - INFO - step: 85005 loss: 16.4086 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8921 +[titan] 2025-07-10 08:17:20,648 - root - INFO - lr: 4.4997e-05 gnorm: 0.99 [18:42:17< 3:17:58] +[titan] 2025-07-10 08:17:24,564 - root - INFO - step: 85010 loss: 16.5373 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 13.9935 +[titan] 2025-07-10 08:17:24,564 - root - INFO - lr: 4.4987e-05 gnorm: 0.95 [18:42:21< 3:17:54] +[titan] 2025-07-10 08:17:28,471 - root - INFO - step: 85015 loss: 15.9786 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4129 global_avg_mtp_loss: 13.5658 +[titan] 2025-07-10 08:17:28,471 - root - INFO - lr: 4.4977e-05 gnorm: 1.02 [18:42:25< 3:17:50] +[titan] 2025-07-10 08:17:32,396 - root - INFO - step: 85020 loss: 16.3774 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8806 +[titan] 2025-07-10 08:17:32,396 - root - INFO - lr: 4.4968e-05 gnorm: 1.00 [18:42:29< 3:17:46] +[titan] 2025-07-10 08:17:36,296 - root - INFO - step: 85025 loss: 16.2856 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7994 +[titan] 2025-07-10 08:17:36,297 - root - INFO - lr: 4.4958e-05 gnorm: 0.98 [18:42:33< 3:17:42] +[titan] 2025-07-10 08:17:40,195 - root - INFO - step: 85030 loss: 16.4092 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.9081 +[titan] 2025-07-10 08:17:40,195 - root - INFO - lr: 4.4948e-05 gnorm: 0.98 [18:42:36< 3:17:38] +[titan] 2025-07-10 08:17:44,124 - root - INFO - step: 85035 loss: 16.1466 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.6834 +[titan] 2025-07-10 08:17:44,124 - root - INFO - lr: 4.4938e-05 gnorm: 1.05 [18:42:40< 3:17:34] +[titan] 2025-07-10 08:17:48,060 - root - INFO - step: 85040 loss: 16.1312 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.6593 +[titan] 2025-07-10 08:17:48,060 - root - INFO - lr: 4.4928e-05 gnorm: 1.03 [18:42:44< 3:17:30] +[titan] 2025-07-10 08:17:51,987 - root - INFO - step: 85045 loss: 16.1951 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7170 +[titan] 2025-07-10 08:17:51,987 - root - INFO - lr: 4.4919e-05 gnorm: 1.01 [18:42:48< 3:17:26] +[titan] 2025-07-10 08:17:55,120 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:17:55,915 - root - INFO - step: 85050 loss: 16.6086 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 14.0524 +[titan] 2025-07-10 08:17:55,915 - root - INFO - lr: 4.4909e-05 gnorm: 1.00 [18:42:52< 3:17:22] +[titan] 2025-07-10 08:17:59,849 - root - INFO - step: 85055 loss: 16.3877 memory: 44.58GiB(31.99%) tps: 83,308 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.8850 +[titan] 2025-07-10 08:17:59,849 - root - INFO - lr: 4.4899e-05 gnorm: 0.96 [18:42:56< 3:17:18] +[titan] 2025-07-10 08:18:03,791 - root - INFO - step: 85060 loss: 16.2772 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7885 +[titan] 2025-07-10 08:18:03,791 - root - INFO - lr: 4.4889e-05 gnorm: 0.99 [18:43:00< 3:17:14] +[titan] 2025-07-10 08:18:07,696 - root - INFO - step: 85065 loss: 16.3308 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.8403 +[titan] 2025-07-10 08:18:07,697 - root - INFO - lr: 4.4880e-05 gnorm: 0.97 [18:43:04< 3:17:10] +[titan] 2025-07-10 08:18:11,603 - root - INFO - step: 85070 loss: 16.2279 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7558 +[titan] 2025-07-10 08:18:11,604 - root - INFO - lr: 4.4870e-05 gnorm: 0.97 [18:43:08< 3:17:06] +[titan] 2025-07-10 08:18:15,497 - root - INFO - step: 85075 loss: 16.1985 memory: 44.58GiB(31.99%) tps: 84,167 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7210 +[titan] 2025-07-10 08:18:15,497 - root - INFO - lr: 4.4860e-05 gnorm: 1.01 [18:43:12< 3:17:02] +[titan] 2025-07-10 08:18:19,402 - root - INFO - step: 85080 loss: 16.3499 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4905 global_avg_mtp_loss: 13.8593 +[titan] 2025-07-10 08:18:19,402 - root - INFO - lr: 4.4850e-05 gnorm: 1.02 [18:43:16< 3:16:58] +[titan] 2025-07-10 08:18:23,297 - root - INFO - step: 85085 loss: 16.1423 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6848 +[titan] 2025-07-10 08:18:23,297 - root - INFO - lr: 4.4840e-05 gnorm: 1.04 [18:43:20< 3:16:54] +[titan] 2025-07-10 08:18:27,216 - root - INFO - step: 85090 loss: 16.1203 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4626 global_avg_mtp_loss: 13.6577 +[titan] 2025-07-10 08:18:27,216 - root - INFO - lr: 4.4831e-05 gnorm: 0.97 [18:43:23< 3:16:50] +[titan] 2025-07-10 08:18:31,147 - root - INFO - step: 85095 loss: 16.3152 memory: 44.58GiB(31.99%) tps: 83,357 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 08:18:31,147 - root - INFO - lr: 4.4821e-05 gnorm: 1.00 [18:43:27< 3:16:46] +[titan] 2025-07-10 08:18:34,255 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:18:35,043 - root - INFO - step: 85100 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 84,109 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 08:18:35,044 - root - INFO - lr: 4.4811e-05 gnorm: 0.99 [18:43:31< 3:16:43] +[titan] 2025-07-10 08:18:38,952 - root - INFO - step: 85105 loss: 15.9002 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4179 global_avg_mtp_loss: 13.4822 +[titan] 2025-07-10 08:18:38,952 - root - INFO - lr: 4.4801e-05 gnorm: 1.09 [18:43:35< 3:16:39] +[titan] 2025-07-10 08:18:42,892 - root - INFO - step: 85110 loss: 16.4566 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.9405 +[titan] 2025-07-10 08:18:42,892 - root - INFO - lr: 4.4792e-05 gnorm: 1.01 [18:43:39< 3:16:35] +[titan] 2025-07-10 08:18:46,823 - root - INFO - step: 85115 loss: 16.0672 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4521 global_avg_mtp_loss: 13.6151 +[titan] 2025-07-10 08:18:46,823 - root - INFO - lr: 4.4782e-05 gnorm: 1.06 [18:43:43< 3:16:31] +[titan] 2025-07-10 08:18:50,736 - root - INFO - step: 85120 loss: 15.9959 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5541 +[titan] 2025-07-10 08:18:50,736 - root - INFO - lr: 4.4772e-05 gnorm: 0.94 [18:43:47< 3:16:27] +[titan] 2025-07-10 08:18:54,685 - root - INFO - step: 85125 loss: 16.3410 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8411 +[titan] 2025-07-10 08:18:54,685 - root - INFO - lr: 4.4762e-05 gnorm: 0.98 [18:43:51< 3:16:23] +[titan] 2025-07-10 08:18:58,614 - root - INFO - step: 85130 loss: 15.9306 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4335 global_avg_mtp_loss: 13.4971 +[titan] 2025-07-10 08:18:58,615 - root - INFO - lr: 4.4753e-05 gnorm: 1.07 [18:43:55< 3:16:19] +[titan] 2025-07-10 08:19:02,511 - root - INFO - step: 85135 loss: 16.2612 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7802 +[titan] 2025-07-10 08:19:02,511 - root - INFO - lr: 4.4743e-05 gnorm: 0.95 [18:43:59< 3:16:15] +[titan] 2025-07-10 08:19:06,412 - root - INFO - step: 85140 loss: 16.0751 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.6036 +[titan] 2025-07-10 08:19:06,413 - root - INFO - lr: 4.4733e-05 gnorm: 0.98 [18:44:03< 3:16:11] +[titan] 2025-07-10 08:19:10,321 - root - INFO - step: 85145 loss: 15.9566 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4201 global_avg_mtp_loss: 13.5365 +[titan] 2025-07-10 08:19:10,321 - root - INFO - lr: 4.4724e-05 gnorm: 1.08 [18:44:07< 3:16:07] +[titan] 2025-07-10 08:19:13,437 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:19:14,238 - root - INFO - step: 85150 loss: 16.2795 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7907 +[titan] 2025-07-10 08:19:14,238 - root - INFO - lr: 4.4714e-05 gnorm: 1.02 [18:44:11< 3:16:03] +[titan] 2025-07-10 08:19:18,141 - root - INFO - step: 85155 loss: 16.1893 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.7244 +[titan] 2025-07-10 08:19:18,141 - root - INFO - lr: 4.4704e-05 gnorm: 0.99 [18:44:14< 3:15:59] +[titan] 2025-07-10 08:19:22,073 - root - INFO - step: 85160 loss: 15.8928 memory: 44.58GiB(31.99%) tps: 83,355 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.4675 +[titan] 2025-07-10 08:19:22,073 - root - INFO - lr: 4.4694e-05 gnorm: 0.98 [18:44:18< 3:15:55] +[titan] 2025-07-10 08:19:26,020 - root - INFO - step: 85165 loss: 16.2086 memory: 44.58GiB(31.99%) tps: 83,015 tflops: 286.50 mfu: 28.97% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7428 +[titan] 2025-07-10 08:19:26,020 - root - INFO - lr: 4.4685e-05 gnorm: 0.96 [18:44:22< 3:15:51] +[titan] 2025-07-10 08:19:29,970 - root - INFO - step: 85170 loss: 16.3037 memory: 44.58GiB(31.99%) tps: 82,963 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.8141 +[titan] 2025-07-10 08:19:29,971 - root - INFO - lr: 4.4675e-05 gnorm: 0.94 [18:44:26< 3:15:47] +[titan] 2025-07-10 08:19:33,883 - root - INFO - step: 85175 loss: 16.3483 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 08:19:33,883 - root - INFO - lr: 4.4665e-05 gnorm: 0.97 [18:44:30< 3:15:43] +[titan] 2025-07-10 08:19:37,806 - root - INFO - step: 85180 loss: 16.3635 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 08:19:37,806 - root - INFO - lr: 4.4656e-05 gnorm: 1.04 [18:44:34< 3:15:39] +[titan] 2025-07-10 08:19:41,721 - root - INFO - step: 85185 loss: 16.3295 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 08:19:41,721 - root - INFO - lr: 4.4646e-05 gnorm: 1.01 [18:44:38< 3:15:35] +[titan] 2025-07-10 08:19:45,677 - root - INFO - step: 85190 loss: 16.3080 memory: 44.58GiB(31.99%) tps: 82,838 tflops: 285.89 mfu: 28.91% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8172 +[titan] 2025-07-10 08:19:45,677 - root - INFO - lr: 4.4636e-05 gnorm: 0.98 [18:44:42< 3:15:31] +[titan] 2025-07-10 08:19:49,609 - root - INFO - step: 85195 loss: 16.2823 memory: 44.58GiB(31.99%) tps: 83,339 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.7923 +[titan] 2025-07-10 08:19:49,609 - root - INFO - lr: 4.4626e-05 gnorm: 1.02 [18:44:46< 3:15:27] +[titan] 2025-07-10 08:19:52,737 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:19:53,529 - root - INFO - step: 85200 loss: 16.1508 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4406 global_avg_mtp_loss: 13.7103 +[titan] 2025-07-10 08:19:53,529 - root - INFO - lr: 4.4617e-05 gnorm: 0.95 [18:44:50< 3:15:23] +[titan] 2025-07-10 08:19:57,451 - root - INFO - step: 85205 loss: 16.0963 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6343 +[titan] 2025-07-10 08:19:57,451 - root - INFO - lr: 4.4607e-05 gnorm: 1.07 [18:44:54< 3:15:19] +[titan] 2025-07-10 08:20:01,376 - root - INFO - step: 85210 loss: 16.1627 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.7134 +[titan] 2025-07-10 08:20:01,376 - root - INFO - lr: 4.4597e-05 gnorm: 0.99 [18:44:58< 3:15:15] +[titan] 2025-07-10 08:20:05,334 - root - INFO - step: 85215 loss: 16.3013 memory: 44.58GiB(31.99%) tps: 82,810 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 08:20:05,334 - root - INFO - lr: 4.4588e-05 gnorm: 0.97 [18:45:02< 3:15:11] +[titan] 2025-07-10 08:20:09,288 - root - INFO - step: 85220 loss: 16.0831 memory: 44.58GiB(31.99%) tps: 82,861 tflops: 285.97 mfu: 28.91% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6191 +[titan] 2025-07-10 08:20:09,289 - root - INFO - lr: 4.4578e-05 gnorm: 1.00 [18:45:06< 3:15:07] +[titan] 2025-07-10 08:20:13,220 - root - INFO - step: 85225 loss: 16.2544 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7728 +[titan] 2025-07-10 08:20:13,220 - root - INFO - lr: 4.4568e-05 gnorm: 1.00 [18:45:09< 3:15:03] +[titan] 2025-07-10 08:20:17,155 - root - INFO - step: 85230 loss: 16.1095 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6558 +[titan] 2025-07-10 08:20:17,156 - root - INFO - lr: 4.4559e-05 gnorm: 1.02 [18:45:13< 3:14:59] +[titan] 2025-07-10 08:20:21,085 - root - INFO - step: 85235 loss: 16.0708 memory: 44.58GiB(31.99%) tps: 83,405 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4507 global_avg_mtp_loss: 13.6201 +[titan] 2025-07-10 08:20:21,085 - root - INFO - lr: 4.4549e-05 gnorm: 0.99 [18:45:17< 3:14:55] +[titan] 2025-07-10 08:20:25,073 - root - INFO - step: 85240 loss: 16.2701 memory: 44.58GiB(31.99%) tps: 82,161 tflops: 283.55 mfu: 28.67% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.7859 +[titan] 2025-07-10 08:20:25,074 - root - INFO - lr: 4.4539e-05 gnorm: 1.00 [18:45:21< 3:14:51] +[titan] 2025-07-10 08:20:29,020 - root - INFO - step: 85245 loss: 16.2747 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7900 +[titan] 2025-07-10 08:20:29,021 - root - INFO - lr: 4.4530e-05 gnorm: 0.98 [18:45:25< 3:14:47] +[titan] 2025-07-10 08:20:32,146 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:20:32,937 - root - INFO - step: 85250 loss: 16.1509 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.6801 +[titan] 2025-07-10 08:20:32,938 - root - INFO - lr: 4.4520e-05 gnorm: 0.96 [18:45:29< 3:14:44] +[titan] 2025-07-10 08:20:36,861 - root - INFO - step: 85255 loss: 15.8626 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4184 global_avg_mtp_loss: 13.4442 +[titan] 2025-07-10 08:20:36,861 - root - INFO - lr: 4.4510e-05 gnorm: 1.01 [18:45:33< 3:14:40] +[titan] 2025-07-10 08:20:40,802 - root - INFO - step: 85260 loss: 16.0085 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4442 global_avg_mtp_loss: 13.5643 +[titan] 2025-07-10 08:20:40,802 - root - INFO - lr: 4.4501e-05 gnorm: 0.95 [18:45:37< 3:14:36] +[titan] 2025-07-10 08:20:44,714 - root - INFO - step: 85265 loss: 16.1107 memory: 44.58GiB(31.99%) tps: 83,758 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6480 +[titan] 2025-07-10 08:20:44,715 - root - INFO - lr: 4.4491e-05 gnorm: 1.02 [18:45:41< 3:14:32] +[titan] 2025-07-10 08:20:48,622 - root - INFO - step: 85270 loss: 16.3822 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8752 +[titan] 2025-07-10 08:20:48,623 - root - INFO - lr: 4.4481e-05 gnorm: 0.99 [18:45:45< 3:14:28] +[titan] 2025-07-10 08:20:52,605 - root - INFO - step: 85275 loss: 16.0567 memory: 44.58GiB(31.99%) tps: 82,291 tflops: 284.00 mfu: 28.72% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.6094 +[titan] 2025-07-10 08:20:52,605 - root - INFO - lr: 4.4472e-05 gnorm: 1.04 [18:45:49< 3:14:24] +[titan] 2025-07-10 08:20:56,532 - root - INFO - step: 85280 loss: 16.0896 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.6312 +[titan] 2025-07-10 08:20:56,532 - root - INFO - lr: 4.4462e-05 gnorm: 0.97 [18:45:53< 3:14:20] +[titan] 2025-07-10 08:21:00,435 - root - INFO - step: 85285 loss: 16.2354 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.7465 +[titan] 2025-07-10 08:21:00,435 - root - INFO - lr: 4.4452e-05 gnorm: 1.05 [18:45:57< 3:14:16] +[titan] 2025-07-10 08:21:04,347 - root - INFO - step: 85290 loss: 16.2991 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8035 +[titan] 2025-07-10 08:21:04,348 - root - INFO - lr: 4.4443e-05 gnorm: 1.07 [18:46:01< 3:14:12] +[titan] 2025-07-10 08:21:08,259 - root - INFO - step: 85295 loss: 16.1595 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 08:21:08,260 - root - INFO - lr: 4.4433e-05 gnorm: 1.03 [18:46:05< 3:14:08] +[titan] 2025-07-10 08:21:11,409 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:21:12,203 - root - INFO - step: 85300 loss: 16.2987 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8038 +[titan] 2025-07-10 08:21:12,204 - root - INFO - lr: 4.4423e-05 gnorm: 0.98 [18:46:08< 3:14:04] +[titan] 2025-07-10 08:21:16,103 - root - INFO - step: 85305 loss: 16.2850 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7970 +[titan] 2025-07-10 08:21:16,104 - root - INFO - lr: 4.4414e-05 gnorm: 0.98 [18:46:12< 3:14:00] +[titan] 2025-07-10 08:21:20,038 - root - INFO - step: 85310 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7985 +[titan] 2025-07-10 08:21:20,038 - root - INFO - lr: 4.4404e-05 gnorm: 1.00 [18:46:16< 3:13:56] +[titan] 2025-07-10 08:21:23,958 - root - INFO - step: 85315 loss: 16.3028 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8036 +[titan] 2025-07-10 08:21:23,958 - root - INFO - lr: 4.4395e-05 gnorm: 0.97 [18:46:20< 3:13:52] +[titan] 2025-07-10 08:21:27,855 - root - INFO - step: 85320 loss: 16.0279 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4430 global_avg_mtp_loss: 13.5849 +[titan] 2025-07-10 08:21:27,855 - root - INFO - lr: 4.4385e-05 gnorm: 1.01 [18:46:24< 3:13:48] +[titan] 2025-07-10 08:21:31,792 - root - INFO - step: 85325 loss: 16.3131 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.8219 +[titan] 2025-07-10 08:21:31,792 - root - INFO - lr: 4.4375e-05 gnorm: 1.01 [18:46:28< 3:13:44] +[titan] 2025-07-10 08:21:35,703 - root - INFO - step: 85330 loss: 16.3383 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.8153 +[titan] 2025-07-10 08:21:35,704 - root - INFO - lr: 4.4366e-05 gnorm: 0.97 [18:46:32< 3:13:40] +[titan] 2025-07-10 08:21:39,624 - root - INFO - step: 85335 loss: 16.1851 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7099 +[titan] 2025-07-10 08:21:39,625 - root - INFO - lr: 4.4356e-05 gnorm: 0.97 [18:46:36< 3:13:36] +[titan] 2025-07-10 08:21:43,523 - root - INFO - step: 85340 loss: 16.1639 memory: 44.58GiB(31.99%) tps: 84,066 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6942 +[titan] 2025-07-10 08:21:43,523 - root - INFO - lr: 4.4346e-05 gnorm: 1.02 [18:46:40< 3:13:32] +[titan] 2025-07-10 08:21:47,426 - root - INFO - step: 85345 loss: 16.1273 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4531 global_avg_mtp_loss: 13.6742 +[titan] 2025-07-10 08:21:47,426 - root - INFO - lr: 4.4337e-05 gnorm: 1.01 [18:46:44< 3:13:28] +[titan] 2025-07-10 08:21:50,538 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:21:51,327 - root - INFO - step: 85350 loss: 15.9748 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4332 global_avg_mtp_loss: 13.5415 +[titan] 2025-07-10 08:21:51,327 - root - INFO - lr: 4.4327e-05 gnorm: 1.00 [18:46:48< 3:13:24] +[titan] 2025-07-10 08:21:55,228 - root - INFO - step: 85355 loss: 16.4421 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5166 global_avg_mtp_loss: 13.9255 +[titan] 2025-07-10 08:21:55,228 - root - INFO - lr: 4.4318e-05 gnorm: 1.01 [18:46:51< 3:13:20] +[titan] 2025-07-10 08:21:59,141 - root - INFO - step: 85360 loss: 16.3232 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8290 +[titan] 2025-07-10 08:21:59,141 - root - INFO - lr: 4.4308e-05 gnorm: 0.99 [18:46:55< 3:13:16] +[titan] 2025-07-10 08:22:03,049 - root - INFO - step: 85365 loss: 16.2938 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.7989 +[titan] 2025-07-10 08:22:03,049 - root - INFO - lr: 4.4298e-05 gnorm: 0.97 [18:46:59< 3:13:12] +[titan] 2025-07-10 08:22:06,962 - root - INFO - step: 85370 loss: 16.0251 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.5794 +[titan] 2025-07-10 08:22:06,963 - root - INFO - lr: 4.4289e-05 gnorm: 0.96 [18:47:03< 3:13:08] +[titan] 2025-07-10 08:22:10,909 - root - INFO - step: 85375 loss: 16.4101 memory: 44.58GiB(31.99%) tps: 83,032 tflops: 286.56 mfu: 28.97% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.9014 +[titan] 2025-07-10 08:22:10,910 - root - INFO - lr: 4.4279e-05 gnorm: 1.04 [18:47:07< 3:13:04] +[titan] 2025-07-10 08:22:14,831 - root - INFO - step: 85380 loss: 16.1347 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 08:22:14,832 - root - INFO - lr: 4.4270e-05 gnorm: 1.03 [18:47:11< 3:13:00] +[titan] 2025-07-10 08:22:18,756 - root - INFO - step: 85385 loss: 15.9395 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4318 global_avg_mtp_loss: 13.5077 +[titan] 2025-07-10 08:22:18,757 - root - INFO - lr: 4.4260e-05 gnorm: 1.01 [18:47:15< 3:12:56] +[titan] 2025-07-10 08:22:22,661 - root - INFO - step: 85390 loss: 16.0137 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4464 global_avg_mtp_loss: 13.5673 +[titan] 2025-07-10 08:22:22,661 - root - INFO - lr: 4.4251e-05 gnorm: 1.04 [18:47:19< 3:12:52] +[titan] 2025-07-10 08:22:26,590 - root - INFO - step: 85395 loss: 16.1455 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 08:22:26,590 - root - INFO - lr: 4.4241e-05 gnorm: 1.04 [18:47:23< 3:12:48] +[titan] 2025-07-10 08:22:29,739 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:22:30,529 - root - INFO - step: 85400 loss: 16.1892 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7154 +[titan] 2025-07-10 08:22:30,529 - root - INFO - lr: 4.4231e-05 gnorm: 1.00 [18:47:27< 3:12:44] +[titan] 2025-07-10 08:22:34,436 - root - INFO - step: 85405 loss: 16.2461 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7674 +[titan] 2025-07-10 08:22:34,436 - root - INFO - lr: 4.4222e-05 gnorm: 0.99 [18:47:31< 3:12:41] +[titan] 2025-07-10 08:22:38,346 - root - INFO - step: 85410 loss: 16.1827 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.23 mfu: 29.25% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7140 +[titan] 2025-07-10 08:22:38,346 - root - INFO - lr: 4.4212e-05 gnorm: 1.03 [18:47:35< 3:12:37] +[titan] 2025-07-10 08:22:42,262 - root - INFO - step: 85415 loss: 16.3446 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 08:22:42,262 - root - INFO - lr: 4.4203e-05 gnorm: 0.98 [18:47:38< 3:12:33] +[titan] 2025-07-10 08:22:46,217 - root - INFO - step: 85420 loss: 16.2976 memory: 44.58GiB(31.99%) tps: 82,857 tflops: 285.95 mfu: 28.91% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.7979 +[titan] 2025-07-10 08:22:46,218 - root - INFO - lr: 4.4193e-05 gnorm: 1.04 [18:47:42< 3:12:29] +[titan] 2025-07-10 08:22:50,140 - root - INFO - step: 85425 loss: 16.1482 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6821 +[titan] 2025-07-10 08:22:50,141 - root - INFO - lr: 4.4184e-05 gnorm: 0.99 [18:47:46< 3:12:25] +[titan] 2025-07-10 08:22:54,067 - root - INFO - step: 85430 loss: 15.9451 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4345 global_avg_mtp_loss: 13.5106 +[titan] 2025-07-10 08:22:54,067 - root - INFO - lr: 4.4174e-05 gnorm: 0.99 [18:47:50< 3:12:21] +[titan] 2025-07-10 08:22:57,967 - root - INFO - step: 85435 loss: 16.3905 memory: 44.58GiB(31.99%) tps: 84,021 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5046 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 08:22:57,967 - root - INFO - lr: 4.4164e-05 gnorm: 1.00 [18:47:54< 3:12:17] +[titan] 2025-07-10 08:23:01,912 - root - INFO - step: 85440 loss: 16.1001 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6416 +[titan] 2025-07-10 08:23:01,912 - root - INFO - lr: 4.4155e-05 gnorm: 0.97 [18:47:58< 3:12:13] +[titan] 2025-07-10 08:23:05,823 - root - INFO - step: 85445 loss: 16.1912 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7188 +[titan] 2025-07-10 08:23:05,823 - root - INFO - lr: 4.4145e-05 gnorm: 0.97 [18:48:02< 3:12:09] +[titan] 2025-07-10 08:23:08,940 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:23:09,725 - root - INFO - step: 85450 loss: 16.1949 memory: 44.58GiB(31.99%) tps: 83,987 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7234 +[titan] 2025-07-10 08:23:09,725 - root - INFO - lr: 4.4136e-05 gnorm: 0.99 [18:48:06< 3:12:05] +[titan] 2025-07-10 08:23:13,648 - root - INFO - step: 85455 loss: 16.0206 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4395 global_avg_mtp_loss: 13.5811 +[titan] 2025-07-10 08:23:13,648 - root - INFO - lr: 4.4126e-05 gnorm: 1.04 [18:48:10< 3:12:01] +[titan] 2025-07-10 08:23:17,585 - root - INFO - step: 85460 loss: 16.0768 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4481 global_avg_mtp_loss: 13.6287 +[titan] 2025-07-10 08:23:17,585 - root - INFO - lr: 4.4117e-05 gnorm: 1.00 [18:48:14< 3:11:57] +[titan] 2025-07-10 08:23:21,508 - root - INFO - step: 85465 loss: 16.2081 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.7401 +[titan] 2025-07-10 08:23:21,508 - root - INFO - lr: 4.4107e-05 gnorm: 1.01 [18:48:18< 3:11:53] +[titan] 2025-07-10 08:23:25,443 - root - INFO - step: 85470 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4938 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-10 08:23:25,444 - root - INFO - lr: 4.4098e-05 gnorm: 1.01 [18:48:22< 3:11:49] +[titan] 2025-07-10 08:23:29,350 - root - INFO - step: 85475 loss: 16.2452 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7613 +[titan] 2025-07-10 08:23:29,350 - root - INFO - lr: 4.4088e-05 gnorm: 0.98 [18:48:26< 3:11:45] +[titan] 2025-07-10 08:23:33,244 - root - INFO - step: 85480 loss: 16.2224 memory: 44.58GiB(31.99%) tps: 84,148 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7396 +[titan] 2025-07-10 08:23:33,245 - root - INFO - lr: 4.4079e-05 gnorm: 0.98 [18:48:29< 3:11:41] +[titan] 2025-07-10 08:23:37,161 - root - INFO - step: 85485 loss: 16.1772 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4563 global_avg_mtp_loss: 13.7209 +[titan] 2025-07-10 08:23:37,161 - root - INFO - lr: 4.4069e-05 gnorm: 1.04 [18:48:33< 3:11:37] +[titan] 2025-07-10 08:23:41,076 - root - INFO - step: 85490 loss: 16.1922 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7264 +[titan] 2025-07-10 08:23:41,077 - root - INFO - lr: 4.4060e-05 gnorm: 0.94 [18:48:37< 3:11:33] +[titan] 2025-07-10 08:23:44,983 - root - INFO - step: 85495 loss: 16.2725 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7852 +[titan] 2025-07-10 08:23:44,983 - root - INFO - lr: 4.4050e-05 gnorm: 0.98 [18:48:41< 3:11:29] +[titan] 2025-07-10 08:23:48,095 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:23:48,884 - root - INFO - step: 85500 loss: 16.1853 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7162 +[titan] 2025-07-10 08:23:48,885 - root - INFO - lr: 4.4041e-05 gnorm: 1.00 [18:48:45< 3:11:25] +[titan] 2025-07-10 08:23:52,285 - root - INFO - Dumping profiler traces at step 85504 +[titan] 2025-07-10 08:23:52,318 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:23:53,111 - root - INFO - step: 85505 loss: 16.2302 memory: 44.58GiB(31.99%) tps: 77,533 tflops: 267.58 mfu: 27.06% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7515 +[titan] 2025-07-10 08:23:53,111 - root - INFO - lr: 4.4031e-05 gnorm: 1.00 [18:48:49< 3:11:21] +[titan] 2025-07-10 08:23:57,070 - root - INFO - step: 85510 loss: 15.6430 memory: 44.58GiB(31.99%) tps: 82,775 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.3757 global_avg_mtp_loss: 13.2673 +[titan] 2025-07-10 08:23:57,070 - root - INFO - lr: 4.4022e-05 gnorm: 0.95 [18:48:53< 3:11:17] +[titan] 2025-07-10 08:24:01,013 - root - INFO - step: 85515 loss: 15.8683 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4231 global_avg_mtp_loss: 13.4452 +[titan] 2025-07-10 08:24:01,014 - root - INFO - lr: 4.4012e-05 gnorm: 1.02 [18:48:57< 3:11:13] +[titan] 2025-07-10 08:24:04,989 - root - INFO - step: 85520 loss: 16.4237 memory: 44.58GiB(31.99%) tps: 82,423 tflops: 284.45 mfu: 28.76% global_avg_ntp_loss: 2.5092 global_avg_mtp_loss: 13.9145 +[titan] 2025-07-10 08:24:04,990 - root - INFO - lr: 4.4003e-05 gnorm: 1.01 [18:49:01< 3:11:09] +[titan] 2025-07-10 08:24:08,910 - root - INFO - step: 85525 loss: 15.9318 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4188 global_avg_mtp_loss: 13.5131 +[titan] 2025-07-10 08:24:08,910 - root - INFO - lr: 4.3993e-05 gnorm: 0.94 [18:49:05< 3:11:05] +[titan] 2025-07-10 08:24:12,828 - root - INFO - step: 85530 loss: 16.2725 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7947 +[titan] 2025-07-10 08:24:12,828 - root - INFO - lr: 4.3984e-05 gnorm: 1.03 [18:49:09< 3:11:01] +[titan] 2025-07-10 08:24:16,752 - root - INFO - step: 85535 loss: 16.0404 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4470 global_avg_mtp_loss: 13.5934 +[titan] 2025-07-10 08:24:16,753 - root - INFO - lr: 4.3974e-05 gnorm: 0.98 [18:49:13< 3:10:57] +[titan] 2025-07-10 08:24:20,672 - root - INFO - step: 85540 loss: 16.1435 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.6724 +[titan] 2025-07-10 08:24:20,672 - root - INFO - lr: 4.3965e-05 gnorm: 0.98 [18:49:17< 3:10:53] +[titan] 2025-07-10 08:24:24,619 - root - INFO - step: 85545 loss: 16.4276 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5185 global_avg_mtp_loss: 13.9092 +[titan] 2025-07-10 08:24:24,619 - root - INFO - lr: 4.3955e-05 gnorm: 0.92 [18:49:21< 3:10:49] +[titan] 2025-07-10 08:24:27,750 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:24:28,544 - root - INFO - step: 85550 loss: 16.2104 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7273 +[titan] 2025-07-10 08:24:28,545 - root - INFO - lr: 4.3946e-05 gnorm: 0.99 [18:49:25< 3:10:46] +[titan] 2025-07-10 08:24:32,464 - root - INFO - step: 85555 loss: 16.2122 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7398 +[titan] 2025-07-10 08:24:32,465 - root - INFO - lr: 4.3936e-05 gnorm: 0.95 [18:49:29< 3:10:42] +[titan] 2025-07-10 08:24:36,450 - root - INFO - step: 85560 loss: 16.0738 memory: 44.58GiB(31.99%) tps: 82,228 tflops: 283.78 mfu: 28.69% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6185 +[titan] 2025-07-10 08:24:36,450 - root - INFO - lr: 4.3927e-05 gnorm: 1.06 [18:49:33< 3:10:38] +[titan] 2025-07-10 08:24:40,445 - root - INFO - step: 85565 loss: 16.2356 memory: 44.58GiB(31.99%) tps: 82,029 tflops: 283.10 mfu: 28.62% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7493 +[titan] 2025-07-10 08:24:40,445 - root - INFO - lr: 4.3917e-05 gnorm: 1.00 [18:49:37< 3:10:34] +[titan] 2025-07-10 08:24:44,365 - root - INFO - step: 85570 loss: 16.1603 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6951 +[titan] 2025-07-10 08:24:44,366 - root - INFO - lr: 4.3908e-05 gnorm: 0.96 [18:49:41< 3:10:30] +[titan] 2025-07-10 08:24:48,296 - root - INFO - step: 85575 loss: 15.8375 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4106 global_avg_mtp_loss: 13.4269 +[titan] 2025-07-10 08:24:48,296 - root - INFO - lr: 4.3898e-05 gnorm: 1.00 [18:49:45< 3:10:26] +[titan] 2025-07-10 08:24:52,205 - root - INFO - step: 85580 loss: 15.8476 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4045 global_avg_mtp_loss: 13.4431 +[titan] 2025-07-10 08:24:52,206 - root - INFO - lr: 4.3889e-05 gnorm: 0.98 [18:49:48< 3:10:22] +[titan] 2025-07-10 08:24:56,119 - root - INFO - step: 85585 loss: 16.3998 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8964 +[titan] 2025-07-10 08:24:56,119 - root - INFO - lr: 4.3879e-05 gnorm: 1.07 [18:49:52< 3:10:18] +[titan] 2025-07-10 08:25:00,050 - root - INFO - step: 85590 loss: 16.1763 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.7176 +[titan] 2025-07-10 08:25:00,051 - root - INFO - lr: 4.3870e-05 gnorm: 0.99 [18:49:56< 3:10:14] +[titan] 2025-07-10 08:25:03,969 - root - INFO - step: 85595 loss: 16.2935 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-10 08:25:03,970 - root - INFO - lr: 4.3860e-05 gnorm: 0.98 [18:50:00< 3:10:10] +[titan] 2025-07-10 08:25:07,104 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:25:07,897 - root - INFO - step: 85600 loss: 16.1702 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.7074 +[titan] 2025-07-10 08:25:07,897 - root - INFO - lr: 4.3851e-05 gnorm: 1.00 [18:50:04< 3:10:06] +[titan] 2025-07-10 08:25:11,851 - root - INFO - step: 85605 loss: 16.2117 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.7234 +[titan] 2025-07-10 08:25:11,851 - root - INFO - lr: 4.3841e-05 gnorm: 1.02 [18:50:08< 3:10:02] +[titan] 2025-07-10 08:25:15,775 - root - INFO - step: 85610 loss: 16.1984 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.7341 +[titan] 2025-07-10 08:25:15,775 - root - INFO - lr: 4.3832e-05 gnorm: 1.00 [18:50:12< 3:09:58] +[titan] 2025-07-10 08:25:19,683 - root - INFO - step: 85615 loss: 16.4540 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5247 global_avg_mtp_loss: 13.9293 +[titan] 2025-07-10 08:25:19,683 - root - INFO - lr: 4.3823e-05 gnorm: 0.99 [18:50:16< 3:09:54] +[titan] 2025-07-10 08:25:23,611 - root - INFO - step: 85620 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4622 global_avg_mtp_loss: 13.6598 +[titan] 2025-07-10 08:25:23,612 - root - INFO - lr: 4.3813e-05 gnorm: 0.96 [18:50:20< 3:09:50] +[titan] 2025-07-10 08:25:27,536 - root - INFO - step: 85625 loss: 15.9271 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.4959 +[titan] 2025-07-10 08:25:27,536 - root - INFO - lr: 4.3804e-05 gnorm: 1.01 [18:50:24< 3:09:46] +[titan] 2025-07-10 08:25:31,450 - root - INFO - step: 85630 loss: 16.4318 memory: 44.58GiB(31.99%) tps: 83,729 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.9216 +[titan] 2025-07-10 08:25:31,450 - root - INFO - lr: 4.3794e-05 gnorm: 0.96 [18:50:28< 3:09:42] +[titan] 2025-07-10 08:25:35,448 - root - INFO - step: 85635 loss: 16.0331 memory: 44.58GiB(31.99%) tps: 81,970 tflops: 282.89 mfu: 28.60% global_avg_ntp_loss: 2.4398 global_avg_mtp_loss: 13.5933 +[titan] 2025-07-10 08:25:35,448 - root - INFO - lr: 4.3785e-05 gnorm: 0.99 [18:50:32< 3:09:38] +[titan] 2025-07-10 08:25:39,392 - root - INFO - step: 85640 loss: 16.3398 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.8425 +[titan] 2025-07-10 08:25:39,392 - root - INFO - lr: 4.3775e-05 gnorm: 0.99 [18:50:36< 3:09:34] +[titan] 2025-07-10 08:25:43,313 - root - INFO - step: 85645 loss: 16.5695 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5366 global_avg_mtp_loss: 14.0328 +[titan] 2025-07-10 08:25:43,313 - root - INFO - lr: 4.3766e-05 gnorm: 0.96 [18:50:40< 3:09:30] +[titan] 2025-07-10 08:25:46,437 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:25:47,223 - root - INFO - step: 85650 loss: 16.2087 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7405 +[titan] 2025-07-10 08:25:47,224 - root - INFO - lr: 4.3757e-05 gnorm: 0.97 [18:50:43< 3:09:26] +[titan] 2025-07-10 08:25:51,119 - root - INFO - step: 85655 loss: 16.1717 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.7208 +[titan] 2025-07-10 08:25:51,119 - root - INFO - lr: 4.3747e-05 gnorm: 0.97 [18:50:47< 3:09:22] +[titan] 2025-07-10 08:25:55,015 - root - INFO - step: 85660 loss: 16.1418 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6676 +[titan] 2025-07-10 08:25:55,016 - root - INFO - lr: 4.3738e-05 gnorm: 1.01 [18:50:51< 3:09:18] +[titan] 2025-07-10 08:25:58,913 - root - INFO - step: 85665 loss: 16.2580 memory: 44.58GiB(31.99%) tps: 84,085 tflops: 290.19 mfu: 29.34% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7799 +[titan] 2025-07-10 08:25:58,913 - root - INFO - lr: 4.3728e-05 gnorm: 1.02 [18:50:55< 3:09:14] +[titan] 2025-07-10 08:26:02,833 - root - INFO - step: 85670 loss: 16.2957 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8045 +[titan] 2025-07-10 08:26:02,833 - root - INFO - lr: 4.3719e-05 gnorm: 0.98 [18:50:59< 3:09:10] +[titan] 2025-07-10 08:26:06,736 - root - INFO - step: 85675 loss: 16.2911 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.8048 +[titan] 2025-07-10 08:26:06,736 - root - INFO - lr: 4.3710e-05 gnorm: 1.00 [18:51:03< 3:09:06] +[titan] 2025-07-10 08:26:10,679 - root - INFO - step: 85680 loss: 15.9843 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.5418 +[titan] 2025-07-10 08:26:10,680 - root - INFO - lr: 4.3700e-05 gnorm: 0.98 [18:51:07< 3:09:02] +[titan] 2025-07-10 08:26:14,570 - root - INFO - step: 85685 loss: 16.3156 memory: 44.58GiB(31.99%) tps: 84,238 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8199 +[titan] 2025-07-10 08:26:14,570 - root - INFO - lr: 4.3691e-05 gnorm: 0.98 [18:51:11< 3:08:58] +[titan] 2025-07-10 08:26:18,486 - root - INFO - step: 85690 loss: 16.2438 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7585 +[titan] 2025-07-10 08:26:18,486 - root - INFO - lr: 4.3681e-05 gnorm: 0.95 [18:51:15< 3:08:54] +[titan] 2025-07-10 08:26:22,412 - root - INFO - step: 85695 loss: 16.2239 memory: 44.58GiB(31.99%) tps: 83,454 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7423 +[titan] 2025-07-10 08:26:22,413 - root - INFO - lr: 4.3672e-05 gnorm: 1.02 [18:51:19< 3:08:51] +[titan] 2025-07-10 08:26:25,528 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:26:26,338 - root - INFO - step: 85700 loss: 16.2060 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7317 +[titan] 2025-07-10 08:26:26,338 - root - INFO - lr: 4.3663e-05 gnorm: 1.01 [18:51:23< 3:08:47] +[titan] 2025-07-10 08:26:30,259 - root - INFO - step: 85705 loss: 16.0723 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6075 +[titan] 2025-07-10 08:26:30,259 - root - INFO - lr: 4.3653e-05 gnorm: 1.05 [18:51:26< 3:08:43] +[titan] 2025-07-10 08:26:34,201 - root - INFO - step: 85710 loss: 16.3075 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8130 +[titan] 2025-07-10 08:26:34,202 - root - INFO - lr: 4.3644e-05 gnorm: 0.97 [18:51:30< 3:08:39] +[titan] 2025-07-10 08:26:38,134 - root - INFO - step: 85715 loss: 16.3777 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5052 global_avg_mtp_loss: 13.8725 +[titan] 2025-07-10 08:26:38,134 - root - INFO - lr: 4.3634e-05 gnorm: 1.02 [18:51:34< 3:08:35] +[titan] 2025-07-10 08:26:42,042 - root - INFO - step: 85720 loss: 16.4174 memory: 44.58GiB(31.99%) tps: 83,846 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9045 +[titan] 2025-07-10 08:26:42,043 - root - INFO - lr: 4.3625e-05 gnorm: 1.11 [18:51:38< 3:08:31] +[titan] 2025-07-10 08:26:45,954 - root - INFO - step: 85725 loss: 15.9747 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4417 global_avg_mtp_loss: 13.5330 +[titan] 2025-07-10 08:26:45,954 - root - INFO - lr: 4.3616e-05 gnorm: 1.00 [18:51:42< 3:08:27] +[titan] 2025-07-10 08:26:49,858 - root - INFO - step: 85730 loss: 16.4009 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8896 +[titan] 2025-07-10 08:26:49,858 - root - INFO - lr: 4.3606e-05 gnorm: 1.01 [18:51:46< 3:08:23] +[titan] 2025-07-10 08:26:53,812 - root - INFO - step: 85735 loss: 16.2321 memory: 44.58GiB(31.99%) tps: 82,874 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7507 +[titan] 2025-07-10 08:26:53,812 - root - INFO - lr: 4.3597e-05 gnorm: 0.99 [18:51:50< 3:08:19] +[titan] 2025-07-10 08:26:57,729 - root - INFO - step: 85740 loss: 16.2249 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7556 +[titan] 2025-07-10 08:26:57,729 - root - INFO - lr: 4.3588e-05 gnorm: 1.03 [18:51:54< 3:08:15] +[titan] 2025-07-10 08:27:01,635 - root - INFO - step: 85745 loss: 16.1465 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4526 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 08:27:01,635 - root - INFO - lr: 4.3578e-05 gnorm: 1.01 [18:51:58< 3:08:11] +[titan] 2025-07-10 08:27:04,769 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:27:05,562 - root - INFO - step: 85750 loss: 16.3128 memory: 44.58GiB(31.99%) tps: 83,458 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8119 +[titan] 2025-07-10 08:27:05,562 - root - INFO - lr: 4.3569e-05 gnorm: 1.03 [18:52:02< 3:08:07] +[titan] 2025-07-10 08:27:09,472 - root - INFO - step: 85755 loss: 16.1732 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.7119 +[titan] 2025-07-10 08:27:09,472 - root - INFO - lr: 4.3559e-05 gnorm: 1.00 [18:52:06< 3:08:03] +[titan] 2025-07-10 08:27:13,404 - root - INFO - step: 85760 loss: 16.4278 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5260 global_avg_mtp_loss: 13.9019 +[titan] 2025-07-10 08:27:13,404 - root - INFO - lr: 4.3550e-05 gnorm: 1.00 [18:52:10< 3:07:59] +[titan] 2025-07-10 08:27:17,313 - root - INFO - step: 85765 loss: 16.3575 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8619 +[titan] 2025-07-10 08:27:17,313 - root - INFO - lr: 4.3541e-05 gnorm: 0.96 [18:52:14< 3:07:55] +[titan] 2025-07-10 08:27:21,242 - root - INFO - step: 85770 loss: 15.9841 memory: 44.58GiB(31.99%) tps: 83,406 tflops: 287.85 mfu: 29.10% global_avg_ntp_loss: 2.4343 global_avg_mtp_loss: 13.5498 +[titan] 2025-07-10 08:27:21,242 - root - INFO - lr: 4.3531e-05 gnorm: 1.05 [18:52:17< 3:07:51] +[titan] 2025-07-10 08:27:25,163 - root - INFO - step: 85775 loss: 16.1155 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4512 global_avg_mtp_loss: 13.6643 +[titan] 2025-07-10 08:27:25,164 - root - INFO - lr: 4.3522e-05 gnorm: 1.00 [18:52:21< 3:07:47] +[titan] 2025-07-10 08:27:29,101 - root - INFO - step: 85780 loss: 16.2090 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7348 +[titan] 2025-07-10 08:27:29,101 - root - INFO - lr: 4.3513e-05 gnorm: 0.97 [18:52:25< 3:07:43] +[titan] 2025-07-10 08:27:33,010 - root - INFO - step: 85785 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.6841 +[titan] 2025-07-10 08:27:33,010 - root - INFO - lr: 4.3503e-05 gnorm: 0.98 [18:52:29< 3:07:39] +[titan] 2025-07-10 08:27:36,943 - root - INFO - step: 85790 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.7942 +[titan] 2025-07-10 08:27:36,943 - root - INFO - lr: 4.3494e-05 gnorm: 0.98 [18:52:33< 3:07:35] +[titan] 2025-07-10 08:27:40,871 - root - INFO - step: 85795 loss: 16.0283 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.5847 +[titan] 2025-07-10 08:27:40,872 - root - INFO - lr: 4.3485e-05 gnorm: 0.98 [18:52:37< 3:07:31] +[titan] 2025-07-10 08:27:44,025 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:27:44,813 - root - INFO - step: 85800 loss: 16.0863 memory: 44.58GiB(31.99%) tps: 83,142 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.6363 +[titan] 2025-07-10 08:27:44,813 - root - INFO - lr: 4.3475e-05 gnorm: 0.97 [18:52:41< 3:07:27] +[titan] 2025-07-10 08:27:48,736 - root - INFO - step: 85805 loss: 15.9846 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4348 global_avg_mtp_loss: 13.5498 +[titan] 2025-07-10 08:27:48,737 - root - INFO - lr: 4.3466e-05 gnorm: 0.98 [18:52:45< 3:07:23] +[titan] 2025-07-10 08:27:52,634 - root - INFO - step: 85810 loss: 16.4118 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8965 +[titan] 2025-07-10 08:27:52,634 - root - INFO - lr: 4.3457e-05 gnorm: 1.00 [18:52:49< 3:07:19] +[titan] 2025-07-10 08:27:56,562 - root - INFO - step: 85815 loss: 16.4476 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.5194 global_avg_mtp_loss: 13.9282 +[titan] 2025-07-10 08:27:56,562 - root - INFO - lr: 4.3447e-05 gnorm: 0.99 [18:52:53< 3:07:15] +[titan] 2025-07-10 08:28:00,499 - root - INFO - step: 85820 loss: 15.9626 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4303 global_avg_mtp_loss: 13.5323 +[titan] 2025-07-10 08:28:00,499 - root - INFO - lr: 4.3438e-05 gnorm: 0.96 [18:52:57< 3:07:11] +[titan] 2025-07-10 08:28:04,450 - root - INFO - step: 85825 loss: 16.2578 memory: 44.58GiB(31.99%) tps: 82,944 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 08:28:04,451 - root - INFO - lr: 4.3429e-05 gnorm: 0.95 [18:53:01< 3:07:07] +[titan] 2025-07-10 08:28:08,393 - root - INFO - step: 85830 loss: 16.3565 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8569 +[titan] 2025-07-10 08:28:08,393 - root - INFO - lr: 4.3419e-05 gnorm: 0.98 [18:53:05< 3:07:03] +[titan] 2025-07-10 08:28:12,308 - root - INFO - step: 85835 loss: 15.6382 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.3780 global_avg_mtp_loss: 13.2602 +[titan] 2025-07-10 08:28:12,308 - root - INFO - lr: 4.3410e-05 gnorm: 1.04 [18:53:09< 3:06:59] +[titan] 2025-07-10 08:28:16,217 - root - INFO - step: 85840 loss: 16.2931 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.8118 +[titan] 2025-07-10 08:28:16,217 - root - INFO - lr: 4.3401e-05 gnorm: 0.97 [18:53:12< 3:06:55] +[titan] 2025-07-10 08:28:20,144 - root - INFO - step: 85845 loss: 16.1125 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.6480 +[titan] 2025-07-10 08:28:20,145 - root - INFO - lr: 4.3392e-05 gnorm: 0.95 [18:53:16< 3:06:52] +[titan] 2025-07-10 08:28:23,281 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:28:24,081 - root - INFO - step: 85850 loss: 16.0618 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6003 +[titan] 2025-07-10 08:28:24,081 - root - INFO - lr: 4.3382e-05 gnorm: 0.99 [18:53:20< 3:06:48] +[titan] 2025-07-10 08:28:28,004 - root - INFO - step: 85855 loss: 16.0714 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6187 +[titan] 2025-07-10 08:28:28,004 - root - INFO - lr: 4.3373e-05 gnorm: 1.02 [18:53:24< 3:06:44] +[titan] 2025-07-10 08:28:31,936 - root - INFO - step: 85860 loss: 16.2486 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7640 +[titan] 2025-07-10 08:28:31,936 - root - INFO - lr: 4.3364e-05 gnorm: 0.96 [18:53:28< 3:06:40] +[titan] 2025-07-10 08:28:35,887 - root - INFO - step: 85865 loss: 16.2383 memory: 44.58GiB(31.99%) tps: 82,930 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7626 +[titan] 2025-07-10 08:28:35,888 - root - INFO - lr: 4.3354e-05 gnorm: 1.04 [18:53:32< 3:06:36] +[titan] 2025-07-10 08:28:39,854 - root - INFO - step: 85870 loss: 16.3013 memory: 44.58GiB(31.99%) tps: 82,611 tflops: 285.10 mfu: 28.83% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8089 +[titan] 2025-07-10 08:28:39,855 - root - INFO - lr: 4.3345e-05 gnorm: 1.01 [18:53:36< 3:06:32] +[titan] 2025-07-10 08:28:43,816 - root - INFO - step: 85875 loss: 16.0357 memory: 44.58GiB(31.99%) tps: 82,728 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.5761 +[titan] 2025-07-10 08:28:43,816 - root - INFO - lr: 4.3336e-05 gnorm: 1.02 [18:53:40< 3:06:28] +[titan] 2025-07-10 08:28:47,751 - root - INFO - step: 85880 loss: 16.0523 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4491 global_avg_mtp_loss: 13.6033 +[titan] 2025-07-10 08:28:47,751 - root - INFO - lr: 4.3326e-05 gnorm: 0.98 [18:53:44< 3:06:24] +[titan] 2025-07-10 08:28:51,709 - root - INFO - step: 85885 loss: 16.0346 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.5866 +[titan] 2025-07-10 08:28:51,709 - root - INFO - lr: 4.3317e-05 gnorm: 1.00 [18:53:48< 3:06:20] +[titan] 2025-07-10 08:28:55,656 - root - INFO - step: 85890 loss: 16.3801 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8701 +[titan] 2025-07-10 08:28:55,656 - root - INFO - lr: 4.3308e-05 gnorm: 1.06 [18:53:52< 3:06:16] +[titan] 2025-07-10 08:28:59,565 - root - INFO - step: 85895 loss: 16.3296 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5062 global_avg_mtp_loss: 13.8234 +[titan] 2025-07-10 08:28:59,566 - root - INFO - lr: 4.3299e-05 gnorm: 1.00 [18:53:56< 3:06:12] +[titan] 2025-07-10 08:29:02,713 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:29:03,510 - root - INFO - step: 85900 loss: 16.3949 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.8784 +[titan] 2025-07-10 08:29:03,511 - root - INFO - lr: 4.3289e-05 gnorm: 1.02 [18:54:00< 3:06:08] +[titan] 2025-07-10 08:29:07,451 - root - INFO - step: 85905 loss: 16.4870 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.5333 global_avg_mtp_loss: 13.9537 +[titan] 2025-07-10 08:29:07,451 - root - INFO - lr: 4.3280e-05 gnorm: 1.01 [18:54:04< 3:06:04] +[titan] 2025-07-10 08:29:11,369 - root - INFO - step: 85910 loss: 16.1506 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.6832 +[titan] 2025-07-10 08:29:11,370 - root - INFO - lr: 4.3271e-05 gnorm: 1.02 [18:54:08< 3:06:00] +[titan] 2025-07-10 08:29:15,285 - root - INFO - step: 85915 loss: 16.3690 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5108 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 08:29:15,285 - root - INFO - lr: 4.3262e-05 gnorm: 1.08 [18:54:11< 3:05:56] +[titan] 2025-07-10 08:29:19,255 - root - INFO - step: 85920 loss: 16.0914 memory: 44.58GiB(31.99%) tps: 82,549 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.6286 +[titan] 2025-07-10 08:29:19,255 - root - INFO - lr: 4.3252e-05 gnorm: 0.98 [18:54:15< 3:05:52] +[titan] 2025-07-10 08:29:23,161 - root - INFO - step: 85925 loss: 16.1528 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6878 +[titan] 2025-07-10 08:29:23,162 - root - INFO - lr: 4.3243e-05 gnorm: 1.01 [18:54:19< 3:05:48] +[titan] 2025-07-10 08:29:27,104 - root - INFO - step: 85930 loss: 16.0771 memory: 44.58GiB(31.99%) tps: 83,122 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.6338 +[titan] 2025-07-10 08:29:27,104 - root - INFO - lr: 4.3234e-05 gnorm: 1.02 [18:54:23< 3:05:44] +[titan] 2025-07-10 08:29:31,035 - root - INFO - step: 85935 loss: 16.2086 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4691 global_avg_mtp_loss: 13.7395 +[titan] 2025-07-10 08:29:31,035 - root - INFO - lr: 4.3225e-05 gnorm: 0.99 [18:54:27< 3:05:40] +[titan] 2025-07-10 08:29:34,959 - root - INFO - step: 85940 loss: 16.3773 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8805 +[titan] 2025-07-10 08:29:34,960 - root - INFO - lr: 4.3215e-05 gnorm: 0.99 [18:54:31< 3:05:36] +[titan] 2025-07-10 08:29:38,919 - root - INFO - step: 85945 loss: 16.1979 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.7406 +[titan] 2025-07-10 08:29:38,919 - root - INFO - lr: 4.3206e-05 gnorm: 1.01 [18:54:35< 3:05:32] +[titan] 2025-07-10 08:29:42,075 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:29:42,868 - root - INFO - step: 85950 loss: 16.2191 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7486 +[titan] 2025-07-10 08:29:42,869 - root - INFO - lr: 4.3197e-05 gnorm: 0.98 [18:54:39< 3:05:28] +[titan] 2025-07-10 08:29:46,809 - root - INFO - step: 85955 loss: 16.2681 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7776 +[titan] 2025-07-10 08:29:46,810 - root - INFO - lr: 4.3188e-05 gnorm: 1.04 [18:54:43< 3:05:24] +[titan] 2025-07-10 08:29:50,716 - root - INFO - step: 85960 loss: 15.9627 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4254 global_avg_mtp_loss: 13.5373 +[titan] 2025-07-10 08:29:50,717 - root - INFO - lr: 4.3178e-05 gnorm: 0.97 [18:54:47< 3:05:20] +[titan] 2025-07-10 08:29:54,648 - root - INFO - step: 85965 loss: 16.1602 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6985 +[titan] 2025-07-10 08:29:54,648 - root - INFO - lr: 4.3169e-05 gnorm: 0.97 [18:54:51< 3:05:16] +[titan] 2025-07-10 08:29:58,585 - root - INFO - step: 85970 loss: 15.9811 memory: 44.58GiB(31.99%) tps: 83,226 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4340 global_avg_mtp_loss: 13.5470 +[titan] 2025-07-10 08:29:58,586 - root - INFO - lr: 4.3160e-05 gnorm: 0.95 [18:54:55< 3:05:12] +[titan] 2025-07-10 08:30:02,484 - root - INFO - step: 85975 loss: 16.2303 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7595 +[titan] 2025-07-10 08:30:02,484 - root - INFO - lr: 4.3151e-05 gnorm: 1.00 [18:54:59< 3:05:08] +[titan] 2025-07-10 08:30:06,440 - root - INFO - step: 85980 loss: 16.2081 memory: 44.58GiB(31.99%) tps: 82,830 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.7402 +[titan] 2025-07-10 08:30:06,441 - root - INFO - lr: 4.3142e-05 gnorm: 0.98 [18:55:03< 3:05:04] +[titan] 2025-07-10 08:30:10,371 - root - INFO - step: 85985 loss: 16.0600 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.6200 +[titan] 2025-07-10 08:30:10,371 - root - INFO - lr: 4.3132e-05 gnorm: 0.99 [18:55:07< 3:05:01] +[titan] 2025-07-10 08:30:14,332 - root - INFO - step: 85990 loss: 15.9093 memory: 44.58GiB(31.99%) tps: 82,727 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4375 global_avg_mtp_loss: 13.4718 +[titan] 2025-07-10 08:30:14,332 - root - INFO - lr: 4.3123e-05 gnorm: 1.00 [18:55:11< 3:04:57] +[titan] 2025-07-10 08:30:18,252 - root - INFO - step: 85995 loss: 16.3859 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8789 +[titan] 2025-07-10 08:30:18,252 - root - INFO - lr: 4.3114e-05 gnorm: 0.97 [18:55:14< 3:04:53] +[titan] 2025-07-10 08:30:21,377 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:30:22,177 - root - INFO - step: 86000 loss: 16.1252 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6625 +[titan] 2025-07-10 08:30:22,177 - root - INFO - lr: 4.3105e-05 gnorm: 0.96 [18:55:18< 3:04:49] +[titan] 2025-07-10 08:30:26,083 - root - INFO - step: 86005 loss: 16.0658 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6218 +[titan] 2025-07-10 08:30:26,083 - root - INFO - lr: 4.3095e-05 gnorm: 1.02 [18:55:22< 3:04:45] +[titan] 2025-07-10 08:30:30,000 - root - INFO - step: 86010 loss: 16.0571 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4467 global_avg_mtp_loss: 13.6103 +[titan] 2025-07-10 08:30:30,000 - root - INFO - lr: 4.3086e-05 gnorm: 1.03 [18:55:26< 3:04:41] +[titan] 2025-07-10 08:30:34,006 - root - INFO - step: 86015 loss: 16.3669 memory: 44.58GiB(31.99%) tps: 81,807 tflops: 282.33 mfu: 28.55% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8589 +[titan] 2025-07-10 08:30:34,006 - root - INFO - lr: 4.3077e-05 gnorm: 1.00 [18:55:30< 3:04:37] +[titan] 2025-07-10 08:30:34,942 - root - INFO - Dumping profiler traces at step 86016 +[titan] 2025-07-10 08:30:34,975 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:30:38,107 - root - INFO - step: 86020 loss: 16.1334 memory: 44.58GiB(31.99%) tps: 79,914 tflops: 275.80 mfu: 27.89% global_avg_ntp_loss: 2.4540 global_avg_mtp_loss: 13.6794 +[titan] 2025-07-10 08:30:38,107 - root - INFO - lr: 4.3068e-05 gnorm: 1.02 [18:55:34< 3:04:33] +[titan] 2025-07-10 08:30:42,023 - root - INFO - step: 86025 loss: 16.0918 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4565 global_avg_mtp_loss: 13.6353 +[titan] 2025-07-10 08:30:42,024 - root - INFO - lr: 4.3059e-05 gnorm: 1.01 [18:55:38< 3:04:29] +[titan] 2025-07-10 08:30:45,954 - root - INFO - step: 86030 loss: 16.1680 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4664 global_avg_mtp_loss: 13.7016 +[titan] 2025-07-10 08:30:45,954 - root - INFO - lr: 4.3049e-05 gnorm: 1.02 [18:55:42< 3:04:25] +[titan] 2025-07-10 08:30:49,892 - root - INFO - step: 86035 loss: 16.3058 memory: 44.58GiB(31.99%) tps: 83,221 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.8188 +[titan] 2025-07-10 08:30:49,892 - root - INFO - lr: 4.3040e-05 gnorm: 1.12 [18:55:46< 3:04:21] +[titan] 2025-07-10 08:30:53,857 - root - INFO - step: 86040 loss: 16.3218 memory: 44.58GiB(31.99%) tps: 82,645 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8285 +[titan] 2025-07-10 08:30:53,857 - root - INFO - lr: 4.3031e-05 gnorm: 1.09 [18:55:50< 3:04:17] +[titan] 2025-07-10 08:30:57,847 - root - INFO - step: 86045 loss: 16.4437 memory: 44.58GiB(31.99%) tps: 82,130 tflops: 283.44 mfu: 28.66% global_avg_ntp_loss: 2.5236 global_avg_mtp_loss: 13.9200 +[titan] 2025-07-10 08:30:57,848 - root - INFO - lr: 4.3022e-05 gnorm: 0.98 [18:55:54< 3:04:13] +[titan] 2025-07-10 08:31:00,989 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:31:01,776 - root - INFO - step: 86050 loss: 16.3738 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5004 global_avg_mtp_loss: 13.8734 +[titan] 2025-07-10 08:31:01,776 - root - INFO - lr: 4.3013e-05 gnorm: 0.96 [18:55:58< 3:04:09] +[titan] 2025-07-10 08:31:05,685 - root - INFO - step: 86055 loss: 16.2825 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4893 global_avg_mtp_loss: 13.7932 +[titan] 2025-07-10 08:31:05,686 - root - INFO - lr: 4.3004e-05 gnorm: 1.03 [18:56:02< 3:04:05] +[titan] 2025-07-10 08:31:09,594 - root - INFO - step: 86060 loss: 16.0340 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4410 global_avg_mtp_loss: 13.5930 +[titan] 2025-07-10 08:31:09,595 - root - INFO - lr: 4.2994e-05 gnorm: 1.01 [18:56:06< 3:04:01] +[titan] 2025-07-10 08:31:13,490 - root - INFO - step: 86065 loss: 16.4101 memory: 44.58GiB(31.99%) tps: 84,130 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 08:31:13,490 - root - INFO - lr: 4.2985e-05 gnorm: 0.95 [18:56:10< 3:03:57] +[titan] 2025-07-10 08:31:17,405 - root - INFO - step: 86070 loss: 16.2283 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7465 +[titan] 2025-07-10 08:31:17,406 - root - INFO - lr: 4.2976e-05 gnorm: 1.00 [18:56:14< 3:03:53] +[titan] 2025-07-10 08:31:21,316 - root - INFO - step: 86075 loss: 16.1635 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 08:31:21,317 - root - INFO - lr: 4.2967e-05 gnorm: 0.99 [18:56:17< 3:03:49] +[titan] 2025-07-10 08:31:25,209 - root - INFO - step: 86080 loss: 16.2217 memory: 44.58GiB(31.99%) tps: 84,188 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7532 +[titan] 2025-07-10 08:31:25,209 - root - INFO - lr: 4.2958e-05 gnorm: 1.00 [18:56:21< 3:03:45] +[titan] 2025-07-10 08:31:29,130 - root - INFO - step: 86085 loss: 16.2925 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.8047 +[titan] 2025-07-10 08:31:29,130 - root - INFO - lr: 4.2949e-05 gnorm: 1.03 [18:56:25< 3:03:41] +[titan] 2025-07-10 08:31:33,029 - root - INFO - step: 86090 loss: 16.1945 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.7267 +[titan] 2025-07-10 08:31:33,029 - root - INFO - lr: 4.2939e-05 gnorm: 0.97 [18:56:29< 3:03:37] +[titan] 2025-07-10 08:31:36,944 - root - INFO - step: 86095 loss: 16.1947 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7153 +[titan] 2025-07-10 08:31:36,944 - root - INFO - lr: 4.2930e-05 gnorm: 1.06 [18:56:33< 3:03:33] +[titan] 2025-07-10 08:31:40,075 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:31:40,861 - root - INFO - step: 86100 loss: 16.1590 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.6951 +[titan] 2025-07-10 08:31:40,861 - root - INFO - lr: 4.2921e-05 gnorm: 1.04 [18:56:37< 3:03:29] +[titan] 2025-07-10 08:31:44,789 - root - INFO - step: 86105 loss: 16.2411 memory: 44.58GiB(31.99%) tps: 83,424 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7581 +[titan] 2025-07-10 08:31:44,790 - root - INFO - lr: 4.2912e-05 gnorm: 1.00 [18:56:41< 3:03:25] +[titan] 2025-07-10 08:31:48,719 - root - INFO - step: 86110 loss: 16.2582 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7752 +[titan] 2025-07-10 08:31:48,719 - root - INFO - lr: 4.2903e-05 gnorm: 1.03 [18:56:45< 3:03:21] +[titan] 2025-07-10 08:31:52,637 - root - INFO - step: 86115 loss: 16.1578 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6904 +[titan] 2025-07-10 08:31:52,637 - root - INFO - lr: 4.2894e-05 gnorm: 1.00 [18:56:49< 3:03:17] +[titan] 2025-07-10 08:31:56,541 - root - INFO - step: 86120 loss: 16.2710 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.7819 +[titan] 2025-07-10 08:31:56,541 - root - INFO - lr: 4.2885e-05 gnorm: 0.97 [18:56:53< 3:03:13] +[titan] 2025-07-10 08:32:00,487 - root - INFO - step: 86125 loss: 16.3203 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-10 08:32:00,487 - root - INFO - lr: 4.2875e-05 gnorm: 0.97 [18:56:57< 3:03:09] +[titan] 2025-07-10 08:32:04,459 - root - INFO - step: 86130 loss: 16.1655 memory: 44.58GiB(31.99%) tps: 82,497 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.7020 +[titan] 2025-07-10 08:32:04,460 - root - INFO - lr: 4.2866e-05 gnorm: 1.01 [18:57:01< 3:03:06] +[titan] 2025-07-10 08:32:08,389 - root - INFO - step: 86135 loss: 15.9193 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4288 global_avg_mtp_loss: 13.4905 +[titan] 2025-07-10 08:32:08,389 - root - INFO - lr: 4.2857e-05 gnorm: 1.02 [18:57:05< 3:03:02] +[titan] 2025-07-10 08:32:12,314 - root - INFO - step: 86140 loss: 16.2903 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.8074 +[titan] 2025-07-10 08:32:12,314 - root - INFO - lr: 4.2848e-05 gnorm: 0.97 [18:57:08< 3:02:58] +[titan] 2025-07-10 08:32:16,229 - root - INFO - step: 86145 loss: 16.2689 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7760 +[titan] 2025-07-10 08:32:16,229 - root - INFO - lr: 4.2839e-05 gnorm: 1.01 [18:57:12< 3:02:54] +[titan] 2025-07-10 08:32:19,341 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:32:20,130 - root - INFO - step: 86150 loss: 16.3949 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8945 +[titan] 2025-07-10 08:32:20,130 - root - INFO - lr: 4.2830e-05 gnorm: 1.00 [18:57:16< 3:02:50] +[titan] 2025-07-10 08:32:24,040 - root - INFO - step: 86155 loss: 16.2844 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.7908 +[titan] 2025-07-10 08:32:24,040 - root - INFO - lr: 4.2821e-05 gnorm: 1.05 [18:57:20< 3:02:46] +[titan] 2025-07-10 08:32:27,925 - root - INFO - step: 86160 loss: 16.1659 memory: 44.58GiB(31.99%) tps: 84,355 tflops: 291.12 mfu: 29.44% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.7012 +[titan] 2025-07-10 08:32:27,925 - root - INFO - lr: 4.2812e-05 gnorm: 1.05 [18:57:24< 3:02:42] +[titan] 2025-07-10 08:32:31,857 - root - INFO - step: 86165 loss: 16.1114 memory: 44.58GiB(31.99%) tps: 83,354 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4542 global_avg_mtp_loss: 13.6572 +[titan] 2025-07-10 08:32:31,857 - root - INFO - lr: 4.2803e-05 gnorm: 1.04 [18:57:28< 3:02:38] +[titan] 2025-07-10 08:32:35,751 - root - INFO - step: 86170 loss: 16.3312 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-10 08:32:35,751 - root - INFO - lr: 4.2793e-05 gnorm: 1.00 [18:57:32< 3:02:34] +[titan] 2025-07-10 08:32:39,665 - root - INFO - step: 86175 loss: 15.8958 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4157 global_avg_mtp_loss: 13.4801 +[titan] 2025-07-10 08:32:39,665 - root - INFO - lr: 4.2784e-05 gnorm: 1.00 [18:57:36< 3:02:30] +[titan] 2025-07-10 08:32:43,584 - root - INFO - step: 86180 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6917 +[titan] 2025-07-10 08:32:43,584 - root - INFO - lr: 4.2775e-05 gnorm: 1.05 [18:57:40< 3:02:26] +[titan] 2025-07-10 08:32:47,506 - root - INFO - step: 86185 loss: 16.2422 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7663 +[titan] 2025-07-10 08:32:47,506 - root - INFO - lr: 4.2766e-05 gnorm: 1.01 [18:57:44< 3:02:22] +[titan] 2025-07-10 08:32:51,451 - root - INFO - step: 86190 loss: 16.1687 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.7017 +[titan] 2025-07-10 08:32:51,451 - root - INFO - lr: 4.2757e-05 gnorm: 0.96 [18:57:48< 3:02:18] +[titan] 2025-07-10 08:32:55,369 - root - INFO - step: 86195 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.8878 +[titan] 2025-07-10 08:32:55,369 - root - INFO - lr: 4.2748e-05 gnorm: 1.01 [18:57:52< 3:02:14] +[titan] 2025-07-10 08:32:58,499 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:32:59,308 - root - INFO - step: 86200 loss: 16.0818 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.6355 +[titan] 2025-07-10 08:32:59,308 - root - INFO - lr: 4.2739e-05 gnorm: 1.00 [18:57:55< 3:02:10] +[titan] 2025-07-10 08:33:03,262 - root - INFO - step: 86205 loss: 16.2072 memory: 44.58GiB(31.99%) tps: 82,884 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.7398 +[titan] 2025-07-10 08:33:03,262 - root - INFO - lr: 4.2730e-05 gnorm: 0.97 [18:57:59< 3:02:06] +[titan] 2025-07-10 08:33:07,181 - root - INFO - step: 86210 loss: 16.1443 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4563 global_avg_mtp_loss: 13.6881 +[titan] 2025-07-10 08:33:07,182 - root - INFO - lr: 4.2721e-05 gnorm: 1.01 [18:58:03< 3:02:02] +[titan] 2025-07-10 08:33:11,132 - root - INFO - step: 86215 loss: 16.2498 memory: 44.58GiB(31.99%) tps: 82,943 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7714 +[titan] 2025-07-10 08:33:11,133 - root - INFO - lr: 4.2712e-05 gnorm: 1.02 [18:58:07< 3:01:58] +[titan] 2025-07-10 08:33:15,059 - root - INFO - step: 86220 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5284 global_avg_mtp_loss: 13.9165 +[titan] 2025-07-10 08:33:15,059 - root - INFO - lr: 4.2703e-05 gnorm: 1.01 [18:58:11< 3:01:54] +[titan] 2025-07-10 08:33:18,993 - root - INFO - step: 86225 loss: 16.0636 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6081 +[titan] 2025-07-10 08:33:18,993 - root - INFO - lr: 4.2693e-05 gnorm: 0.99 [18:58:15< 3:01:50] +[titan] 2025-07-10 08:33:22,901 - root - INFO - step: 86230 loss: 16.2129 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.27% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.7417 +[titan] 2025-07-10 08:33:22,901 - root - INFO - lr: 4.2684e-05 gnorm: 1.02 [18:58:19< 3:01:46] +[titan] 2025-07-10 08:33:26,824 - root - INFO - step: 86235 loss: 16.1803 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.6968 +[titan] 2025-07-10 08:33:26,824 - root - INFO - lr: 4.2675e-05 gnorm: 0.98 [18:58:23< 3:01:42] +[titan] 2025-07-10 08:33:30,764 - root - INFO - step: 86240 loss: 16.0411 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4526 global_avg_mtp_loss: 13.5885 +[titan] 2025-07-10 08:33:30,764 - root - INFO - lr: 4.2666e-05 gnorm: 1.02 [18:58:27< 3:01:38] +[titan] 2025-07-10 08:33:34,718 - root - INFO - step: 86245 loss: 16.4778 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5487 global_avg_mtp_loss: 13.9291 +[titan] 2025-07-10 08:33:34,718 - root - INFO - lr: 4.2657e-05 gnorm: 1.02 [18:58:31< 3:01:34] +[titan] 2025-07-10 08:33:37,838 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:33:38,633 - root - INFO - step: 86250 loss: 16.2200 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.7456 +[titan] 2025-07-10 08:33:38,633 - root - INFO - lr: 4.2648e-05 gnorm: 0.99 [18:58:35< 3:01:30] +[titan] 2025-07-10 08:33:42,561 - root - INFO - step: 86255 loss: 16.2457 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 08:33:42,562 - root - INFO - lr: 4.2639e-05 gnorm: 0.99 [18:58:39< 3:01:26] +[titan] 2025-07-10 08:33:46,505 - root - INFO - step: 86260 loss: 16.3519 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5090 global_avg_mtp_loss: 13.8429 +[titan] 2025-07-10 08:33:46,505 - root - INFO - lr: 4.2630e-05 gnorm: 0.99 [18:58:43< 3:01:22] +[titan] 2025-07-10 08:33:50,458 - root - INFO - step: 86265 loss: 16.8089 memory: 44.58GiB(31.99%) tps: 82,890 tflops: 286.07 mfu: 28.92% global_avg_ntp_loss: 2.5934 global_avg_mtp_loss: 14.2155 +[titan] 2025-07-10 08:33:50,459 - root - INFO - lr: 4.2621e-05 gnorm: 1.03 [18:58:47< 3:01:18] +[titan] 2025-07-10 08:33:54,375 - root - INFO - step: 86270 loss: 16.3573 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.8672 +[titan] 2025-07-10 08:33:54,375 - root - INFO - lr: 4.2612e-05 gnorm: 1.04 [18:58:51< 3:01:14] +[titan] 2025-07-10 08:33:58,293 - root - INFO - step: 86275 loss: 16.1512 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6823 +[titan] 2025-07-10 08:33:58,293 - root - INFO - lr: 4.2603e-05 gnorm: 0.99 [18:58:54< 3:01:11] +[titan] 2025-07-10 08:34:02,199 - root - INFO - step: 86280 loss: 16.1046 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6423 +[titan] 2025-07-10 08:34:02,200 - root - INFO - lr: 4.2594e-05 gnorm: 0.99 [18:58:58< 3:01:07] +[titan] 2025-07-10 08:34:06,130 - root - INFO - step: 86285 loss: 15.9408 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4437 global_avg_mtp_loss: 13.4971 +[titan] 2025-07-10 08:34:06,131 - root - INFO - lr: 4.2585e-05 gnorm: 1.06 [18:59:02< 3:01:03] +[titan] 2025-07-10 08:34:10,053 - root - INFO - step: 86290 loss: 15.9207 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4171 global_avg_mtp_loss: 13.5036 +[titan] 2025-07-10 08:34:10,053 - root - INFO - lr: 4.2576e-05 gnorm: 1.00 [18:59:06< 3:00:59] +[titan] 2025-07-10 08:34:13,951 - root - INFO - step: 86295 loss: 16.1355 memory: 44.58GiB(31.99%) tps: 84,078 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.6647 +[titan] 2025-07-10 08:34:13,951 - root - INFO - lr: 4.2567e-05 gnorm: 1.01 [18:59:10< 3:00:55] +[titan] 2025-07-10 08:34:17,064 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:34:17,853 - root - INFO - step: 86300 loss: 16.6184 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.5586 global_avg_mtp_loss: 14.0598 +[titan] 2025-07-10 08:34:17,854 - root - INFO - lr: 4.2558e-05 gnorm: 1.04 [18:59:14< 3:00:51] +[titan] 2025-07-10 08:34:21,765 - root - INFO - step: 86305 loss: 16.2697 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7876 +[titan] 2025-07-10 08:34:21,766 - root - INFO - lr: 4.2549e-05 gnorm: 0.98 [18:59:18< 3:00:47] +[titan] 2025-07-10 08:34:25,697 - root - INFO - step: 86310 loss: 16.4040 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8987 +[titan] 2025-07-10 08:34:25,698 - root - INFO - lr: 4.2540e-05 gnorm: 0.98 [18:59:22< 3:00:43] +[titan] 2025-07-10 08:34:29,618 - root - INFO - step: 86315 loss: 16.0871 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6298 +[titan] 2025-07-10 08:34:29,618 - root - INFO - lr: 4.2531e-05 gnorm: 1.00 [18:59:26< 3:00:39] +[titan] 2025-07-10 08:34:33,517 - root - INFO - step: 86320 loss: 16.2481 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 08:34:33,518 - root - INFO - lr: 4.2522e-05 gnorm: 0.97 [18:59:30< 3:00:35] +[titan] 2025-07-10 08:34:37,425 - root - INFO - step: 86325 loss: 16.1676 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.7061 +[titan] 2025-07-10 08:34:37,425 - root - INFO - lr: 4.2513e-05 gnorm: 1.06 [18:59:34< 3:00:31] +[titan] 2025-07-10 08:34:41,336 - root - INFO - step: 86330 loss: 15.9472 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4214 global_avg_mtp_loss: 13.5258 +[titan] 2025-07-10 08:34:41,337 - root - INFO - lr: 4.2504e-05 gnorm: 0.99 [18:59:37< 3:00:27] +[titan] 2025-07-10 08:34:45,245 - root - INFO - step: 86335 loss: 16.2490 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.7569 +[titan] 2025-07-10 08:34:45,245 - root - INFO - lr: 4.2495e-05 gnorm: 1.02 [18:59:41< 3:00:23] +[titan] 2025-07-10 08:34:49,181 - root - INFO - step: 86340 loss: 16.1924 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7191 +[titan] 2025-07-10 08:34:49,181 - root - INFO - lr: 4.2486e-05 gnorm: 0.97 [18:59:45< 3:00:19] +[titan] 2025-07-10 08:34:53,093 - root - INFO - step: 86345 loss: 16.0151 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4406 global_avg_mtp_loss: 13.5744 +[titan] 2025-07-10 08:34:53,094 - root - INFO - lr: 4.2477e-05 gnorm: 1.00 [18:59:49< 3:00:15] +[titan] 2025-07-10 08:34:56,213 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:34:57,006 - root - INFO - step: 86350 loss: 16.4032 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.8898 +[titan] 2025-07-10 08:34:57,007 - root - INFO - lr: 4.2468e-05 gnorm: 0.99 [18:59:53< 3:00:11] +[titan] 2025-07-10 08:35:00,914 - root - INFO - step: 86355 loss: 16.0544 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4334 global_avg_mtp_loss: 13.6210 +[titan] 2025-07-10 08:35:00,914 - root - INFO - lr: 4.2459e-05 gnorm: 1.01 [18:59:57< 3:00:07] +[titan] 2025-07-10 08:35:04,807 - root - INFO - step: 86360 loss: 16.4097 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.5093 global_avg_mtp_loss: 13.9004 +[titan] 2025-07-10 08:35:04,807 - root - INFO - lr: 4.2450e-05 gnorm: 0.96 [19:00:01< 3:00:03] +[titan] 2025-07-10 08:35:08,759 - root - INFO - step: 86365 loss: 16.2237 memory: 44.58GiB(31.99%) tps: 82,927 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7530 +[titan] 2025-07-10 08:35:08,759 - root - INFO - lr: 4.2441e-05 gnorm: 1.00 [19:00:05< 2:59:59] +[titan] 2025-07-10 08:35:12,665 - root - INFO - step: 86370 loss: 16.4474 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.9316 +[titan] 2025-07-10 08:35:12,665 - root - INFO - lr: 4.2432e-05 gnorm: 0.97 [19:00:09< 2:59:55] +[titan] 2025-07-10 08:35:16,589 - root - INFO - step: 86375 loss: 16.4656 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.9481 +[titan] 2025-07-10 08:35:16,589 - root - INFO - lr: 4.2423e-05 gnorm: 0.99 [19:00:13< 2:59:51] +[titan] 2025-07-10 08:35:20,493 - root - INFO - step: 86380 loss: 16.0064 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4268 global_avg_mtp_loss: 13.5796 +[titan] 2025-07-10 08:35:20,493 - root - INFO - lr: 4.2414e-05 gnorm: 1.05 [19:00:17< 2:59:47] +[titan] 2025-07-10 08:35:24,404 - root - INFO - step: 86385 loss: 16.1690 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4656 global_avg_mtp_loss: 13.7034 +[titan] 2025-07-10 08:35:24,404 - root - INFO - lr: 4.2405e-05 gnorm: 1.01 [19:00:21< 2:59:43] +[titan] 2025-07-10 08:35:28,338 - root - INFO - step: 86390 loss: 16.1067 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6368 +[titan] 2025-07-10 08:35:28,339 - root - INFO - lr: 4.2396e-05 gnorm: 0.98 [19:00:24< 2:59:39] +[titan] 2025-07-10 08:35:32,248 - root - INFO - step: 86395 loss: 16.3434 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 08:35:32,248 - root - INFO - lr: 4.2387e-05 gnorm: 0.98 [19:00:28< 2:59:35] +[titan] 2025-07-10 08:35:35,367 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:35:36,154 - root - INFO - step: 86400 loss: 16.2473 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7710 +[titan] 2025-07-10 08:35:36,154 - root - INFO - lr: 4.2378e-05 gnorm: 0.97 [19:00:32< 2:59:31] +[titan] 2025-07-10 08:35:40,084 - root - INFO - step: 86405 loss: 16.3604 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8597 +[titan] 2025-07-10 08:35:40,084 - root - INFO - lr: 4.2369e-05 gnorm: 1.04 [19:00:36< 2:59:27] +[titan] 2025-07-10 08:35:44,039 - root - INFO - step: 86410 loss: 16.2671 memory: 44.58GiB(31.99%) tps: 82,863 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.4918 global_avg_mtp_loss: 13.7753 +[titan] 2025-07-10 08:35:44,039 - root - INFO - lr: 4.2360e-05 gnorm: 1.01 [19:00:40< 2:59:23] +[titan] 2025-07-10 08:35:47,950 - root - INFO - step: 86415 loss: 16.0211 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.5810 +[titan] 2025-07-10 08:35:47,950 - root - INFO - lr: 4.2351e-05 gnorm: 0.98 [19:00:44< 2:59:19] +[titan] 2025-07-10 08:35:51,889 - root - INFO - step: 86420 loss: 16.1618 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6944 +[titan] 2025-07-10 08:35:51,889 - root - INFO - lr: 4.2342e-05 gnorm: 1.00 [19:00:48< 2:59:15] +[titan] 2025-07-10 08:35:55,793 - root - INFO - step: 86425 loss: 16.4734 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.9619 +[titan] 2025-07-10 08:35:55,793 - root - INFO - lr: 4.2333e-05 gnorm: 1.03 [19:00:52< 2:59:12] +[titan] 2025-07-10 08:35:59,707 - root - INFO - step: 86430 loss: 16.1009 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4558 global_avg_mtp_loss: 13.6451 +[titan] 2025-07-10 08:35:59,707 - root - INFO - lr: 4.2324e-05 gnorm: 1.05 [19:00:56< 2:59:08] +[titan] 2025-07-10 08:36:03,621 - root - INFO - step: 86435 loss: 16.3535 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8583 +[titan] 2025-07-10 08:36:03,621 - root - INFO - lr: 4.2315e-05 gnorm: 1.13 [19:01:00< 2:59:04] +[titan] 2025-07-10 08:36:07,516 - root - INFO - step: 86440 loss: 16.2297 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 08:36:07,517 - root - INFO - lr: 4.2306e-05 gnorm: 1.02 [19:01:04< 2:59:00] +[titan] 2025-07-10 08:36:11,422 - root - INFO - step: 86445 loss: 16.2329 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7621 +[titan] 2025-07-10 08:36:11,422 - root - INFO - lr: 4.2298e-05 gnorm: 1.05 [19:01:08< 2:58:56] +[titan] 2025-07-10 08:36:14,541 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:36:15,329 - root - INFO - step: 86450 loss: 16.5265 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.9997 +[titan] 2025-07-10 08:36:15,329 - root - INFO - lr: 4.2289e-05 gnorm: 0.97 [19:01:11< 2:58:52] +[titan] 2025-07-10 08:36:19,235 - root - INFO - step: 86455 loss: 16.1436 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.6821 +[titan] 2025-07-10 08:36:19,235 - root - INFO - lr: 4.2280e-05 gnorm: 0.99 [19:01:15< 2:58:48] +[titan] 2025-07-10 08:36:23,158 - root - INFO - step: 86460 loss: 16.3888 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8802 +[titan] 2025-07-10 08:36:23,158 - root - INFO - lr: 4.2271e-05 gnorm: 1.03 [19:01:19< 2:58:44] +[titan] 2025-07-10 08:36:27,076 - root - INFO - step: 86465 loss: 16.1125 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.6436 +[titan] 2025-07-10 08:36:27,076 - root - INFO - lr: 4.2262e-05 gnorm: 1.02 [19:01:23< 2:58:40] +[titan] 2025-07-10 08:36:30,991 - root - INFO - step: 86470 loss: 16.1221 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.6686 +[titan] 2025-07-10 08:36:30,991 - root - INFO - lr: 4.2253e-05 gnorm: 1.05 [19:01:27< 2:58:36] +[titan] 2025-07-10 08:36:34,917 - root - INFO - step: 86475 loss: 16.2726 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.7864 +[titan] 2025-07-10 08:36:34,917 - root - INFO - lr: 4.2244e-05 gnorm: 1.02 [19:01:31< 2:58:32] +[titan] 2025-07-10 08:36:38,813 - root - INFO - step: 86480 loss: 16.1132 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.6597 +[titan] 2025-07-10 08:36:38,813 - root - INFO - lr: 4.2235e-05 gnorm: 1.11 [19:01:35< 2:58:28] +[titan] 2025-07-10 08:36:42,726 - root - INFO - step: 86485 loss: 16.1364 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 08:36:42,726 - root - INFO - lr: 4.2226e-05 gnorm: 0.99 [19:01:39< 2:58:24] +[titan] 2025-07-10 08:36:46,639 - root - INFO - step: 86490 loss: 16.1002 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.6626 +[titan] 2025-07-10 08:36:46,640 - root - INFO - lr: 4.2217e-05 gnorm: 0.99 [19:01:43< 2:58:20] +[titan] 2025-07-10 08:36:50,552 - root - INFO - step: 86495 loss: 16.1383 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6697 +[titan] 2025-07-10 08:36:50,553 - root - INFO - lr: 4.2208e-05 gnorm: 0.98 [19:01:47< 2:58:16] +[titan] 2025-07-10 08:36:53,672 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:36:54,460 - root - INFO - step: 86500 loss: 16.0459 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4408 global_avg_mtp_loss: 13.6052 +[titan] 2025-07-10 08:36:54,460 - root - INFO - lr: 4.2199e-05 gnorm: 1.07 [19:01:51< 2:58:12] +[titan] 2025-07-10 08:36:58,377 - root - INFO - step: 86505 loss: 16.1145 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6554 +[titan] 2025-07-10 08:36:58,377 - root - INFO - lr: 4.2191e-05 gnorm: 0.98 [19:01:55< 2:58:08] +[titan] 2025-07-10 08:37:02,268 - root - INFO - step: 86510 loss: 16.2287 memory: 44.58GiB(31.99%) tps: 84,213 tflops: 290.63 mfu: 29.39% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7436 +[titan] 2025-07-10 08:37:02,269 - root - INFO - lr: 4.2182e-05 gnorm: 1.07 [19:01:58< 2:58:04] +[titan] 2025-07-10 08:37:06,197 - root - INFO - step: 86515 loss: 16.2081 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.7032 +[titan] 2025-07-10 08:37:06,197 - root - INFO - lr: 4.2173e-05 gnorm: 1.01 [19:02:02< 2:58:00] +[titan] 2025-07-10 08:37:10,094 - root - INFO - step: 86520 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9653 +[titan] 2025-07-10 08:37:10,094 - root - INFO - lr: 4.2164e-05 gnorm: 1.00 [19:02:06< 2:57:56] +[titan] 2025-07-10 08:37:14,098 - root - INFO - step: 86525 loss: 16.0307 memory: 44.58GiB(31.99%) tps: 81,860 tflops: 282.51 mfu: 28.57% global_avg_ntp_loss: 2.4412 global_avg_mtp_loss: 13.5894 +[titan] 2025-07-10 08:37:14,098 - root - INFO - lr: 4.2155e-05 gnorm: 0.96 [19:02:10< 2:57:52] +[titan] 2025-07-10 08:37:16,599 - root - INFO - Dumping profiler traces at step 86528 +[titan] 2025-07-10 08:37:16,633 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:37:18,204 - root - INFO - step: 86530 loss: 16.0722 memory: 44.58GiB(31.99%) tps: 79,804 tflops: 275.42 mfu: 27.85% global_avg_ntp_loss: 2.4399 global_avg_mtp_loss: 13.6323 +[titan] 2025-07-10 08:37:18,204 - root - INFO - lr: 4.2146e-05 gnorm: 1.00 [19:02:14< 2:57:48] +[titan] 2025-07-10 08:37:22,148 - root - INFO - step: 86535 loss: 16.2014 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4749 global_avg_mtp_loss: 13.7264 +[titan] 2025-07-10 08:37:22,148 - root - INFO - lr: 4.2137e-05 gnorm: 0.98 [19:02:18< 2:57:44] +[titan] 2025-07-10 08:37:26,068 - root - INFO - step: 86540 loss: 16.2641 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.7743 +[titan] 2025-07-10 08:37:26,069 - root - INFO - lr: 4.2128e-05 gnorm: 0.97 [19:02:22< 2:57:40] +[titan] 2025-07-10 08:37:30,059 - root - INFO - step: 86545 loss: 16.3467 memory: 44.58GiB(31.99%) tps: 82,117 tflops: 283.40 mfu: 28.66% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.8440 +[titan] 2025-07-10 08:37:30,059 - root - INFO - lr: 4.2119e-05 gnorm: 1.02 [19:02:26< 2:57:36] +[titan] 2025-07-10 08:37:33,174 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:37:33,965 - root - INFO - step: 86550 loss: 16.3447 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5113 global_avg_mtp_loss: 13.8334 +[titan] 2025-07-10 08:37:33,965 - root - INFO - lr: 4.2111e-05 gnorm: 0.98 [19:02:30< 2:57:32] +[titan] 2025-07-10 08:37:37,891 - root - INFO - step: 86555 loss: 16.2944 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.8114 +[titan] 2025-07-10 08:37:37,891 - root - INFO - lr: 4.2102e-05 gnorm: 0.98 [19:02:34< 2:57:28] +[titan] 2025-07-10 08:37:41,825 - root - INFO - step: 86560 loss: 16.1604 memory: 44.58GiB(31.99%) tps: 83,313 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4607 global_avg_mtp_loss: 13.6997 +[titan] 2025-07-10 08:37:41,825 - root - INFO - lr: 4.2093e-05 gnorm: 1.01 [19:02:38< 2:57:24] +[titan] 2025-07-10 08:37:45,736 - root - INFO - step: 86565 loss: 16.4919 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9698 +[titan] 2025-07-10 08:37:45,737 - root - INFO - lr: 4.2084e-05 gnorm: 1.00 [19:02:42< 2:57:20] +[titan] 2025-07-10 08:37:49,651 - root - INFO - step: 86570 loss: 16.1876 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.7150 +[titan] 2025-07-10 08:37:49,652 - root - INFO - lr: 4.2075e-05 gnorm: 0.99 [19:02:46< 2:57:17] +[titan] 2025-07-10 08:37:53,579 - root - INFO - step: 86575 loss: 16.1187 memory: 44.58GiB(31.99%) tps: 83,433 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6509 +[titan] 2025-07-10 08:37:53,580 - root - INFO - lr: 4.2066e-05 gnorm: 1.09 [19:02:50< 2:57:13] +[titan] 2025-07-10 08:37:57,546 - root - INFO - step: 86580 loss: 16.1702 memory: 44.58GiB(31.99%) tps: 82,610 tflops: 285.10 mfu: 28.83% global_avg_ntp_loss: 2.4646 global_avg_mtp_loss: 13.7056 +[titan] 2025-07-10 08:37:57,547 - root - INFO - lr: 4.2057e-05 gnorm: 1.03 [19:02:54< 2:57:09] +[titan] 2025-07-10 08:38:01,451 - root - INFO - step: 86585 loss: 16.1587 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.7006 +[titan] 2025-07-10 08:38:01,451 - root - INFO - lr: 4.2049e-05 gnorm: 0.97 [19:02:58< 2:57:05] +[titan] 2025-07-10 08:38:05,368 - root - INFO - step: 86590 loss: 16.2547 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4849 global_avg_mtp_loss: 13.7699 +[titan] 2025-07-10 08:38:05,369 - root - INFO - lr: 4.2040e-05 gnorm: 0.97 [19:03:01< 2:57:01] +[titan] 2025-07-10 08:38:09,295 - root - INFO - step: 86595 loss: 16.2652 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7839 +[titan] 2025-07-10 08:38:09,295 - root - INFO - lr: 4.2031e-05 gnorm: 0.96 [19:03:05< 2:56:57] +[titan] 2025-07-10 08:38:12,449 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:38:13,243 - root - INFO - step: 86600 loss: 16.1157 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.6722 +[titan] 2025-07-10 08:38:13,243 - root - INFO - lr: 4.2022e-05 gnorm: 1.02 [19:03:09< 2:56:53] +[titan] 2025-07-10 08:38:17,189 - root - INFO - step: 86605 loss: 16.3574 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 08:38:17,189 - root - INFO - lr: 4.2013e-05 gnorm: 0.98 [19:03:13< 2:56:49] +[titan] 2025-07-10 08:38:21,095 - root - INFO - step: 86610 loss: 16.3387 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5323 global_avg_mtp_loss: 13.8064 +[titan] 2025-07-10 08:38:21,095 - root - INFO - lr: 4.2004e-05 gnorm: 1.00 [19:03:17< 2:56:45] +[titan] 2025-07-10 08:38:25,015 - root - INFO - step: 86615 loss: 16.2633 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7870 +[titan] 2025-07-10 08:38:25,015 - root - INFO - lr: 4.1996e-05 gnorm: 1.01 [19:03:21< 2:56:41] +[titan] 2025-07-10 08:38:28,962 - root - INFO - step: 86620 loss: 16.2843 memory: 44.58GiB(31.99%) tps: 83,039 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.7757 +[titan] 2025-07-10 08:38:28,962 - root - INFO - lr: 4.1987e-05 gnorm: 1.03 [19:03:25< 2:56:37] +[titan] 2025-07-10 08:38:32,875 - root - INFO - step: 86625 loss: 16.3294 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8378 +[titan] 2025-07-10 08:38:32,876 - root - INFO - lr: 4.1978e-05 gnorm: 1.03 [19:03:29< 2:56:33] +[titan] 2025-07-10 08:38:36,791 - root - INFO - step: 86630 loss: 15.9850 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4391 global_avg_mtp_loss: 13.5458 +[titan] 2025-07-10 08:38:36,791 - root - INFO - lr: 4.1969e-05 gnorm: 0.97 [19:03:33< 2:56:29] +[titan] 2025-07-10 08:38:40,706 - root - INFO - step: 86635 loss: 15.9328 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4282 global_avg_mtp_loss: 13.5046 +[titan] 2025-07-10 08:38:40,706 - root - INFO - lr: 4.1960e-05 gnorm: 1.04 [19:03:37< 2:56:25] +[titan] 2025-07-10 08:38:44,640 - root - INFO - step: 86640 loss: 16.0684 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4434 global_avg_mtp_loss: 13.6250 +[titan] 2025-07-10 08:38:44,641 - root - INFO - lr: 4.1952e-05 gnorm: 1.00 [19:03:41< 2:56:21] +[titan] 2025-07-10 08:38:48,553 - root - INFO - step: 86645 loss: 16.4074 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5143 global_avg_mtp_loss: 13.8931 +[titan] 2025-07-10 08:38:48,553 - root - INFO - lr: 4.1943e-05 gnorm: 1.03 [19:03:45< 2:56:17] +[titan] 2025-07-10 08:38:51,665 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:38:52,498 - root - INFO - step: 86650 loss: 16.3750 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-10 08:38:52,498 - root - INFO - lr: 4.1934e-05 gnorm: 0.99 [19:03:49< 2:56:13] +[titan] 2025-07-10 08:38:56,426 - root - INFO - step: 86655 loss: 16.1957 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7217 +[titan] 2025-07-10 08:38:56,426 - root - INFO - lr: 4.1925e-05 gnorm: 1.03 [19:03:53< 2:56:09] +[titan] 2025-07-10 08:39:00,331 - root - INFO - step: 86660 loss: 16.4360 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.9243 +[titan] 2025-07-10 08:39:00,331 - root - INFO - lr: 4.1916e-05 gnorm: 1.04 [19:03:56< 2:56:05] +[titan] 2025-07-10 08:39:04,222 - root - INFO - step: 86665 loss: 16.2869 memory: 44.58GiB(31.99%) tps: 84,233 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.8026 +[titan] 2025-07-10 08:39:04,222 - root - INFO - lr: 4.1907e-05 gnorm: 1.01 [19:04:00< 2:56:01] +[titan] 2025-07-10 08:39:08,135 - root - INFO - step: 86670 loss: 16.2227 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7474 +[titan] 2025-07-10 08:39:08,135 - root - INFO - lr: 4.1899e-05 gnorm: 1.03 [19:04:04< 2:55:57] +[titan] 2025-07-10 08:39:12,049 - root - INFO - step: 86675 loss: 15.9065 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4196 global_avg_mtp_loss: 13.4870 +[titan] 2025-07-10 08:39:12,049 - root - INFO - lr: 4.1890e-05 gnorm: 1.07 [19:04:08< 2:55:53] +[titan] 2025-07-10 08:39:15,987 - root - INFO - step: 86680 loss: 16.1371 memory: 44.58GiB(31.99%) tps: 83,217 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4622 global_avg_mtp_loss: 13.6749 +[titan] 2025-07-10 08:39:15,987 - root - INFO - lr: 4.1881e-05 gnorm: 1.08 [19:04:12< 2:55:49] +[titan] 2025-07-10 08:39:19,944 - root - INFO - step: 86685 loss: 16.1346 memory: 44.58GiB(31.99%) tps: 82,816 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6728 +[titan] 2025-07-10 08:39:19,945 - root - INFO - lr: 4.1872e-05 gnorm: 1.04 [19:04:16< 2:55:45] +[titan] 2025-07-10 08:39:23,914 - root - INFO - step: 86690 loss: 15.9692 memory: 44.58GiB(31.99%) tps: 82,553 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.4413 global_avg_mtp_loss: 13.5279 +[titan] 2025-07-10 08:39:23,914 - root - INFO - lr: 4.1864e-05 gnorm: 1.03 [19:04:20< 2:55:41] +[titan] 2025-07-10 08:39:27,838 - root - INFO - step: 86695 loss: 16.1729 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.6960 +[titan] 2025-07-10 08:39:27,838 - root - INFO - lr: 4.1855e-05 gnorm: 1.04 [19:04:24< 2:55:37] +[titan] 2025-07-10 08:39:30,954 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:39:31,742 - root - INFO - step: 86700 loss: 16.3897 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8958 +[titan] 2025-07-10 08:39:31,742 - root - INFO - lr: 4.1846e-05 gnorm: 0.98 [19:04:28< 2:55:33] +[titan] 2025-07-10 08:39:35,651 - root - INFO - step: 86705 loss: 16.1811 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7028 +[titan] 2025-07-10 08:39:35,651 - root - INFO - lr: 4.1837e-05 gnorm: 0.99 [19:04:32< 2:55:29] +[titan] 2025-07-10 08:39:39,565 - root - INFO - step: 86710 loss: 16.0922 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6384 +[titan] 2025-07-10 08:39:39,565 - root - INFO - lr: 4.1828e-05 gnorm: 0.98 [19:04:36< 2:55:25] +[titan] 2025-07-10 08:39:43,485 - root - INFO - step: 86715 loss: 16.3481 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.5151 global_avg_mtp_loss: 13.8330 +[titan] 2025-07-10 08:39:43,485 - root - INFO - lr: 4.1820e-05 gnorm: 0.96 [19:04:40< 2:55:21] +[titan] 2025-07-10 08:39:47,398 - root - INFO - step: 86720 loss: 16.2690 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.7927 +[titan] 2025-07-10 08:39:47,398 - root - INFO - lr: 4.1811e-05 gnorm: 0.96 [19:04:44< 2:55:18] +[titan] 2025-07-10 08:39:51,342 - root - INFO - step: 86725 loss: 16.2179 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.7480 +[titan] 2025-07-10 08:39:51,342 - root - INFO - lr: 4.1802e-05 gnorm: 0.99 [19:04:47< 2:55:14] +[titan] 2025-07-10 08:39:55,238 - root - INFO - step: 86730 loss: 16.0164 memory: 44.58GiB(31.99%) tps: 84,112 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5738 +[titan] 2025-07-10 08:39:55,238 - root - INFO - lr: 4.1793e-05 gnorm: 1.01 [19:04:51< 2:55:10] +[titan] 2025-07-10 08:39:59,163 - root - INFO - step: 86735 loss: 16.0020 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5618 +[titan] 2025-07-10 08:39:59,164 - root - INFO - lr: 4.1785e-05 gnorm: 1.01 [19:04:55< 2:55:06] +[titan] 2025-07-10 08:40:03,085 - root - INFO - step: 86740 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.6993 +[titan] 2025-07-10 08:40:03,085 - root - INFO - lr: 4.1776e-05 gnorm: 1.03 [19:04:59< 2:55:02] +[titan] 2025-07-10 08:40:07,019 - root - INFO - step: 86745 loss: 16.2885 memory: 44.58GiB(31.99%) tps: 83,291 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.8003 +[titan] 2025-07-10 08:40:07,020 - root - INFO - lr: 4.1767e-05 gnorm: 1.05 [19:05:03< 2:54:58] +[titan] 2025-07-10 08:40:10,147 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:40:10,946 - root - INFO - step: 86750 loss: 16.0960 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6474 +[titan] 2025-07-10 08:40:10,946 - root - INFO - lr: 4.1758e-05 gnorm: 0.96 [19:05:07< 2:54:54] +[titan] 2025-07-10 08:40:14,895 - root - INFO - step: 86755 loss: 16.0130 memory: 44.58GiB(31.99%) tps: 82,982 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4474 global_avg_mtp_loss: 13.5656 +[titan] 2025-07-10 08:40:14,895 - root - INFO - lr: 4.1750e-05 gnorm: 0.99 [19:05:11< 2:54:50] +[titan] 2025-07-10 08:40:18,801 - root - INFO - step: 86760 loss: 16.1806 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.6987 +[titan] 2025-07-10 08:40:18,802 - root - INFO - lr: 4.1741e-05 gnorm: 1.08 [19:05:15< 2:54:46] +[titan] 2025-07-10 08:40:22,741 - root - INFO - step: 86765 loss: 16.4886 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9624 +[titan] 2025-07-10 08:40:22,742 - root - INFO - lr: 4.1732e-05 gnorm: 0.98 [19:05:19< 2:54:42] +[titan] 2025-07-10 08:40:26,696 - root - INFO - step: 86770 loss: 16.4545 memory: 44.58GiB(31.99%) tps: 82,873 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.5164 global_avg_mtp_loss: 13.9381 +[titan] 2025-07-10 08:40:26,696 - root - INFO - lr: 4.1723e-05 gnorm: 1.00 [19:05:23< 2:54:38] +[titan] 2025-07-10 08:40:30,644 - root - INFO - step: 86775 loss: 16.3887 memory: 44.58GiB(31.99%) tps: 83,007 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8884 +[titan] 2025-07-10 08:40:30,644 - root - INFO - lr: 4.1715e-05 gnorm: 1.06 [19:05:27< 2:54:34] +[titan] 2025-07-10 08:40:34,575 - root - INFO - step: 86780 loss: 16.0137 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4371 global_avg_mtp_loss: 13.5766 +[titan] 2025-07-10 08:40:34,575 - root - INFO - lr: 4.1706e-05 gnorm: 0.99 [19:05:31< 2:54:30] +[titan] 2025-07-10 08:40:38,479 - root - INFO - step: 86785 loss: 16.2316 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7536 +[titan] 2025-07-10 08:40:38,479 - root - INFO - lr: 4.1697e-05 gnorm: 1.00 [19:05:35< 2:54:26] +[titan] 2025-07-10 08:40:42,414 - root - INFO - step: 86790 loss: 16.4228 memory: 44.58GiB(31.99%) tps: 83,284 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.9116 +[titan] 2025-07-10 08:40:42,414 - root - INFO - lr: 4.1689e-05 gnorm: 1.00 [19:05:39< 2:54:22] +[titan] 2025-07-10 08:40:46,328 - root - INFO - step: 86795 loss: 16.1236 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.6723 +[titan] 2025-07-10 08:40:46,328 - root - INFO - lr: 4.1680e-05 gnorm: 1.00 [19:05:42< 2:54:18] +[titan] 2025-07-10 08:40:49,443 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:40:50,228 - root - INFO - step: 86800 loss: 16.3564 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.5009 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 08:40:50,228 - root - INFO - lr: 4.1671e-05 gnorm: 0.97 [19:05:46< 2:54:14] +[titan] 2025-07-10 08:40:54,142 - root - INFO - step: 86805 loss: 16.0132 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4342 global_avg_mtp_loss: 13.5790 +[titan] 2025-07-10 08:40:54,143 - root - INFO - lr: 4.1662e-05 gnorm: 0.98 [19:05:50< 2:54:10] +[titan] 2025-07-10 08:40:58,038 - root - INFO - step: 86810 loss: 16.1279 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4625 global_avg_mtp_loss: 13.6654 +[titan] 2025-07-10 08:40:58,039 - root - INFO - lr: 4.1654e-05 gnorm: 1.00 [19:05:54< 2:54:06] +[titan] 2025-07-10 08:41:02,009 - root - INFO - step: 86815 loss: 15.9601 memory: 44.58GiB(31.99%) tps: 82,537 tflops: 284.85 mfu: 28.80% global_avg_ntp_loss: 2.4187 global_avg_mtp_loss: 13.5414 +[titan] 2025-07-10 08:41:02,009 - root - INFO - lr: 4.1645e-05 gnorm: 0.99 [19:05:58< 2:54:02] +[titan] 2025-07-10 08:41:05,912 - root - INFO - step: 86820 loss: 16.2827 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.8005 +[titan] 2025-07-10 08:41:05,912 - root - INFO - lr: 4.1636e-05 gnorm: 1.00 [19:06:02< 2:53:58] +[titan] 2025-07-10 08:41:09,844 - root - INFO - step: 86825 loss: 15.9654 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4277 global_avg_mtp_loss: 13.5377 +[titan] 2025-07-10 08:41:09,844 - root - INFO - lr: 4.1628e-05 gnorm: 1.09 [19:06:06< 2:53:54] +[titan] 2025-07-10 08:41:13,761 - root - INFO - step: 86830 loss: 16.2679 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7877 +[titan] 2025-07-10 08:41:13,761 - root - INFO - lr: 4.1619e-05 gnorm: 1.02 [19:06:10< 2:53:50] +[titan] 2025-07-10 08:41:17,668 - root - INFO - step: 86835 loss: 16.1904 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7141 +[titan] 2025-07-10 08:41:17,668 - root - INFO - lr: 4.1610e-05 gnorm: 0.97 [19:06:14< 2:53:46] +[titan] 2025-07-10 08:41:21,616 - root - INFO - step: 86840 loss: 16.1382 memory: 44.58GiB(31.99%) tps: 83,014 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6801 +[titan] 2025-07-10 08:41:21,616 - root - INFO - lr: 4.1602e-05 gnorm: 0.97 [19:06:18< 2:53:42] +[titan] 2025-07-10 08:41:25,537 - root - INFO - step: 86845 loss: 16.1683 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7018 +[titan] 2025-07-10 08:41:25,538 - root - INFO - lr: 4.1593e-05 gnorm: 0.99 [19:06:22< 2:53:38] +[titan] 2025-07-10 08:41:28,687 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:41:29,491 - root - INFO - step: 86850 loss: 16.4138 memory: 44.58GiB(31.99%) tps: 82,886 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.8997 +[titan] 2025-07-10 08:41:29,491 - root - INFO - lr: 4.1584e-05 gnorm: 0.97 [19:06:26< 2:53:34] +[titan] 2025-07-10 08:41:33,396 - root - INFO - step: 86855 loss: 16.3008 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.8016 +[titan] 2025-07-10 08:41:33,397 - root - INFO - lr: 4.1575e-05 gnorm: 1.07 [19:06:29< 2:53:30] +[titan] 2025-07-10 08:41:37,317 - root - INFO - step: 86860 loss: 16.2146 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7402 +[titan] 2025-07-10 08:41:37,318 - root - INFO - lr: 4.1567e-05 gnorm: 1.03 [19:06:33< 2:53:27] +[titan] 2025-07-10 08:41:41,258 - root - INFO - step: 86865 loss: 16.2028 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7334 +[titan] 2025-07-10 08:41:41,259 - root - INFO - lr: 4.1558e-05 gnorm: 1.01 [19:06:37< 2:53:23] +[titan] 2025-07-10 08:41:45,219 - root - INFO - step: 86870 loss: 16.4120 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5138 global_avg_mtp_loss: 13.8982 +[titan] 2025-07-10 08:41:45,219 - root - INFO - lr: 4.1549e-05 gnorm: 0.97 [19:06:41< 2:53:19] +[titan] 2025-07-10 08:41:49,170 - root - INFO - step: 86875 loss: 15.9891 memory: 44.58GiB(31.99%) tps: 82,931 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4373 global_avg_mtp_loss: 13.5518 +[titan] 2025-07-10 08:41:49,171 - root - INFO - lr: 4.1541e-05 gnorm: 0.98 [19:06:45< 2:53:15] +[titan] 2025-07-10 08:41:53,082 - root - INFO - step: 86880 loss: 16.4359 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.9117 +[titan] 2025-07-10 08:41:53,082 - root - INFO - lr: 4.1532e-05 gnorm: 0.99 [19:06:49< 2:53:11] +[titan] 2025-07-10 08:41:56,987 - root - INFO - step: 86885 loss: 16.4518 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.9350 +[titan] 2025-07-10 08:41:56,987 - root - INFO - lr: 4.1523e-05 gnorm: 1.00 [19:06:53< 2:53:07] +[titan] 2025-07-10 08:42:00,961 - root - INFO - step: 86890 loss: 16.3766 memory: 44.58GiB(31.99%) tps: 82,454 tflops: 284.56 mfu: 28.77% global_avg_ntp_loss: 2.4987 global_avg_mtp_loss: 13.8779 +[titan] 2025-07-10 08:42:00,962 - root - INFO - lr: 4.1515e-05 gnorm: 1.03 [19:06:57< 2:53:03] +[titan] 2025-07-10 08:42:04,905 - root - INFO - step: 86895 loss: 16.1304 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.6581 +[titan] 2025-07-10 08:42:04,905 - root - INFO - lr: 4.1506e-05 gnorm: 1.03 [19:07:01< 2:52:59] +[titan] 2025-07-10 08:42:08,033 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:42:08,843 - root - INFO - step: 86900 loss: 15.9289 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4183 global_avg_mtp_loss: 13.5106 +[titan] 2025-07-10 08:42:08,844 - root - INFO - lr: 4.1498e-05 gnorm: 1.04 [19:07:05< 2:52:55] +[titan] 2025-07-10 08:42:12,792 - root - INFO - step: 86905 loss: 16.3847 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.8655 +[titan] 2025-07-10 08:42:12,793 - root - INFO - lr: 4.1489e-05 gnorm: 1.00 [19:07:09< 2:52:51] +[titan] 2025-07-10 08:42:16,712 - root - INFO - step: 86910 loss: 15.9859 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4309 global_avg_mtp_loss: 13.5550 +[titan] 2025-07-10 08:42:16,712 - root - INFO - lr: 4.1480e-05 gnorm: 1.04 [19:07:13< 2:52:47] +[titan] 2025-07-10 08:42:20,633 - root - INFO - step: 86915 loss: 16.0694 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.6261 +[titan] 2025-07-10 08:42:20,633 - root - INFO - lr: 4.1472e-05 gnorm: 1.01 [19:07:17< 2:52:43] +[titan] 2025-07-10 08:42:24,540 - root - INFO - step: 86920 loss: 16.2115 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4703 global_avg_mtp_loss: 13.7413 +[titan] 2025-07-10 08:42:24,540 - root - INFO - lr: 4.1463e-05 gnorm: 0.98 [19:07:21< 2:52:39] +[titan] 2025-07-10 08:42:28,469 - root - INFO - step: 86925 loss: 16.0452 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4417 global_avg_mtp_loss: 13.6034 +[titan] 2025-07-10 08:42:28,469 - root - INFO - lr: 4.1454e-05 gnorm: 1.04 [19:07:25< 2:52:35] +[titan] 2025-07-10 08:42:32,399 - root - INFO - step: 86930 loss: 16.1052 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.6485 +[titan] 2025-07-10 08:42:32,399 - root - INFO - lr: 4.1446e-05 gnorm: 1.03 [19:07:28< 2:52:31] +[titan] 2025-07-10 08:42:36,340 - root - INFO - step: 86935 loss: 16.1562 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.6890 +[titan] 2025-07-10 08:42:36,340 - root - INFO - lr: 4.1437e-05 gnorm: 0.97 [19:07:32< 2:52:27] +[titan] 2025-07-10 08:42:40,260 - root - INFO - step: 86940 loss: 16.2771 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.8008 +[titan] 2025-07-10 08:42:40,260 - root - INFO - lr: 4.1428e-05 gnorm: 1.00 [19:07:36< 2:52:23] +[titan] 2025-07-10 08:42:44,165 - root - INFO - step: 86945 loss: 16.2380 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7659 +[titan] 2025-07-10 08:42:44,165 - root - INFO - lr: 4.1420e-05 gnorm: 0.96 [19:07:40< 2:52:19] +[titan] 2025-07-10 08:42:47,291 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:42:48,095 - root - INFO - step: 86950 loss: 16.1393 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6774 +[titan] 2025-07-10 08:42:48,095 - root - INFO - lr: 4.1411e-05 gnorm: 0.98 [19:07:44< 2:52:15] +[titan] 2025-07-10 08:42:52,004 - root - INFO - step: 86955 loss: 16.1669 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.7067 +[titan] 2025-07-10 08:42:52,004 - root - INFO - lr: 4.1403e-05 gnorm: 1.01 [19:07:48< 2:52:11] +[titan] 2025-07-10 08:42:55,907 - root - INFO - step: 86960 loss: 16.0721 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.6186 +[titan] 2025-07-10 08:42:55,908 - root - INFO - lr: 4.1394e-05 gnorm: 1.06 [19:07:52< 2:52:07] +[titan] 2025-07-10 08:42:59,826 - root - INFO - step: 86965 loss: 16.2027 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7320 +[titan] 2025-07-10 08:42:59,826 - root - INFO - lr: 4.1385e-05 gnorm: 1.00 [19:07:56< 2:52:03] +[titan] 2025-07-10 08:43:03,736 - root - INFO - step: 86970 loss: 16.1346 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.6618 +[titan] 2025-07-10 08:43:03,736 - root - INFO - lr: 4.1377e-05 gnorm: 1.02 [19:08:00< 2:51:59] +[titan] 2025-07-10 08:43:07,645 - root - INFO - step: 86975 loss: 16.0524 memory: 44.58GiB(31.99%) tps: 83,844 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6004 +[titan] 2025-07-10 08:43:07,645 - root - INFO - lr: 4.1368e-05 gnorm: 1.00 [19:08:04< 2:51:55] +[titan] 2025-07-10 08:43:11,542 - root - INFO - step: 86980 loss: 16.2297 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.7527 +[titan] 2025-07-10 08:43:11,542 - root - INFO - lr: 4.1360e-05 gnorm: 0.94 [19:08:08< 2:51:51] +[titan] 2025-07-10 08:43:15,443 - root - INFO - step: 86985 loss: 15.8597 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4078 global_avg_mtp_loss: 13.4519 +[titan] 2025-07-10 08:43:15,443 - root - INFO - lr: 4.1351e-05 gnorm: 1.01 [19:08:12< 2:51:47] +[titan] 2025-07-10 08:43:19,353 - root - INFO - step: 86990 loss: 16.3153 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.8288 +[titan] 2025-07-10 08:43:19,353 - root - INFO - lr: 4.1342e-05 gnorm: 0.96 [19:08:15< 2:51:43] +[titan] 2025-07-10 08:43:23,256 - root - INFO - step: 86995 loss: 16.2521 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.7794 +[titan] 2025-07-10 08:43:23,256 - root - INFO - lr: 4.1334e-05 gnorm: 0.97 [19:08:19< 2:51:39] +[titan] 2025-07-10 08:43:26,402 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:43:27,192 - root - INFO - step: 87000 loss: 16.2918 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.06% global_avg_ntp_loss: 2.4991 global_avg_mtp_loss: 13.7927 +[titan] 2025-07-10 08:43:27,192 - root - INFO - lr: 4.1325e-05 gnorm: 0.99 [19:08:23< 2:51:35] +[titan] 2025-07-10 08:43:31,122 - root - INFO - step: 87005 loss: 16.0717 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4539 global_avg_mtp_loss: 13.6179 +[titan] 2025-07-10 08:43:31,123 - root - INFO - lr: 4.1317e-05 gnorm: 0.99 [19:08:27< 2:51:32] +[titan] 2025-07-10 08:43:35,063 - root - INFO - step: 87010 loss: 16.2584 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7784 +[titan] 2025-07-10 08:43:35,063 - root - INFO - lr: 4.1308e-05 gnorm: 0.97 [19:08:31< 2:51:28] +[titan] 2025-07-10 08:43:38,994 - root - INFO - step: 87015 loss: 16.2792 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7975 +[titan] 2025-07-10 08:43:38,995 - root - INFO - lr: 4.1299e-05 gnorm: 0.97 [19:08:35< 2:51:24] +[titan] 2025-07-10 08:43:42,935 - root - INFO - step: 87020 loss: 16.1012 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.6401 +[titan] 2025-07-10 08:43:42,936 - root - INFO - lr: 4.1291e-05 gnorm: 0.99 [19:08:39< 2:51:20] +[titan] 2025-07-10 08:43:46,847 - root - INFO - step: 87025 loss: 16.3610 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8546 +[titan] 2025-07-10 08:43:46,848 - root - INFO - lr: 4.1282e-05 gnorm: 1.09 [19:08:43< 2:51:16] +[titan] 2025-07-10 08:43:50,764 - root - INFO - step: 87030 loss: 16.4091 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.8967 +[titan] 2025-07-10 08:43:50,764 - root - INFO - lr: 4.1274e-05 gnorm: 1.03 [19:08:47< 2:51:12] +[titan] 2025-07-10 08:43:54,708 - root - INFO - step: 87035 loss: 16.3388 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.8445 +[titan] 2025-07-10 08:43:54,709 - root - INFO - lr: 4.1265e-05 gnorm: 1.01 [19:08:51< 2:51:08] +[titan] 2025-07-10 08:43:58,715 - root - INFO - step: 87040 loss: 16.4714 memory: 44.58GiB(31.99%) tps: 81,794 tflops: 282.28 mfu: 28.54% global_avg_ntp_loss: 2.5160 global_avg_mtp_loss: 13.9555 +[titan] 2025-07-10 08:43:58,715 - root - INFO - lr: 4.1257e-05 gnorm: 1.02 [19:08:55< 2:51:04] +[titan] 2025-07-10 08:43:58,876 - root - INFO - Dumping profiler traces at step 87040 +[titan] 2025-07-10 08:43:58,928 - root - INFO - Finished dumping profiler traces in 0.05 seconds +[titan] 2025-07-10 08:44:02,820 - root - INFO - step: 87045 loss: 16.0218 memory: 44.58GiB(31.99%) tps: 79,824 tflops: 275.49 mfu: 27.86% global_avg_ntp_loss: 2.4497 global_avg_mtp_loss: 13.5721 +[titan] 2025-07-10 08:44:02,821 - root - INFO - lr: 4.1248e-05 gnorm: 0.98 [19:08:59< 2:51:00] +[titan] 2025-07-10 08:44:05,953 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:44:06,740 - root - INFO - step: 87050 loss: 16.2150 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.7434 +[titan] 2025-07-10 08:44:06,740 - root - INFO - lr: 4.1239e-05 gnorm: 1.00 [19:09:03< 2:50:56] +[titan] 2025-07-10 08:44:10,663 - root - INFO - step: 87055 loss: 16.2294 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7587 +[titan] 2025-07-10 08:44:10,663 - root - INFO - lr: 4.1231e-05 gnorm: 1.00 [19:09:07< 2:50:52] +[titan] 2025-07-10 08:44:14,574 - root - INFO - step: 87060 loss: 15.9183 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4245 global_avg_mtp_loss: 13.4938 +[titan] 2025-07-10 08:44:14,574 - root - INFO - lr: 4.1222e-05 gnorm: 0.99 [19:09:11< 2:50:48] +[titan] 2025-07-10 08:44:18,481 - root - INFO - step: 87065 loss: 16.5033 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5313 global_avg_mtp_loss: 13.9720 +[titan] 2025-07-10 08:44:18,481 - root - INFO - lr: 4.1214e-05 gnorm: 1.00 [19:09:15< 2:50:44] +[titan] 2025-07-10 08:44:22,404 - root - INFO - step: 87070 loss: 16.4671 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.9587 +[titan] 2025-07-10 08:44:22,404 - root - INFO - lr: 4.1205e-05 gnorm: 1.01 [19:09:18< 2:50:40] +[titan] 2025-07-10 08:44:26,316 - root - INFO - step: 87075 loss: 16.3152 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8086 +[titan] 2025-07-10 08:44:26,317 - root - INFO - lr: 4.1197e-05 gnorm: 1.03 [19:09:22< 2:50:36] +[titan] 2025-07-10 08:44:30,234 - root - INFO - step: 87080 loss: 16.1733 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.7206 +[titan] 2025-07-10 08:44:30,234 - root - INFO - lr: 4.1188e-05 gnorm: 0.96 [19:09:26< 2:50:32] +[titan] 2025-07-10 08:44:34,155 - root - INFO - step: 87085 loss: 16.4017 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5033 global_avg_mtp_loss: 13.8984 +[titan] 2025-07-10 08:44:34,155 - root - INFO - lr: 4.1180e-05 gnorm: 1.04 [19:09:30< 2:50:28] +[titan] 2025-07-10 08:44:38,068 - root - INFO - step: 87090 loss: 16.2672 memory: 44.58GiB(31.99%) tps: 83,738 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 08:44:38,068 - root - INFO - lr: 4.1171e-05 gnorm: 0.99 [19:09:34< 2:50:24] +[titan] 2025-07-10 08:44:42,018 - root - INFO - step: 87095 loss: 16.0337 memory: 44.58GiB(31.99%) tps: 82,977 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4476 global_avg_mtp_loss: 13.5861 +[titan] 2025-07-10 08:44:42,018 - root - INFO - lr: 4.1163e-05 gnorm: 1.01 [19:09:38< 2:50:20] +[titan] 2025-07-10 08:44:45,124 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:44:45,912 - root - INFO - step: 87100 loss: 16.1930 memory: 44.58GiB(31.99%) tps: 84,141 tflops: 290.39 mfu: 29.36% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 08:44:45,913 - root - INFO - lr: 4.1154e-05 gnorm: 0.99 [19:09:42< 2:50:16] +[titan] 2025-07-10 08:44:49,816 - root - INFO - step: 87105 loss: 16.3289 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8313 +[titan] 2025-07-10 08:44:49,816 - root - INFO - lr: 4.1145e-05 gnorm: 0.98 [19:09:46< 2:50:12] +[titan] 2025-07-10 08:44:53,730 - root - INFO - step: 87110 loss: 16.3076 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.8216 +[titan] 2025-07-10 08:44:53,730 - root - INFO - lr: 4.1137e-05 gnorm: 0.96 [19:09:50< 2:50:08] +[titan] 2025-07-10 08:44:57,698 - root - INFO - step: 87115 loss: 16.2198 memory: 44.58GiB(31.99%) tps: 82,596 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7351 +[titan] 2025-07-10 08:44:57,698 - root - INFO - lr: 4.1128e-05 gnorm: 1.01 [19:09:54< 2:50:04] +[titan] 2025-07-10 08:45:01,596 - root - INFO - step: 87120 loss: 16.4027 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.9015 +[titan] 2025-07-10 08:45:01,596 - root - INFO - lr: 4.1120e-05 gnorm: 1.00 [19:09:58< 2:50:00] +[titan] 2025-07-10 08:45:05,506 - root - INFO - step: 87125 loss: 16.3115 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8182 +[titan] 2025-07-10 08:45:05,507 - root - INFO - lr: 4.1111e-05 gnorm: 0.96 [19:10:02< 2:49:56] +[titan] 2025-07-10 08:45:09,409 - root - INFO - step: 87130 loss: 16.3774 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5121 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 08:45:09,410 - root - INFO - lr: 4.1103e-05 gnorm: 0.99 [19:10:05< 2:49:52] +[titan] 2025-07-10 08:45:13,335 - root - INFO - step: 87135 loss: 16.3096 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.8062 +[titan] 2025-07-10 08:45:13,335 - root - INFO - lr: 4.1094e-05 gnorm: 1.01 [19:10:09< 2:49:48] +[titan] 2025-07-10 08:45:17,231 - root - INFO - step: 87140 loss: 16.3126 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.8223 +[titan] 2025-07-10 08:45:17,231 - root - INFO - lr: 4.1086e-05 gnorm: 1.05 [19:10:13< 2:49:44] +[titan] 2025-07-10 08:45:21,130 - root - INFO - step: 87145 loss: 16.2241 memory: 44.58GiB(31.99%) tps: 84,049 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7502 +[titan] 2025-07-10 08:45:21,130 - root - INFO - lr: 4.1077e-05 gnorm: 1.01 [19:10:17< 2:49:40] +[titan] 2025-07-10 08:45:24,231 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:45:25,018 - root - INFO - step: 87150 loss: 16.2375 memory: 44.58GiB(31.99%) tps: 84,283 tflops: 290.88 mfu: 29.41% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7579 +[titan] 2025-07-10 08:45:25,019 - root - INFO - lr: 4.1069e-05 gnorm: 1.01 [19:10:21< 2:49:37] +[titan] 2025-07-10 08:45:28,931 - root - INFO - step: 87155 loss: 16.1037 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6447 +[titan] 2025-07-10 08:45:28,931 - root - INFO - lr: 4.1060e-05 gnorm: 1.02 [19:10:25< 2:49:33] +[titan] 2025-07-10 08:45:32,864 - root - INFO - step: 87160 loss: 16.0877 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4485 global_avg_mtp_loss: 13.6391 +[titan] 2025-07-10 08:45:32,865 - root - INFO - lr: 4.1052e-05 gnorm: 0.99 [19:10:29< 2:49:29] +[titan] 2025-07-10 08:45:36,788 - root - INFO - step: 87165 loss: 16.2634 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7868 +[titan] 2025-07-10 08:45:36,788 - root - INFO - lr: 4.1043e-05 gnorm: 1.00 [19:10:33< 2:49:25] +[titan] 2025-07-10 08:45:40,724 - root - INFO - step: 87170 loss: 16.3355 memory: 44.58GiB(31.99%) tps: 83,248 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8415 +[titan] 2025-07-10 08:45:40,724 - root - INFO - lr: 4.1035e-05 gnorm: 0.98 [19:10:37< 2:49:21] +[titan] 2025-07-10 08:45:44,619 - root - INFO - step: 87175 loss: 16.0857 memory: 44.58GiB(31.99%) tps: 84,136 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6330 +[titan] 2025-07-10 08:45:44,619 - root - INFO - lr: 4.1026e-05 gnorm: 0.97 [19:10:41< 2:49:17] +[titan] 2025-07-10 08:45:48,544 - root - INFO - step: 87180 loss: 16.0418 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4363 global_avg_mtp_loss: 13.6055 +[titan] 2025-07-10 08:45:48,544 - root - INFO - lr: 4.1018e-05 gnorm: 0.97 [19:10:45< 2:49:13] +[titan] 2025-07-10 08:45:52,472 - root - INFO - step: 87185 loss: 16.4149 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5124 global_avg_mtp_loss: 13.9026 +[titan] 2025-07-10 08:45:52,472 - root - INFO - lr: 4.1010e-05 gnorm: 1.00 [19:10:49< 2:49:09] +[titan] 2025-07-10 08:45:56,388 - root - INFO - step: 87190 loss: 16.3427 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-10 08:45:56,388 - root - INFO - lr: 4.1001e-05 gnorm: 1.02 [19:10:52< 2:49:05] +[titan] 2025-07-10 08:46:00,315 - root - INFO - step: 87195 loss: 16.0741 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4523 global_avg_mtp_loss: 13.6218 +[titan] 2025-07-10 08:46:00,316 - root - INFO - lr: 4.0993e-05 gnorm: 1.00 [19:10:56< 2:49:01] +[titan] 2025-07-10 08:46:03,430 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:46:04,233 - root - INFO - step: 87200 loss: 16.1166 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.6634 +[titan] 2025-07-10 08:46:04,233 - root - INFO - lr: 4.0984e-05 gnorm: 1.03 [19:11:00< 2:48:57] +[titan] 2025-07-10 08:46:08,173 - root - INFO - step: 87205 loss: 16.2346 memory: 44.58GiB(31.99%) tps: 83,171 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7550 +[titan] 2025-07-10 08:46:08,174 - root - INFO - lr: 4.0976e-05 gnorm: 0.99 [19:11:04< 2:48:53] +[titan] 2025-07-10 08:46:12,096 - root - INFO - step: 87210 loss: 16.5297 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5383 global_avg_mtp_loss: 13.9914 +[titan] 2025-07-10 08:46:12,096 - root - INFO - lr: 4.0967e-05 gnorm: 1.03 [19:11:08< 2:48:49] +[titan] 2025-07-10 08:46:16,064 - root - INFO - step: 87215 loss: 16.1452 memory: 44.58GiB(31.99%) tps: 82,579 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.6770 +[titan] 2025-07-10 08:46:16,065 - root - INFO - lr: 4.0959e-05 gnorm: 1.02 [19:11:12< 2:48:45] +[titan] 2025-07-10 08:46:20,000 - root - INFO - step: 87220 loss: 16.4327 memory: 44.58GiB(31.99%) tps: 83,267 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.9070 +[titan] 2025-07-10 08:46:20,000 - root - INFO - lr: 4.0950e-05 gnorm: 1.05 [19:11:16< 2:48:41] +[titan] 2025-07-10 08:46:23,941 - root - INFO - step: 87225 loss: 16.0064 memory: 44.58GiB(31.99%) tps: 83,154 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4368 global_avg_mtp_loss: 13.5696 +[titan] 2025-07-10 08:46:23,941 - root - INFO - lr: 4.0942e-05 gnorm: 1.09 [19:11:20< 2:48:37] +[titan] 2025-07-10 08:46:27,866 - root - INFO - step: 87230 loss: 16.1468 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.6642 +[titan] 2025-07-10 08:46:27,866 - root - INFO - lr: 4.0933e-05 gnorm: 0.98 [19:11:24< 2:48:33] +[titan] 2025-07-10 08:46:31,797 - root - INFO - step: 87235 loss: 16.3453 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8472 +[titan] 2025-07-10 08:46:31,797 - root - INFO - lr: 4.0925e-05 gnorm: 1.08 [19:11:28< 2:48:29] +[titan] 2025-07-10 08:46:35,707 - root - INFO - step: 87240 loss: 16.2067 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.7176 +[titan] 2025-07-10 08:46:35,707 - root - INFO - lr: 4.0917e-05 gnorm: 1.00 [19:11:32< 2:48:25] +[titan] 2025-07-10 08:46:39,601 - root - INFO - step: 87245 loss: 16.4069 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.8956 +[titan] 2025-07-10 08:46:39,601 - root - INFO - lr: 4.0908e-05 gnorm: 0.98 [19:11:36< 2:48:21] +[titan] 2025-07-10 08:46:42,711 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:46:43,510 - root - INFO - step: 87250 loss: 16.1519 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.6935 +[titan] 2025-07-10 08:46:43,510 - root - INFO - lr: 4.0900e-05 gnorm: 0.96 [19:11:40< 2:48:17] +[titan] 2025-07-10 08:46:47,409 - root - INFO - step: 87255 loss: 16.1628 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6998 +[titan] 2025-07-10 08:46:47,409 - root - INFO - lr: 4.0891e-05 gnorm: 1.01 [19:11:43< 2:48:13] +[titan] 2025-07-10 08:46:51,330 - root - INFO - step: 87260 loss: 16.1286 memory: 44.58GiB(31.99%) tps: 83,567 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6655 +[titan] 2025-07-10 08:46:51,330 - root - INFO - lr: 4.0883e-05 gnorm: 0.99 [19:11:47< 2:48:09] +[titan] 2025-07-10 08:46:55,274 - root - INFO - step: 87265 loss: 16.2562 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.7602 +[titan] 2025-07-10 08:46:55,274 - root - INFO - lr: 4.0874e-05 gnorm: 1.02 [19:11:51< 2:48:05] +[titan] 2025-07-10 08:46:59,232 - root - INFO - step: 87270 loss: 16.1646 memory: 44.58GiB(31.99%) tps: 82,810 tflops: 285.79 mfu: 28.90% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.6824 +[titan] 2025-07-10 08:46:59,232 - root - INFO - lr: 4.0866e-05 gnorm: 1.12 [19:11:55< 2:48:01] +[titan] 2025-07-10 08:47:03,130 - root - INFO - step: 87275 loss: 15.9658 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4293 global_avg_mtp_loss: 13.5365 +[titan] 2025-07-10 08:47:03,131 - root - INFO - lr: 4.0858e-05 gnorm: 1.01 [19:11:59< 2:47:57] +[titan] 2025-07-10 08:47:07,061 - root - INFO - step: 87280 loss: 15.9604 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4279 global_avg_mtp_loss: 13.5325 +[titan] 2025-07-10 08:47:07,062 - root - INFO - lr: 4.0849e-05 gnorm: 1.02 [19:12:03< 2:47:53] +[titan] 2025-07-10 08:47:10,964 - root - INFO - step: 87285 loss: 16.2838 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.8038 +[titan] 2025-07-10 08:47:10,965 - root - INFO - lr: 4.0841e-05 gnorm: 1.01 [19:12:07< 2:47:49] +[titan] 2025-07-10 08:47:14,869 - root - INFO - step: 87290 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.8085 +[titan] 2025-07-10 08:47:14,869 - root - INFO - lr: 4.0832e-05 gnorm: 1.02 [19:12:11< 2:47:45] +[titan] 2025-07-10 08:47:18,795 - root - INFO - step: 87295 loss: 16.4224 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5303 global_avg_mtp_loss: 13.8922 +[titan] 2025-07-10 08:47:18,795 - root - INFO - lr: 4.0824e-05 gnorm: 1.03 [19:12:15< 2:47:42] +[titan] 2025-07-10 08:47:21,939 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:47:22,751 - root - INFO - step: 87300 loss: 16.1596 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.7002 +[titan] 2025-07-10 08:47:22,751 - root - INFO - lr: 4.0815e-05 gnorm: 1.03 [19:12:19< 2:47:38] +[titan] 2025-07-10 08:47:26,683 - root - INFO - step: 87305 loss: 16.0068 memory: 44.58GiB(31.99%) tps: 83,342 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4330 global_avg_mtp_loss: 13.5738 +[titan] 2025-07-10 08:47:26,683 - root - INFO - lr: 4.0807e-05 gnorm: 1.03 [19:12:23< 2:47:34] +[titan] 2025-07-10 08:47:30,601 - root - INFO - step: 87310 loss: 16.2119 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7311 +[titan] 2025-07-10 08:47:30,601 - root - INFO - lr: 4.0799e-05 gnorm: 1.02 [19:12:27< 2:47:30] +[titan] 2025-07-10 08:47:34,516 - root - INFO - step: 87315 loss: 15.8383 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4276 global_avg_mtp_loss: 13.4106 +[titan] 2025-07-10 08:47:34,517 - root - INFO - lr: 4.0790e-05 gnorm: 1.01 [19:12:31< 2:47:26] +[titan] 2025-07-10 08:47:38,429 - root - INFO - step: 87320 loss: 16.1653 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4563 global_avg_mtp_loss: 13.7090 +[titan] 2025-07-10 08:47:38,430 - root - INFO - lr: 4.0782e-05 gnorm: 1.06 [19:12:34< 2:47:22] +[titan] 2025-07-10 08:47:42,332 - root - INFO - step: 87325 loss: 16.3787 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5033 global_avg_mtp_loss: 13.8754 +[titan] 2025-07-10 08:47:42,333 - root - INFO - lr: 4.0774e-05 gnorm: 1.03 [19:12:38< 2:47:18] +[titan] 2025-07-10 08:47:46,295 - root - INFO - step: 87330 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 82,700 tflops: 285.41 mfu: 28.86% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.8606 +[titan] 2025-07-10 08:47:46,295 - root - INFO - lr: 4.0765e-05 gnorm: 0.98 [19:12:42< 2:47:14] +[titan] 2025-07-10 08:47:50,256 - root - INFO - step: 87335 loss: 15.7166 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.3976 global_avg_mtp_loss: 13.3190 +[titan] 2025-07-10 08:47:50,257 - root - INFO - lr: 4.0757e-05 gnorm: 1.51 [19:12:46< 2:47:10] +[titan] 2025-07-10 08:47:54,191 - root - INFO - step: 87340 loss: 16.1983 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7264 +[titan] 2025-07-10 08:47:54,192 - root - INFO - lr: 4.0748e-05 gnorm: 1.00 [19:12:50< 2:47:06] +[titan] 2025-07-10 08:47:58,126 - root - INFO - step: 87345 loss: 16.0846 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.4546 global_avg_mtp_loss: 13.6300 +[titan] 2025-07-10 08:47:58,127 - root - INFO - lr: 4.0740e-05 gnorm: 0.97 [19:12:54< 2:47:02] +[titan] 2025-07-10 08:48:01,267 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:48:02,078 - root - INFO - step: 87350 loss: 16.0434 memory: 44.58GiB(31.99%) tps: 82,934 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.4482 global_avg_mtp_loss: 13.5953 +[titan] 2025-07-10 08:48:02,078 - root - INFO - lr: 4.0732e-05 gnorm: 1.03 [19:12:58< 2:46:58] +[titan] 2025-07-10 08:48:06,049 - root - INFO - step: 87355 loss: 16.0995 memory: 44.58GiB(31.99%) tps: 82,513 tflops: 284.77 mfu: 28.79% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.6506 +[titan] 2025-07-10 08:48:06,050 - root - INFO - lr: 4.0723e-05 gnorm: 1.01 [19:13:02< 2:46:54] +[titan] 2025-07-10 08:48:10,001 - root - INFO - step: 87360 loss: 16.1063 memory: 44.58GiB(31.99%) tps: 82,931 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.6588 +[titan] 2025-07-10 08:48:10,001 - root - INFO - lr: 4.0715e-05 gnorm: 1.00 [19:13:06< 2:46:50] +[titan] 2025-07-10 08:48:13,935 - root - INFO - step: 87365 loss: 16.3535 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.8511 +[titan] 2025-07-10 08:48:13,935 - root - INFO - lr: 4.0707e-05 gnorm: 1.11 [19:13:10< 2:46:46] +[titan] 2025-07-10 08:48:17,844 - root - INFO - step: 87370 loss: 16.4441 memory: 44.58GiB(31.99%) tps: 83,838 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.9314 +[titan] 2025-07-10 08:48:17,844 - root - INFO - lr: 4.0698e-05 gnorm: 1.02 [19:13:14< 2:46:42] +[titan] 2025-07-10 08:48:21,783 - root - INFO - step: 87375 loss: 16.2135 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7376 +[titan] 2025-07-10 08:48:21,784 - root - INFO - lr: 4.0690e-05 gnorm: 1.03 [19:13:18< 2:46:38] +[titan] 2025-07-10 08:48:25,694 - root - INFO - step: 87380 loss: 16.4661 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-10 08:48:25,694 - root - INFO - lr: 4.0681e-05 gnorm: 1.03 [19:13:22< 2:46:34] +[titan] 2025-07-10 08:48:29,672 - root - INFO - step: 87385 loss: 16.4647 memory: 44.58GiB(31.99%) tps: 82,370 tflops: 284.27 mfu: 28.74% global_avg_ntp_loss: 2.5262 global_avg_mtp_loss: 13.9385 +[titan] 2025-07-10 08:48:29,673 - root - INFO - lr: 4.0673e-05 gnorm: 1.05 [19:13:26< 2:46:30] +[titan] 2025-07-10 08:48:33,604 - root - INFO - step: 87390 loss: 16.1750 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.7058 +[titan] 2025-07-10 08:48:33,605 - root - INFO - lr: 4.0665e-05 gnorm: 0.94 [19:13:30< 2:46:26] +[titan] 2025-07-10 08:48:37,556 - root - INFO - step: 87395 loss: 16.1449 memory: 44.58GiB(31.99%) tps: 82,928 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 08:48:37,557 - root - INFO - lr: 4.0656e-05 gnorm: 1.03 [19:13:34< 2:46:22] +[titan] 2025-07-10 08:48:40,705 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:48:41,509 - root - INFO - step: 87400 loss: 16.0408 memory: 44.58GiB(31.99%) tps: 82,914 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.4496 global_avg_mtp_loss: 13.5912 +[titan] 2025-07-10 08:48:41,509 - root - INFO - lr: 4.0648e-05 gnorm: 1.04 [19:13:38< 2:46:18] +[titan] 2025-07-10 08:48:45,422 - root - INFO - step: 87405 loss: 15.9964 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4383 global_avg_mtp_loss: 13.5581 +[titan] 2025-07-10 08:48:45,422 - root - INFO - lr: 4.0640e-05 gnorm: 0.98 [19:13:41< 2:46:14] +[titan] 2025-07-10 08:48:49,343 - root - INFO - step: 87410 loss: 16.2831 memory: 44.58GiB(31.99%) tps: 83,585 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7925 +[titan] 2025-07-10 08:48:49,343 - root - INFO - lr: 4.0631e-05 gnorm: 1.09 [19:13:45< 2:46:10] +[titan] 2025-07-10 08:48:53,254 - root - INFO - step: 87415 loss: 16.3044 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8028 +[titan] 2025-07-10 08:48:53,254 - root - INFO - lr: 4.0623e-05 gnorm: 0.99 [19:13:49< 2:46:06] +[titan] 2025-07-10 08:48:57,186 - root - INFO - step: 87420 loss: 16.3632 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8602 +[titan] 2025-07-10 08:48:57,186 - root - INFO - lr: 4.0615e-05 gnorm: 1.05 [19:13:53< 2:46:02] +[titan] 2025-07-10 08:49:01,093 - root - INFO - step: 87425 loss: 16.2408 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 08:49:01,093 - root - INFO - lr: 4.0606e-05 gnorm: 1.02 [19:13:57< 2:45:58] +[titan] 2025-07-10 08:49:05,002 - root - INFO - step: 87430 loss: 16.4313 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.5115 global_avg_mtp_loss: 13.9198 +[titan] 2025-07-10 08:49:05,002 - root - INFO - lr: 4.0598e-05 gnorm: 1.01 [19:14:01< 2:45:55] +[titan] 2025-07-10 08:49:08,911 - root - INFO - step: 87435 loss: 16.1724 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.6960 +[titan] 2025-07-10 08:49:08,911 - root - INFO - lr: 4.0590e-05 gnorm: 1.00 [19:14:05< 2:45:51] +[titan] 2025-07-10 08:49:12,842 - root - INFO - step: 87440 loss: 16.4302 memory: 44.58GiB(31.99%) tps: 83,353 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 08:49:12,843 - root - INFO - lr: 4.0581e-05 gnorm: 0.98 [19:14:09< 2:45:47] +[titan] 2025-07-10 08:49:16,771 - root - INFO - step: 87445 loss: 16.2569 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.7555 +[titan] 2025-07-10 08:49:16,771 - root - INFO - lr: 4.0573e-05 gnorm: 1.04 [19:14:13< 2:45:43] +[titan] 2025-07-10 08:49:19,940 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:49:20,742 - root - INFO - step: 87450 loss: 16.2618 memory: 44.58GiB(31.99%) tps: 82,526 tflops: 284.81 mfu: 28.80% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7761 +[titan] 2025-07-10 08:49:20,742 - root - INFO - lr: 4.0565e-05 gnorm: 0.96 [19:14:17< 2:45:39] +[titan] 2025-07-10 08:49:24,678 - root - INFO - step: 87455 loss: 16.0570 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.6143 +[titan] 2025-07-10 08:49:24,678 - root - INFO - lr: 4.0557e-05 gnorm: 0.96 [19:14:21< 2:45:35] +[titan] 2025-07-10 08:49:28,602 - root - INFO - step: 87460 loss: 16.1932 memory: 44.58GiB(31.99%) tps: 83,506 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7075 +[titan] 2025-07-10 08:49:28,602 - root - INFO - lr: 4.0548e-05 gnorm: 1.01 [19:14:25< 2:45:31] +[titan] 2025-07-10 08:49:32,513 - root - INFO - step: 87465 loss: 16.6140 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5594 global_avg_mtp_loss: 14.0546 +[titan] 2025-07-10 08:49:32,513 - root - INFO - lr: 4.0540e-05 gnorm: 1.00 [19:14:29< 2:45:27] +[titan] 2025-07-10 08:49:36,476 - root - INFO - step: 87470 loss: 16.1168 memory: 44.58GiB(31.99%) tps: 82,675 tflops: 285.33 mfu: 28.85% global_avg_ntp_loss: 2.4580 global_avg_mtp_loss: 13.6588 +[titan] 2025-07-10 08:49:36,477 - root - INFO - lr: 4.0532e-05 gnorm: 1.02 [19:14:33< 2:45:23] +[titan] 2025-07-10 08:49:40,423 - root - INFO - step: 87475 loss: 16.4813 memory: 44.58GiB(31.99%) tps: 83,027 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.5310 global_avg_mtp_loss: 13.9503 +[titan] 2025-07-10 08:49:40,424 - root - INFO - lr: 4.0523e-05 gnorm: 1.03 [19:14:36< 2:45:19] +[titan] 2025-07-10 08:49:44,360 - root - INFO - step: 87480 loss: 16.3333 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 08:49:44,361 - root - INFO - lr: 4.0515e-05 gnorm: 0.99 [19:14:40< 2:45:15] +[titan] 2025-07-10 08:49:48,317 - root - INFO - step: 87485 loss: 16.2611 memory: 44.58GiB(31.99%) tps: 82,831 tflops: 285.86 mfu: 28.90% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7801 +[titan] 2025-07-10 08:49:48,317 - root - INFO - lr: 4.0507e-05 gnorm: 0.97 [19:14:44< 2:45:11] +[titan] 2025-07-10 08:49:52,258 - root - INFO - step: 87490 loss: 16.1322 memory: 44.58GiB(31.99%) tps: 83,158 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.6668 +[titan] 2025-07-10 08:49:52,258 - root - INFO - lr: 4.0499e-05 gnorm: 1.07 [19:14:48< 2:45:07] +[titan] 2025-07-10 08:49:56,161 - root - INFO - step: 87495 loss: 16.6211 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5835 global_avg_mtp_loss: 14.0376 +[titan] 2025-07-10 08:49:56,161 - root - INFO - lr: 4.0490e-05 gnorm: 1.02 [19:14:52< 2:45:03] +[titan] 2025-07-10 08:49:59,270 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:50:00,055 - root - INFO - step: 87500 loss: 16.2211 memory: 44.58GiB(31.99%) tps: 84,140 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7435 +[titan] 2025-07-10 08:50:00,056 - root - INFO - lr: 4.0482e-05 gnorm: 1.02 [19:14:56< 2:44:59] +[titan] 2025-07-10 08:50:03,961 - root - INFO - step: 87505 loss: 16.2262 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7519 +[titan] 2025-07-10 08:50:03,962 - root - INFO - lr: 4.0474e-05 gnorm: 1.01 [19:15:00< 2:44:55] +[titan] 2025-07-10 08:50:07,873 - root - INFO - step: 87510 loss: 16.4078 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8956 +[titan] 2025-07-10 08:50:07,873 - root - INFO - lr: 4.0465e-05 gnorm: 1.03 [19:15:04< 2:44:51] +[titan] 2025-07-10 08:50:11,801 - root - INFO - step: 87515 loss: 16.0180 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4505 global_avg_mtp_loss: 13.5675 +[titan] 2025-07-10 08:50:11,801 - root - INFO - lr: 4.0457e-05 gnorm: 1.01 [19:15:08< 2:44:47] +[titan] 2025-07-10 08:50:15,766 - root - INFO - step: 87520 loss: 16.1260 memory: 44.58GiB(31.99%) tps: 82,642 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.6616 +[titan] 2025-07-10 08:50:15,767 - root - INFO - lr: 4.0449e-05 gnorm: 1.03 [19:15:12< 2:44:43] +[titan] 2025-07-10 08:50:19,689 - root - INFO - step: 87525 loss: 16.5121 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5379 global_avg_mtp_loss: 13.9743 +[titan] 2025-07-10 08:50:19,689 - root - INFO - lr: 4.0441e-05 gnorm: 1.02 [19:15:16< 2:44:39] +[titan] 2025-07-10 08:50:23,614 - root - INFO - step: 87530 loss: 16.3509 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 08:50:23,615 - root - INFO - lr: 4.0432e-05 gnorm: 1.04 [19:15:20< 2:44:35] +[titan] 2025-07-10 08:50:27,525 - root - INFO - step: 87535 loss: 16.1491 memory: 44.58GiB(31.99%) tps: 83,805 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.6943 +[titan] 2025-07-10 08:50:27,525 - root - INFO - lr: 4.0424e-05 gnorm: 0.99 [19:15:24< 2:44:31] +[titan] 2025-07-10 08:50:31,454 - root - INFO - step: 87540 loss: 15.9573 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4294 global_avg_mtp_loss: 13.5279 +[titan] 2025-07-10 08:50:31,455 - root - INFO - lr: 4.0416e-05 gnorm: 1.03 [19:15:27< 2:44:27] +[titan] 2025-07-10 08:50:35,361 - root - INFO - step: 87545 loss: 16.2186 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7277 +[titan] 2025-07-10 08:50:35,362 - root - INFO - lr: 4.0408e-05 gnorm: 1.04 [19:15:31< 2:44:23] +[titan] 2025-07-10 08:50:38,614 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:50:39,405 - root - INFO - step: 87550 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 81,051 tflops: 279.72 mfu: 28.28% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7845 +[titan] 2025-07-10 08:50:39,405 - root - INFO - lr: 4.0399e-05 gnorm: 1.02 [19:15:35< 2:44:19] +[titan] 2025-07-10 08:50:41,136 - root - INFO - Dumping profiler traces at step 87552 +[titan] 2025-07-10 08:50:41,168 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:50:43,548 - root - INFO - step: 87555 loss: 16.3685 memory: 44.58GiB(31.99%) tps: 79,103 tflops: 273.00 mfu: 27.60% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8618 +[titan] 2025-07-10 08:50:43,548 - root - INFO - lr: 4.0391e-05 gnorm: 0.96 [19:15:40< 2:44:15] +[titan] 2025-07-10 08:50:47,491 - root - INFO - step: 87560 loss: 16.1212 memory: 44.58GiB(31.99%) tps: 83,110 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6665 +[titan] 2025-07-10 08:50:47,491 - root - INFO - lr: 4.0383e-05 gnorm: 1.02 [19:15:44< 2:44:11] +[titan] 2025-07-10 08:50:51,402 - root - INFO - step: 87565 loss: 15.7826 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.3995 global_avg_mtp_loss: 13.3831 +[titan] 2025-07-10 08:50:51,402 - root - INFO - lr: 4.0375e-05 gnorm: 1.03 [19:15:47< 2:44:08] +[titan] 2025-07-10 08:50:55,305 - root - INFO - step: 87570 loss: 16.2855 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4930 global_avg_mtp_loss: 13.7926 +[titan] 2025-07-10 08:50:55,306 - root - INFO - lr: 4.0366e-05 gnorm: 1.03 [19:15:51< 2:44:04] +[titan] 2025-07-10 08:50:59,216 - root - INFO - step: 87575 loss: 16.2154 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.7177 +[titan] 2025-07-10 08:50:59,216 - root - INFO - lr: 4.0358e-05 gnorm: 1.03 [19:15:55< 2:44:00] +[titan] 2025-07-10 08:51:03,146 - root - INFO - step: 87580 loss: 16.1630 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6981 +[titan] 2025-07-10 08:51:03,146 - root - INFO - lr: 4.0350e-05 gnorm: 1.00 [19:15:59< 2:43:56] +[titan] 2025-07-10 08:51:07,050 - root - INFO - step: 87585 loss: 16.3959 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.9043 +[titan] 2025-07-10 08:51:07,050 - root - INFO - lr: 4.0342e-05 gnorm: 0.99 [19:16:03< 2:43:52] +[titan] 2025-07-10 08:51:10,955 - root - INFO - step: 87590 loss: 16.3286 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.8396 +[titan] 2025-07-10 08:51:10,955 - root - INFO - lr: 4.0334e-05 gnorm: 0.95 [19:16:07< 2:43:48] +[titan] 2025-07-10 08:51:14,864 - root - INFO - step: 87595 loss: 16.3015 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8067 +[titan] 2025-07-10 08:51:14,865 - root - INFO - lr: 4.0325e-05 gnorm: 0.99 [19:16:11< 2:43:44] +[titan] 2025-07-10 08:51:17,995 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:51:18,789 - root - INFO - step: 87600 loss: 16.3407 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 08:51:18,790 - root - INFO - lr: 4.0317e-05 gnorm: 0.96 [19:16:15< 2:43:40] +[titan] 2025-07-10 08:51:22,724 - root - INFO - step: 87605 loss: 16.2417 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7698 +[titan] 2025-07-10 08:51:22,725 - root - INFO - lr: 4.0309e-05 gnorm: 0.99 [19:16:19< 2:43:36] +[titan] 2025-07-10 08:51:26,658 - root - INFO - step: 87610 loss: 16.1303 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4654 global_avg_mtp_loss: 13.6649 +[titan] 2025-07-10 08:51:26,658 - root - INFO - lr: 4.0301e-05 gnorm: 1.02 [19:16:23< 2:43:32] +[titan] 2025-07-10 08:51:30,609 - root - INFO - step: 87615 loss: 16.0955 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.4466 global_avg_mtp_loss: 13.6489 +[titan] 2025-07-10 08:51:30,609 - root - INFO - lr: 4.0292e-05 gnorm: 1.07 [19:16:27< 2:43:28] +[titan] 2025-07-10 08:51:34,523 - root - INFO - step: 87620 loss: 15.9876 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.5495 +[titan] 2025-07-10 08:51:34,523 - root - INFO - lr: 4.0284e-05 gnorm: 1.03 [19:16:31< 2:43:24] +[titan] 2025-07-10 08:51:38,429 - root - INFO - step: 87625 loss: 16.1290 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6690 +[titan] 2025-07-10 08:51:38,429 - root - INFO - lr: 4.0276e-05 gnorm: 0.99 [19:16:34< 2:43:20] +[titan] 2025-07-10 08:51:42,362 - root - INFO - step: 87630 loss: 16.2599 memory: 44.58GiB(31.99%) tps: 83,335 tflops: 287.60 mfu: 29.08% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7771 +[titan] 2025-07-10 08:51:42,362 - root - INFO - lr: 4.0268e-05 gnorm: 0.97 [19:16:38< 2:43:16] +[titan] 2025-07-10 08:51:46,290 - root - INFO - step: 87635 loss: 16.1481 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.6741 +[titan] 2025-07-10 08:51:46,290 - root - INFO - lr: 4.0260e-05 gnorm: 1.13 [19:16:42< 2:43:12] +[titan] 2025-07-10 08:51:50,229 - root - INFO - step: 87640 loss: 16.2828 memory: 44.58GiB(31.99%) tps: 83,185 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.8013 +[titan] 2025-07-10 08:51:50,230 - root - INFO - lr: 4.0251e-05 gnorm: 0.99 [19:16:46< 2:43:08] +[titan] 2025-07-10 08:51:54,152 - root - INFO - step: 87645 loss: 16.5005 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.5429 global_avg_mtp_loss: 13.9576 +[titan] 2025-07-10 08:51:54,153 - root - INFO - lr: 4.0243e-05 gnorm: 1.00 [19:16:50< 2:43:04] +[titan] 2025-07-10 08:51:57,309 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:51:58,099 - root - INFO - step: 87650 loss: 16.0250 memory: 44.58GiB(31.99%) tps: 83,044 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.5639 +[titan] 2025-07-10 08:51:58,099 - root - INFO - lr: 4.0235e-05 gnorm: 1.04 [19:16:54< 2:43:00] +[titan] 2025-07-10 08:52:02,057 - root - INFO - step: 87655 loss: 16.2600 memory: 44.58GiB(31.99%) tps: 82,795 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7797 +[titan] 2025-07-10 08:52:02,057 - root - INFO - lr: 4.0227e-05 gnorm: 1.06 [19:16:58< 2:42:56] +[titan] 2025-07-10 08:52:05,977 - root - INFO - step: 87660 loss: 16.3128 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8138 +[titan] 2025-07-10 08:52:05,978 - root - INFO - lr: 4.0219e-05 gnorm: 1.02 [19:17:02< 2:42:52] +[titan] 2025-07-10 08:52:09,903 - root - INFO - step: 87665 loss: 15.9846 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4369 global_avg_mtp_loss: 13.5477 +[titan] 2025-07-10 08:52:09,903 - root - INFO - lr: 4.0211e-05 gnorm: 1.02 [19:17:06< 2:42:48] +[titan] 2025-07-10 08:52:13,822 - root - INFO - step: 87670 loss: 16.1542 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.6813 +[titan] 2025-07-10 08:52:13,823 - root - INFO - lr: 4.0202e-05 gnorm: 1.00 [19:17:10< 2:42:44] +[titan] 2025-07-10 08:52:17,753 - root - INFO - step: 87675 loss: 16.1433 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6835 +[titan] 2025-07-10 08:52:17,753 - root - INFO - lr: 4.0194e-05 gnorm: 0.98 [19:17:14< 2:42:40] +[titan] 2025-07-10 08:52:21,702 - root - INFO - step: 87680 loss: 16.1618 memory: 44.58GiB(31.99%) tps: 82,982 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4680 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 08:52:21,703 - root - INFO - lr: 4.0186e-05 gnorm: 0.99 [19:17:18< 2:42:36] +[titan] 2025-07-10 08:52:25,632 - root - INFO - step: 87685 loss: 16.1633 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6955 +[titan] 2025-07-10 08:52:25,632 - root - INFO - lr: 4.0178e-05 gnorm: 1.04 [19:17:22< 2:42:32] +[titan] 2025-07-10 08:52:29,565 - root - INFO - step: 87690 loss: 16.3052 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8060 +[titan] 2025-07-10 08:52:29,566 - root - INFO - lr: 4.0170e-05 gnorm: 3.85 [19:17:26< 2:42:28] +[titan] 2025-07-10 08:52:33,531 - root - INFO - step: 87695 loss: 15.9905 memory: 44.58GiB(31.99%) tps: 82,645 tflops: 285.22 mfu: 28.84% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.5478 +[titan] 2025-07-10 08:52:33,531 - root - INFO - lr: 4.0162e-05 gnorm: 0.99 [19:17:30< 2:42:24] +[titan] 2025-07-10 08:52:36,684 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:52:37,483 - root - INFO - step: 87700 loss: 16.4595 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.9416 +[titan] 2025-07-10 08:52:37,484 - root - INFO - lr: 4.0153e-05 gnorm: 1.01 [19:17:33< 2:42:20] +[titan] 2025-07-10 08:52:41,388 - root - INFO - step: 87705 loss: 16.2112 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 08:52:41,388 - root - INFO - lr: 4.0145e-05 gnorm: 1.04 [19:17:37< 2:42:17] +[titan] 2025-07-10 08:52:45,309 - root - INFO - step: 87710 loss: 16.2834 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.8016 +[titan] 2025-07-10 08:52:45,309 - root - INFO - lr: 4.0137e-05 gnorm: 1.07 [19:17:41< 2:42:13] +[titan] 2025-07-10 08:52:49,243 - root - INFO - step: 87715 loss: 16.0911 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.6343 +[titan] 2025-07-10 08:52:49,243 - root - INFO - lr: 4.0129e-05 gnorm: 0.97 [19:17:45< 2:42:09] +[titan] 2025-07-10 08:52:53,155 - root - INFO - step: 87720 loss: 16.2824 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.7947 +[titan] 2025-07-10 08:52:53,155 - root - INFO - lr: 4.0121e-05 gnorm: 1.03 [19:17:49< 2:42:05] +[titan] 2025-07-10 08:52:57,081 - root - INFO - step: 87725 loss: 16.0063 memory: 44.58GiB(31.99%) tps: 83,460 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.5682 +[titan] 2025-07-10 08:52:57,082 - root - INFO - lr: 4.0113e-05 gnorm: 1.06 [19:17:53< 2:42:01] +[titan] 2025-07-10 08:53:01,063 - root - INFO - step: 87730 loss: 16.2352 memory: 44.58GiB(31.99%) tps: 82,299 tflops: 284.03 mfu: 28.72% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 08:53:01,064 - root - INFO - lr: 4.0105e-05 gnorm: 1.00 [19:17:57< 2:41:57] +[titan] 2025-07-10 08:53:04,989 - root - INFO - step: 87735 loss: 16.1673 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6992 +[titan] 2025-07-10 08:53:04,990 - root - INFO - lr: 4.0096e-05 gnorm: 0.96 [19:18:01< 2:41:53] +[titan] 2025-07-10 08:53:08,909 - root - INFO - step: 87740 loss: 16.2398 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.7506 +[titan] 2025-07-10 08:53:08,909 - root - INFO - lr: 4.0088e-05 gnorm: 1.05 [19:18:05< 2:41:49] +[titan] 2025-07-10 08:53:12,817 - root - INFO - step: 87745 loss: 16.1603 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4657 global_avg_mtp_loss: 13.6946 +[titan] 2025-07-10 08:53:12,817 - root - INFO - lr: 4.0080e-05 gnorm: 1.02 [19:18:09< 2:41:45] +[titan] 2025-07-10 08:53:15,957 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:53:16,751 - root - INFO - step: 87750 loss: 16.2839 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4920 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 08:53:16,751 - root - INFO - lr: 4.0072e-05 gnorm: 0.98 [19:18:13< 2:41:41] +[titan] 2025-07-10 08:53:20,658 - root - INFO - step: 87755 loss: 16.1549 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.6966 +[titan] 2025-07-10 08:53:20,658 - root - INFO - lr: 4.0064e-05 gnorm: 1.05 [19:18:17< 2:41:37] +[titan] 2025-07-10 08:53:24,605 - root - INFO - step: 87760 loss: 16.0024 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5598 +[titan] 2025-07-10 08:53:24,605 - root - INFO - lr: 4.0056e-05 gnorm: 1.01 [19:18:21< 2:41:33] +[titan] 2025-07-10 08:53:28,537 - root - INFO - step: 87765 loss: 16.3740 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8728 +[titan] 2025-07-10 08:53:28,537 - root - INFO - lr: 4.0048e-05 gnorm: 1.01 [19:18:25< 2:41:29] +[titan] 2025-07-10 08:53:32,442 - root - INFO - step: 87770 loss: 16.1749 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.6972 +[titan] 2025-07-10 08:53:32,442 - root - INFO - lr: 4.0040e-05 gnorm: 0.96 [19:18:28< 2:41:25] +[titan] 2025-07-10 08:53:36,360 - root - INFO - step: 87775 loss: 16.1964 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.7267 +[titan] 2025-07-10 08:53:36,360 - root - INFO - lr: 4.0032e-05 gnorm: 1.02 [19:18:32< 2:41:21] +[titan] 2025-07-10 08:53:40,278 - root - INFO - step: 87780 loss: 16.0596 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.6216 +[titan] 2025-07-10 08:53:40,279 - root - INFO - lr: 4.0023e-05 gnorm: 1.06 [19:18:36< 2:41:17] +[titan] 2025-07-10 08:53:44,204 - root - INFO - step: 87785 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.8723 +[titan] 2025-07-10 08:53:44,205 - root - INFO - lr: 4.0015e-05 gnorm: 1.02 [19:18:40< 2:41:13] +[titan] 2025-07-10 08:53:48,183 - root - INFO - step: 87790 loss: 16.0055 memory: 44.58GiB(31.99%) tps: 82,364 tflops: 284.25 mfu: 28.74% global_avg_ntp_loss: 2.4384 global_avg_mtp_loss: 13.5671 +[titan] 2025-07-10 08:53:48,183 - root - INFO - lr: 4.0007e-05 gnorm: 1.01 [19:18:44< 2:41:09] +[titan] 2025-07-10 08:53:52,147 - root - INFO - step: 87795 loss: 16.2882 memory: 44.58GiB(31.99%) tps: 82,681 tflops: 285.35 mfu: 28.85% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.7887 +[titan] 2025-07-10 08:53:52,147 - root - INFO - lr: 3.9999e-05 gnorm: 0.99 [19:18:48< 2:41:05] +[titan] 2025-07-10 08:53:55,270 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:53:56,057 - root - INFO - step: 87800 loss: 16.1362 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6767 +[titan] 2025-07-10 08:53:56,057 - root - INFO - lr: 3.9991e-05 gnorm: 1.01 [19:18:52< 2:41:01] +[titan] 2025-07-10 08:53:59,975 - root - INFO - step: 87805 loss: 16.2754 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.7791 +[titan] 2025-07-10 08:53:59,975 - root - INFO - lr: 3.9983e-05 gnorm: 0.99 [19:18:56< 2:40:57] +[titan] 2025-07-10 08:54:03,903 - root - INFO - step: 87810 loss: 16.0892 memory: 44.58GiB(31.99%) tps: 83,425 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4523 global_avg_mtp_loss: 13.6368 +[titan] 2025-07-10 08:54:03,903 - root - INFO - lr: 3.9975e-05 gnorm: 1.02 [19:19:00< 2:40:53] +[titan] 2025-07-10 08:54:07,840 - root - INFO - step: 87815 loss: 16.1416 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6706 +[titan] 2025-07-10 08:54:07,840 - root - INFO - lr: 3.9967e-05 gnorm: 0.99 [19:19:04< 2:40:49] +[titan] 2025-07-10 08:54:11,769 - root - INFO - step: 87820 loss: 16.4286 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.9133 +[titan] 2025-07-10 08:54:11,769 - root - INFO - lr: 3.9959e-05 gnorm: 1.02 [19:19:08< 2:40:45] +[titan] 2025-07-10 08:54:15,712 - root - INFO - step: 87825 loss: 15.9381 memory: 44.58GiB(31.99%) tps: 83,109 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.4273 global_avg_mtp_loss: 13.5109 +[titan] 2025-07-10 08:54:15,712 - root - INFO - lr: 3.9951e-05 gnorm: 1.06 [19:19:12< 2:40:41] +[titan] 2025-07-10 08:54:19,672 - root - INFO - step: 87830 loss: 16.1673 memory: 44.58GiB(31.99%) tps: 82,741 tflops: 285.55 mfu: 28.87% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.6945 +[titan] 2025-07-10 08:54:19,673 - root - INFO - lr: 3.9943e-05 gnorm: 1.06 [19:19:16< 2:40:37] +[titan] 2025-07-10 08:54:23,572 - root - INFO - step: 87835 loss: 16.3487 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4884 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 08:54:23,573 - root - INFO - lr: 3.9935e-05 gnorm: 0.97 [19:19:20< 2:40:33] +[titan] 2025-07-10 08:54:27,512 - root - INFO - step: 87840 loss: 16.2337 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.7670 +[titan] 2025-07-10 08:54:27,512 - root - INFO - lr: 3.9926e-05 gnorm: 0.99 [19:19:24< 2:40:29] +[titan] 2025-07-10 08:54:31,432 - root - INFO - step: 87845 loss: 15.9576 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4392 global_avg_mtp_loss: 13.5184 +[titan] 2025-07-10 08:54:31,432 - root - INFO - lr: 3.9918e-05 gnorm: 1.00 [19:19:27< 2:40:26] +[titan] 2025-07-10 08:54:34,589 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:54:35,380 - root - INFO - step: 87850 loss: 16.4762 memory: 44.58GiB(31.99%) tps: 83,012 tflops: 286.49 mfu: 28.97% global_avg_ntp_loss: 2.5288 global_avg_mtp_loss: 13.9475 +[titan] 2025-07-10 08:54:35,380 - root - INFO - lr: 3.9910e-05 gnorm: 1.05 [19:19:31< 2:40:22] +[titan] 2025-07-10 08:54:39,340 - root - INFO - step: 87855 loss: 16.1731 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.7120 +[titan] 2025-07-10 08:54:39,341 - root - INFO - lr: 3.9902e-05 gnorm: 1.00 [19:19:35< 2:40:18] +[titan] 2025-07-10 08:54:43,245 - root - INFO - step: 87860 loss: 16.3550 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8548 +[titan] 2025-07-10 08:54:43,245 - root - INFO - lr: 3.9894e-05 gnorm: 1.00 [19:19:39< 2:40:14] +[titan] 2025-07-10 08:54:47,158 - root - INFO - step: 87865 loss: 16.1756 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.6960 +[titan] 2025-07-10 08:54:47,158 - root - INFO - lr: 3.9886e-05 gnorm: 0.97 [19:19:43< 2:40:10] +[titan] 2025-07-10 08:54:51,087 - root - INFO - step: 87870 loss: 16.5219 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9918 +[titan] 2025-07-10 08:54:51,087 - root - INFO - lr: 3.9878e-05 gnorm: 0.98 [19:19:47< 2:40:06] +[titan] 2025-07-10 08:54:55,024 - root - INFO - step: 87875 loss: 16.0524 memory: 44.58GiB(31.99%) tps: 83,238 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.6099 +[titan] 2025-07-10 08:54:55,024 - root - INFO - lr: 3.9870e-05 gnorm: 0.96 [19:19:51< 2:40:02] +[titan] 2025-07-10 08:54:58,950 - root - INFO - step: 87880 loss: 16.2034 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7294 +[titan] 2025-07-10 08:54:58,950 - root - INFO - lr: 3.9862e-05 gnorm: 1.00 [19:19:55< 2:39:58] +[titan] 2025-07-10 08:55:02,863 - root - INFO - step: 87885 loss: 16.2526 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.7826 +[titan] 2025-07-10 08:55:02,864 - root - INFO - lr: 3.9854e-05 gnorm: 0.99 [19:19:59< 2:39:54] +[titan] 2025-07-10 08:55:06,770 - root - INFO - step: 87890 loss: 16.2640 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.7963 +[titan] 2025-07-10 08:55:06,770 - root - INFO - lr: 3.9846e-05 gnorm: 1.00 [19:20:03< 2:39:50] +[titan] 2025-07-10 08:55:10,674 - root - INFO - step: 87895 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8265 +[titan] 2025-07-10 08:55:10,675 - root - INFO - lr: 3.9838e-05 gnorm: 1.03 [19:20:07< 2:39:46] +[titan] 2025-07-10 08:55:13,827 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:55:14,615 - root - INFO - step: 87900 loss: 16.3181 memory: 44.58GiB(31.99%) tps: 83,161 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8234 +[titan] 2025-07-10 08:55:14,615 - root - INFO - lr: 3.9830e-05 gnorm: 1.01 [19:20:11< 2:39:42] +[titan] 2025-07-10 08:55:18,528 - root - INFO - step: 87905 loss: 16.3624 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.8640 +[titan] 2025-07-10 08:55:18,528 - root - INFO - lr: 3.9822e-05 gnorm: 1.00 [19:20:15< 2:39:38] +[titan] 2025-07-10 08:55:22,467 - root - INFO - step: 87910 loss: 16.2280 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.7694 +[titan] 2025-07-10 08:55:22,468 - root - INFO - lr: 3.9814e-05 gnorm: 0.97 [19:20:18< 2:39:34] +[titan] 2025-07-10 08:55:26,388 - root - INFO - step: 87915 loss: 16.3458 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8521 +[titan] 2025-07-10 08:55:26,388 - root - INFO - lr: 3.9806e-05 gnorm: 1.06 [19:20:22< 2:39:30] +[titan] 2025-07-10 08:55:30,295 - root - INFO - step: 87920 loss: 16.1166 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6520 +[titan] 2025-07-10 08:55:30,295 - root - INFO - lr: 3.9798e-05 gnorm: 1.01 [19:20:26< 2:39:26] +[titan] 2025-07-10 08:55:34,204 - root - INFO - step: 87925 loss: 16.3548 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5153 global_avg_mtp_loss: 13.8395 +[titan] 2025-07-10 08:55:34,205 - root - INFO - lr: 3.9790e-05 gnorm: 1.05 [19:20:30< 2:39:22] +[titan] 2025-07-10 08:55:38,132 - root - INFO - step: 87930 loss: 15.9224 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4349 global_avg_mtp_loss: 13.4875 +[titan] 2025-07-10 08:55:38,133 - root - INFO - lr: 3.9782e-05 gnorm: 1.03 [19:20:34< 2:39:18] +[titan] 2025-07-10 08:55:42,047 - root - INFO - step: 87935 loss: 16.1806 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.7083 +[titan] 2025-07-10 08:55:42,047 - root - INFO - lr: 3.9774e-05 gnorm: 1.00 [19:20:38< 2:39:14] +[titan] 2025-07-10 08:55:45,969 - root - INFO - step: 87940 loss: 16.1943 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 08:55:45,970 - root - INFO - lr: 3.9766e-05 gnorm: 1.05 [19:20:42< 2:39:10] +[titan] 2025-07-10 08:55:49,908 - root - INFO - step: 87945 loss: 16.4268 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 08:55:49,908 - root - INFO - lr: 3.9758e-05 gnorm: 1.02 [19:20:46< 2:39:06] +[titan] 2025-07-10 08:55:53,010 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:55:53,798 - root - INFO - step: 87950 loss: 16.2588 memory: 44.58GiB(31.99%) tps: 84,237 tflops: 290.72 mfu: 29.40% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7859 +[titan] 2025-07-10 08:55:53,798 - root - INFO - lr: 3.9750e-05 gnorm: 0.99 [19:20:50< 2:39:02] +[titan] 2025-07-10 08:55:57,714 - root - INFO - step: 87955 loss: 16.2792 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4850 global_avg_mtp_loss: 13.7942 +[titan] 2025-07-10 08:55:57,715 - root - INFO - lr: 3.9742e-05 gnorm: 1.03 [19:20:54< 2:38:58] +[titan] 2025-07-10 08:56:01,658 - root - INFO - step: 87960 loss: 15.9055 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4156 global_avg_mtp_loss: 13.4899 +[titan] 2025-07-10 08:56:01,659 - root - INFO - lr: 3.9734e-05 gnorm: 0.99 [19:20:58< 2:38:54] +[titan] 2025-07-10 08:56:05,551 - root - INFO - step: 87965 loss: 16.0258 memory: 44.58GiB(31.99%) tps: 84,190 tflops: 290.55 mfu: 29.38% global_avg_ntp_loss: 2.4475 global_avg_mtp_loss: 13.5783 +[titan] 2025-07-10 08:56:05,551 - root - INFO - lr: 3.9726e-05 gnorm: 1.02 [19:21:02< 2:38:50] +[titan] 2025-07-10 08:56:09,502 - root - INFO - step: 87970 loss: 16.2259 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7498 +[titan] 2025-07-10 08:56:09,503 - root - INFO - lr: 3.9718e-05 gnorm: 1.03 [19:21:05< 2:38:46] +[titan] 2025-07-10 08:56:13,433 - root - INFO - step: 87975 loss: 16.0348 memory: 44.58GiB(31.99%) tps: 83,371 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.5908 +[titan] 2025-07-10 08:56:13,433 - root - INFO - lr: 3.9710e-05 gnorm: 1.01 [19:21:09< 2:38:42] +[titan] 2025-07-10 08:56:17,408 - root - INFO - step: 87980 loss: 16.3782 memory: 44.58GiB(31.99%) tps: 82,439 tflops: 284.51 mfu: 28.77% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8660 +[titan] 2025-07-10 08:56:17,409 - root - INFO - lr: 3.9702e-05 gnorm: 1.09 [19:21:13< 2:38:38] +[titan] 2025-07-10 08:56:21,319 - root - INFO - step: 87985 loss: 16.2016 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.7339 +[titan] 2025-07-10 08:56:21,320 - root - INFO - lr: 3.9694e-05 gnorm: 0.99 [19:21:17< 2:38:35] +[titan] 2025-07-10 08:56:25,239 - root - INFO - step: 87990 loss: 16.2080 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.7516 +[titan] 2025-07-10 08:56:25,240 - root - INFO - lr: 3.9686e-05 gnorm: 0.99 [19:21:21< 2:38:31] +[titan] 2025-07-10 08:56:29,171 - root - INFO - step: 87995 loss: 16.3895 memory: 44.58GiB(31.99%) tps: 83,365 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.5291 global_avg_mtp_loss: 13.8604 +[titan] 2025-07-10 08:56:29,171 - root - INFO - lr: 3.9678e-05 gnorm: 1.04 [19:21:25< 2:38:27] +[titan] 2025-07-10 08:56:32,349 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:56:33,137 - root - INFO - step: 88000 loss: 16.2089 memory: 44.58GiB(31.99%) tps: 82,622 tflops: 285.14 mfu: 28.83% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7381 +[titan] 2025-07-10 08:56:33,137 - root - INFO - lr: 3.9670e-05 gnorm: 0.99 [19:21:29< 2:38:23] +[titan] 2025-07-10 08:56:37,064 - root - INFO - step: 88005 loss: 16.3482 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.8327 +[titan] 2025-07-10 08:56:37,064 - root - INFO - lr: 3.9662e-05 gnorm: 1.07 [19:21:33< 2:38:19] +[titan] 2025-07-10 08:56:40,974 - root - INFO - step: 88010 loss: 16.0786 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.6313 +[titan] 2025-07-10 08:56:40,974 - root - INFO - lr: 3.9654e-05 gnorm: 1.01 [19:21:37< 2:38:15] +[titan] 2025-07-10 08:56:44,914 - root - INFO - step: 88015 loss: 16.4019 memory: 44.58GiB(31.99%) tps: 83,179 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.9088 +[titan] 2025-07-10 08:56:44,914 - root - INFO - lr: 3.9646e-05 gnorm: 1.03 [19:21:41< 2:38:11] +[titan] 2025-07-10 08:56:48,854 - root - INFO - step: 88020 loss: 16.2539 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7772 +[titan] 2025-07-10 08:56:48,855 - root - INFO - lr: 3.9638e-05 gnorm: 1.01 [19:21:45< 2:38:07] +[titan] 2025-07-10 08:56:52,764 - root - INFO - step: 88025 loss: 16.0153 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.5694 +[titan] 2025-07-10 08:56:52,764 - root - INFO - lr: 3.9630e-05 gnorm: 1.04 [19:21:49< 2:38:03] +[titan] 2025-07-10 08:56:56,680 - root - INFO - step: 88030 loss: 15.9575 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4286 global_avg_mtp_loss: 13.5289 +[titan] 2025-07-10 08:56:56,680 - root - INFO - lr: 3.9622e-05 gnorm: 1.17 [19:21:53< 2:37:59] +[titan] 2025-07-10 08:57:00,659 - root - INFO - step: 88035 loss: 16.1690 memory: 44.58GiB(31.99%) tps: 82,366 tflops: 284.26 mfu: 28.74% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.7094 +[titan] 2025-07-10 08:57:00,659 - root - INFO - lr: 3.9614e-05 gnorm: 1.01 [19:21:57< 2:37:55] +[titan] 2025-07-10 08:57:04,588 - root - INFO - step: 88040 loss: 16.0007 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4411 global_avg_mtp_loss: 13.5595 +[titan] 2025-07-10 08:57:04,589 - root - INFO - lr: 3.9607e-05 gnorm: 0.99 [19:22:01< 2:37:51] +[titan] 2025-07-10 08:57:08,491 - root - INFO - step: 88045 loss: 16.3255 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8277 +[titan] 2025-07-10 08:57:08,491 - root - INFO - lr: 3.9599e-05 gnorm: 0.98 [19:22:04< 2:37:47] +[titan] 2025-07-10 08:57:11,624 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:57:12,419 - root - INFO - step: 88050 loss: 16.2367 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7608 +[titan] 2025-07-10 08:57:12,420 - root - INFO - lr: 3.9591e-05 gnorm: 1.03 [19:22:08< 2:37:43] +[titan] 2025-07-10 08:57:16,359 - root - INFO - step: 88055 loss: 16.4530 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.5206 global_avg_mtp_loss: 13.9324 +[titan] 2025-07-10 08:57:16,359 - root - INFO - lr: 3.9583e-05 gnorm: 0.97 [19:22:12< 2:37:39] +[titan] 2025-07-10 08:57:20,324 - root - INFO - step: 88060 loss: 16.0600 memory: 44.58GiB(31.99%) tps: 82,640 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.4353 global_avg_mtp_loss: 13.6247 +[titan] 2025-07-10 08:57:20,325 - root - INFO - lr: 3.9575e-05 gnorm: 1.01 [19:22:16< 2:37:35] +[titan] 2025-07-10 08:57:23,675 - root - INFO - Dumping profiler traces at step 88064 +[titan] 2025-07-10 08:57:23,708 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 08:57:24,494 - root - INFO - step: 88065 loss: 16.4248 memory: 44.58GiB(31.99%) tps: 78,605 tflops: 271.28 mfu: 27.43% global_avg_ntp_loss: 2.5217 global_avg_mtp_loss: 13.9031 +[titan] 2025-07-10 08:57:24,494 - root - INFO - lr: 3.9567e-05 gnorm: 0.96 [19:22:20< 2:37:31] +[titan] 2025-07-10 08:57:28,398 - root - INFO - step: 88070 loss: 16.1431 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6823 +[titan] 2025-07-10 08:57:28,398 - root - INFO - lr: 3.9559e-05 gnorm: 1.01 [19:22:24< 2:37:27] +[titan] 2025-07-10 08:57:32,311 - root - INFO - step: 88075 loss: 16.0347 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.5873 +[titan] 2025-07-10 08:57:32,311 - root - INFO - lr: 3.9551e-05 gnorm: 0.99 [19:22:28< 2:37:23] +[titan] 2025-07-10 08:57:36,250 - root - INFO - step: 88080 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 83,191 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.5150 global_avg_mtp_loss: 13.8538 +[titan] 2025-07-10 08:57:36,251 - root - INFO - lr: 3.9543e-05 gnorm: 1.07 [19:22:32< 2:37:19] +[titan] 2025-07-10 08:57:40,165 - root - INFO - step: 88085 loss: 16.3571 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5015 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 08:57:40,166 - root - INFO - lr: 3.9535e-05 gnorm: 1.01 [19:22:36< 2:37:15] +[titan] 2025-07-10 08:57:44,107 - root - INFO - step: 88090 loss: 16.2472 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.7800 +[titan] 2025-07-10 08:57:44,107 - root - INFO - lr: 3.9527e-05 gnorm: 0.98 [19:22:40< 2:37:11] +[titan] 2025-07-10 08:57:48,009 - root - INFO - step: 88095 loss: 16.1640 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.7032 +[titan] 2025-07-10 08:57:48,010 - root - INFO - lr: 3.9519e-05 gnorm: 1.03 [19:22:44< 2:37:07] +[titan] 2025-07-10 08:57:51,133 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:57:51,925 - root - INFO - step: 88100 loss: 16.0914 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4622 global_avg_mtp_loss: 13.6293 +[titan] 2025-07-10 08:57:51,925 - root - INFO - lr: 3.9512e-05 gnorm: 1.10 [19:22:48< 2:37:03] +[titan] 2025-07-10 08:57:55,846 - root - INFO - step: 88105 loss: 15.7959 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.3874 global_avg_mtp_loss: 13.4086 +[titan] 2025-07-10 08:57:55,846 - root - INFO - lr: 3.9504e-05 gnorm: 1.07 [19:22:52< 2:36:59] +[titan] 2025-07-10 08:57:59,766 - root - INFO - step: 88110 loss: 16.1552 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4625 global_avg_mtp_loss: 13.6928 +[titan] 2025-07-10 08:57:59,766 - root - INFO - lr: 3.9496e-05 gnorm: 1.01 [19:22:56< 2:36:55] +[titan] 2025-07-10 08:58:03,680 - root - INFO - step: 88115 loss: 16.2171 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.7313 +[titan] 2025-07-10 08:58:03,680 - root - INFO - lr: 3.9488e-05 gnorm: 0.98 [19:23:00< 2:36:51] +[titan] 2025-07-10 08:58:07,609 - root - INFO - step: 88120 loss: 16.0610 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4399 global_avg_mtp_loss: 13.6211 +[titan] 2025-07-10 08:58:07,609 - root - INFO - lr: 3.9480e-05 gnorm: 1.04 [19:23:04< 2:36:48] +[titan] 2025-07-10 08:58:11,555 - root - INFO - step: 88125 loss: 16.3650 memory: 44.58GiB(31.99%) tps: 83,046 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8584 +[titan] 2025-07-10 08:58:11,555 - root - INFO - lr: 3.9472e-05 gnorm: 1.01 [19:23:08< 2:36:44] +[titan] 2025-07-10 08:58:15,467 - root - INFO - step: 88130 loss: 16.0255 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4339 global_avg_mtp_loss: 13.5916 +[titan] 2025-07-10 08:58:15,467 - root - INFO - lr: 3.9464e-05 gnorm: 1.05 [19:23:11< 2:36:40] +[titan] 2025-07-10 08:58:19,407 - root - INFO - step: 88135 loss: 16.0557 memory: 44.58GiB(31.99%) tps: 83,175 tflops: 287.05 mfu: 29.02% global_avg_ntp_loss: 2.4431 global_avg_mtp_loss: 13.6126 +[titan] 2025-07-10 08:58:19,407 - root - INFO - lr: 3.9456e-05 gnorm: 1.01 [19:23:15< 2:36:36] +[titan] 2025-07-10 08:58:23,358 - root - INFO - step: 88140 loss: 15.9135 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4399 global_avg_mtp_loss: 13.4735 +[titan] 2025-07-10 08:58:23,358 - root - INFO - lr: 3.9448e-05 gnorm: 1.01 [19:23:19< 2:36:32] +[titan] 2025-07-10 08:58:27,278 - root - INFO - step: 88145 loss: 16.0338 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4428 global_avg_mtp_loss: 13.5910 +[titan] 2025-07-10 08:58:27,278 - root - INFO - lr: 3.9441e-05 gnorm: 1.05 [19:23:23< 2:36:28] +[titan] 2025-07-10 08:58:30,390 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:58:31,181 - root - INFO - step: 88150 loss: 16.5178 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9900 +[titan] 2025-07-10 08:58:31,181 - root - INFO - lr: 3.9433e-05 gnorm: 1.02 [19:23:27< 2:36:24] +[titan] 2025-07-10 08:58:35,087 - root - INFO - step: 88155 loss: 16.0228 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.5679 +[titan] 2025-07-10 08:58:35,087 - root - INFO - lr: 3.9425e-05 gnorm: 1.10 [19:23:31< 2:36:20] +[titan] 2025-07-10 08:58:39,017 - root - INFO - step: 88160 loss: 16.0973 memory: 44.58GiB(31.99%) tps: 83,377 tflops: 287.75 mfu: 29.09% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.6280 +[titan] 2025-07-10 08:58:39,017 - root - INFO - lr: 3.9417e-05 gnorm: 1.05 [19:23:35< 2:36:16] +[titan] 2025-07-10 08:58:42,926 - root - INFO - step: 88165 loss: 16.3709 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 08:58:42,926 - root - INFO - lr: 3.9409e-05 gnorm: 1.06 [19:23:39< 2:36:12] +[titan] 2025-07-10 08:58:46,825 - root - INFO - step: 88170 loss: 16.2743 memory: 44.58GiB(31.99%) tps: 84,041 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-10 08:58:46,825 - root - INFO - lr: 3.9401e-05 gnorm: 1.12 [19:23:43< 2:36:08] +[titan] 2025-07-10 08:58:50,719 - root - INFO - step: 88175 loss: 16.3245 memory: 44.58GiB(31.99%) tps: 84,160 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.8272 +[titan] 2025-07-10 08:58:50,719 - root - INFO - lr: 3.9393e-05 gnorm: 1.01 [19:23:47< 2:36:04] +[titan] 2025-07-10 08:58:54,618 - root - INFO - step: 88180 loss: 16.5466 memory: 44.58GiB(31.99%) tps: 84,050 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 14.0097 +[titan] 2025-07-10 08:58:54,618 - root - INFO - lr: 3.9386e-05 gnorm: 1.02 [19:23:51< 2:36:00] +[titan] 2025-07-10 08:58:58,521 - root - INFO - step: 88185 loss: 16.0132 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5714 +[titan] 2025-07-10 08:58:58,522 - root - INFO - lr: 3.9378e-05 gnorm: 1.05 [19:23:54< 2:35:56] +[titan] 2025-07-10 08:59:02,412 - root - INFO - step: 88190 loss: 16.0591 memory: 44.58GiB(31.99%) tps: 84,228 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.4429 global_avg_mtp_loss: 13.6162 +[titan] 2025-07-10 08:59:02,412 - root - INFO - lr: 3.9370e-05 gnorm: 0.94 [19:23:58< 2:35:52] +[titan] 2025-07-10 08:59:06,360 - root - INFO - step: 88195 loss: 16.2279 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.97% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7586 +[titan] 2025-07-10 08:59:06,360 - root - INFO - lr: 3.9362e-05 gnorm: 1.00 [19:24:02< 2:35:48] +[titan] 2025-07-10 08:59:09,501 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:59:10,286 - root - INFO - step: 88200 loss: 16.0448 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4415 global_avg_mtp_loss: 13.6034 +[titan] 2025-07-10 08:59:10,286 - root - INFO - lr: 3.9354e-05 gnorm: 0.95 [19:24:06< 2:35:44] +[titan] 2025-07-10 08:59:14,220 - root - INFO - step: 88205 loss: 16.2791 memory: 44.58GiB(31.99%) tps: 83,310 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.8045 +[titan] 2025-07-10 08:59:14,220 - root - INFO - lr: 3.9346e-05 gnorm: 1.02 [19:24:10< 2:35:40] +[titan] 2025-07-10 08:59:18,154 - root - INFO - step: 88210 loss: 16.2249 memory: 44.58GiB(31.99%) tps: 83,311 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 08:59:18,154 - root - INFO - lr: 3.9339e-05 gnorm: 0.98 [19:24:14< 2:35:36] +[titan] 2025-07-10 08:59:22,064 - root - INFO - step: 88215 loss: 16.2880 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.8084 +[titan] 2025-07-10 08:59:22,065 - root - INFO - lr: 3.9331e-05 gnorm: 0.98 [19:24:18< 2:35:32] +[titan] 2025-07-10 08:59:25,968 - root - INFO - step: 88220 loss: 16.0835 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.6274 +[titan] 2025-07-10 08:59:25,968 - root - INFO - lr: 3.9323e-05 gnorm: 0.98 [19:24:22< 2:35:28] +[titan] 2025-07-10 08:59:29,919 - root - INFO - step: 88225 loss: 16.0018 memory: 44.58GiB(31.99%) tps: 82,958 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.5644 +[titan] 2025-07-10 08:59:29,919 - root - INFO - lr: 3.9315e-05 gnorm: 1.04 [19:24:26< 2:35:24] +[titan] 2025-07-10 08:59:33,844 - root - INFO - step: 88230 loss: 16.0563 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4438 global_avg_mtp_loss: 13.6125 +[titan] 2025-07-10 08:59:33,844 - root - INFO - lr: 3.9307e-05 gnorm: 0.99 [19:24:30< 2:35:20] +[titan] 2025-07-10 08:59:37,771 - root - INFO - step: 88235 loss: 16.2792 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7973 +[titan] 2025-07-10 08:59:37,771 - root - INFO - lr: 3.9299e-05 gnorm: 0.99 [19:24:34< 2:35:16] +[titan] 2025-07-10 08:59:41,692 - root - INFO - step: 88240 loss: 16.4640 memory: 44.58GiB(31.99%) tps: 83,582 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.5193 global_avg_mtp_loss: 13.9447 +[titan] 2025-07-10 08:59:41,692 - root - INFO - lr: 3.9292e-05 gnorm: 0.98 [19:24:38< 2:35:12] +[titan] 2025-07-10 08:59:45,630 - root - INFO - step: 88245 loss: 16.0904 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6283 +[titan] 2025-07-10 08:59:45,630 - root - INFO - lr: 3.9284e-05 gnorm: 1.08 [19:24:42< 2:35:08] +[titan] 2025-07-10 08:59:48,759 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 08:59:49,553 - root - INFO - step: 88250 loss: 16.0823 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4448 global_avg_mtp_loss: 13.6375 +[titan] 2025-07-10 08:59:49,553 - root - INFO - lr: 3.9276e-05 gnorm: 0.98 [19:24:46< 2:35:04] +[titan] 2025-07-10 08:59:53,477 - root - INFO - step: 88255 loss: 16.3893 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8827 +[titan] 2025-07-10 08:59:53,477 - root - INFO - lr: 3.9268e-05 gnorm: 0.96 [19:24:49< 2:35:00] +[titan] 2025-07-10 08:59:57,408 - root - INFO - step: 88260 loss: 16.1204 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4464 global_avg_mtp_loss: 13.6740 +[titan] 2025-07-10 08:59:57,408 - root - INFO - lr: 3.9260e-05 gnorm: 1.02 [19:24:53< 2:34:57] +[titan] 2025-07-10 09:00:01,313 - root - INFO - step: 88265 loss: 16.0971 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4574 global_avg_mtp_loss: 13.6397 +[titan] 2025-07-10 09:00:01,313 - root - INFO - lr: 3.9253e-05 gnorm: 1.06 [19:24:57< 2:34:53] +[titan] 2025-07-10 09:00:05,236 - root - INFO - step: 88270 loss: 16.1867 memory: 44.58GiB(31.99%) tps: 83,536 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7045 +[titan] 2025-07-10 09:00:05,236 - root - INFO - lr: 3.9245e-05 gnorm: 1.01 [19:25:01< 2:34:49] +[titan] 2025-07-10 09:00:09,172 - root - INFO - step: 88275 loss: 16.2743 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7942 +[titan] 2025-07-10 09:00:09,172 - root - INFO - lr: 3.9237e-05 gnorm: 1.00 [19:25:05< 2:34:45] +[titan] 2025-07-10 09:00:13,071 - root - INFO - step: 88280 loss: 15.8752 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.07 mfu: 29.33% global_avg_ntp_loss: 2.4302 global_avg_mtp_loss: 13.4449 +[titan] 2025-07-10 09:00:13,071 - root - INFO - lr: 3.9229e-05 gnorm: 1.02 [19:25:09< 2:34:41] +[titan] 2025-07-10 09:00:16,996 - root - INFO - step: 88285 loss: 16.1744 memory: 44.58GiB(31.99%) tps: 83,495 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.6989 +[titan] 2025-07-10 09:00:16,996 - root - INFO - lr: 3.9222e-05 gnorm: 1.01 [19:25:13< 2:34:37] +[titan] 2025-07-10 09:00:20,917 - root - INFO - step: 88290 loss: 16.1263 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6677 +[titan] 2025-07-10 09:00:20,917 - root - INFO - lr: 3.9214e-05 gnorm: 0.99 [19:25:17< 2:34:33] +[titan] 2025-07-10 09:00:24,827 - root - INFO - step: 88295 loss: 16.4155 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.9076 +[titan] 2025-07-10 09:00:24,827 - root - INFO - lr: 3.9206e-05 gnorm: 1.06 [19:25:21< 2:34:29] +[titan] 2025-07-10 09:00:27,941 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:00:28,733 - root - INFO - step: 88300 loss: 16.1720 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7062 +[titan] 2025-07-10 09:00:28,734 - root - INFO - lr: 3.9198e-05 gnorm: 1.00 [19:25:25< 2:34:25] +[titan] 2025-07-10 09:00:32,676 - root - INFO - step: 88305 loss: 16.4157 memory: 44.58GiB(31.99%) tps: 83,130 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.5134 global_avg_mtp_loss: 13.9024 +[titan] 2025-07-10 09:00:32,676 - root - INFO - lr: 3.9190e-05 gnorm: 1.01 [19:25:29< 2:34:21] +[titan] 2025-07-10 09:00:36,593 - root - INFO - step: 88310 loss: 16.1773 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.7078 +[titan] 2025-07-10 09:00:36,594 - root - INFO - lr: 3.9183e-05 gnorm: 0.99 [19:25:33< 2:34:17] +[titan] 2025-07-10 09:00:40,553 - root - INFO - step: 88315 loss: 16.1675 memory: 44.58GiB(31.99%) tps: 82,756 tflops: 285.60 mfu: 28.88% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.6989 +[titan] 2025-07-10 09:00:40,554 - root - INFO - lr: 3.9175e-05 gnorm: 1.04 [19:25:37< 2:34:13] +[titan] 2025-07-10 09:00:44,501 - root - INFO - step: 88320 loss: 15.9410 memory: 44.58GiB(31.99%) tps: 83,019 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4221 global_avg_mtp_loss: 13.5189 +[titan] 2025-07-10 09:00:44,501 - root - INFO - lr: 3.9167e-05 gnorm: 1.02 [19:25:40< 2:34:09] +[titan] 2025-07-10 09:00:48,398 - root - INFO - step: 88325 loss: 16.2621 memory: 44.58GiB(31.99%) tps: 84,096 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7742 +[titan] 2025-07-10 09:00:48,398 - root - INFO - lr: 3.9159e-05 gnorm: 1.04 [19:25:44< 2:34:05] +[titan] 2025-07-10 09:00:52,340 - root - INFO - step: 88330 loss: 15.9904 memory: 44.58GiB(31.99%) tps: 83,117 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.5403 +[titan] 2025-07-10 09:00:52,341 - root - INFO - lr: 3.9152e-05 gnorm: 1.02 [19:25:48< 2:34:01] +[titan] 2025-07-10 09:00:56,258 - root - INFO - step: 88335 loss: 16.3160 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.8260 +[titan] 2025-07-10 09:00:56,258 - root - INFO - lr: 3.9144e-05 gnorm: 1.02 [19:25:52< 2:33:57] +[titan] 2025-07-10 09:01:00,171 - root - INFO - step: 88340 loss: 16.0495 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.5899 +[titan] 2025-07-10 09:01:00,171 - root - INFO - lr: 3.9136e-05 gnorm: 1.02 [19:25:56< 2:33:53] +[titan] 2025-07-10 09:01:04,081 - root - INFO - step: 88345 loss: 16.1557 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.6830 +[titan] 2025-07-10 09:01:04,082 - root - INFO - lr: 3.9128e-05 gnorm: 1.05 [19:26:00< 2:33:49] +[titan] 2025-07-10 09:01:07,202 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:01:07,994 - root - INFO - step: 88350 loss: 16.1242 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6635 +[titan] 2025-07-10 09:01:07,995 - root - INFO - lr: 3.9121e-05 gnorm: 0.99 [19:26:04< 2:33:45] +[titan] 2025-07-10 09:01:11,910 - root - INFO - step: 88355 loss: 16.2786 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7902 +[titan] 2025-07-10 09:01:11,911 - root - INFO - lr: 3.9113e-05 gnorm: 0.99 [19:26:08< 2:33:41] +[titan] 2025-07-10 09:01:15,811 - root - INFO - step: 88360 loss: 16.1118 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4411 global_avg_mtp_loss: 13.6706 +[titan] 2025-07-10 09:01:15,811 - root - INFO - lr: 3.9105e-05 gnorm: 0.99 [19:26:12< 2:33:37] +[titan] 2025-07-10 09:01:19,706 - root - INFO - step: 88365 loss: 16.2232 memory: 44.58GiB(31.99%) tps: 84,132 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7423 +[titan] 2025-07-10 09:01:19,706 - root - INFO - lr: 3.9097e-05 gnorm: 1.05 [19:26:16< 2:33:33] +[titan] 2025-07-10 09:01:23,604 - root - INFO - step: 88370 loss: 16.0510 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4445 global_avg_mtp_loss: 13.6065 +[titan] 2025-07-10 09:01:23,604 - root - INFO - lr: 3.9090e-05 gnorm: 1.04 [19:26:20< 2:33:29] +[titan] 2025-07-10 09:01:27,542 - root - INFO - step: 88375 loss: 16.1287 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.6601 +[titan] 2025-07-10 09:01:27,543 - root - INFO - lr: 3.9082e-05 gnorm: 1.06 [19:26:23< 2:33:25] +[titan] 2025-07-10 09:01:31,484 - root - INFO - step: 88380 loss: 16.2960 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8027 +[titan] 2025-07-10 09:01:31,485 - root - INFO - lr: 3.9074e-05 gnorm: 1.01 [19:26:27< 2:33:21] +[titan] 2025-07-10 09:01:35,400 - root - INFO - step: 88385 loss: 16.1674 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4683 global_avg_mtp_loss: 13.6991 +[titan] 2025-07-10 09:01:35,400 - root - INFO - lr: 3.9067e-05 gnorm: 0.96 [19:26:31< 2:33:17] +[titan] 2025-07-10 09:01:39,301 - root - INFO - step: 88390 loss: 16.3418 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 09:01:39,301 - root - INFO - lr: 3.9059e-05 gnorm: 1.02 [19:26:35< 2:33:13] +[titan] 2025-07-10 09:01:43,220 - root - INFO - step: 88395 loss: 16.1166 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4517 global_avg_mtp_loss: 13.6649 +[titan] 2025-07-10 09:01:43,220 - root - INFO - lr: 3.9051e-05 gnorm: 0.98 [19:26:39< 2:33:09] +[titan] 2025-07-10 09:01:46,342 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:01:47,137 - root - INFO - step: 88400 loss: 16.4684 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.5196 global_avg_mtp_loss: 13.9488 +[titan] 2025-07-10 09:01:47,137 - root - INFO - lr: 3.9043e-05 gnorm: 1.03 [19:26:43< 2:33:05] +[titan] 2025-07-10 09:01:51,047 - root - INFO - step: 88405 loss: 16.3706 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4967 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 09:01:51,047 - root - INFO - lr: 3.9036e-05 gnorm: 0.99 [19:26:47< 2:33:02] +[titan] 2025-07-10 09:01:54,969 - root - INFO - step: 88410 loss: 15.9749 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4300 global_avg_mtp_loss: 13.5450 +[titan] 2025-07-10 09:01:54,970 - root - INFO - lr: 3.9028e-05 gnorm: 1.02 [19:26:51< 2:32:58] +[titan] 2025-07-10 09:01:58,901 - root - INFO - step: 88415 loss: 15.9714 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4368 global_avg_mtp_loss: 13.5345 +[titan] 2025-07-10 09:01:58,902 - root - INFO - lr: 3.9020e-05 gnorm: 1.00 [19:26:55< 2:32:54] +[titan] 2025-07-10 09:02:02,813 - root - INFO - step: 88420 loss: 16.0764 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4404 global_avg_mtp_loss: 13.6360 +[titan] 2025-07-10 09:02:02,814 - root - INFO - lr: 3.9013e-05 gnorm: 1.03 [19:26:59< 2:32:50] +[titan] 2025-07-10 09:02:06,711 - root - INFO - step: 88425 loss: 16.4662 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9461 +[titan] 2025-07-10 09:02:06,711 - root - INFO - lr: 3.9005e-05 gnorm: 1.04 [19:27:03< 2:32:46] +[titan] 2025-07-10 09:02:10,625 - root - INFO - step: 88430 loss: 16.2310 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7534 +[titan] 2025-07-10 09:02:10,626 - root - INFO - lr: 3.8997e-05 gnorm: 1.06 [19:27:07< 2:32:42] +[titan] 2025-07-10 09:02:14,555 - root - INFO - step: 88435 loss: 16.1828 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.7153 +[titan] 2025-07-10 09:02:14,555 - root - INFO - lr: 3.8990e-05 gnorm: 1.08 [19:27:10< 2:32:38] +[titan] 2025-07-10 09:02:18,459 - root - INFO - step: 88440 loss: 16.3810 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.5042 global_avg_mtp_loss: 13.8768 +[titan] 2025-07-10 09:02:18,460 - root - INFO - lr: 3.8982e-05 gnorm: 0.99 [19:27:14< 2:32:34] +[titan] 2025-07-10 09:02:22,377 - root - INFO - step: 88445 loss: 16.1062 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6417 +[titan] 2025-07-10 09:02:22,377 - root - INFO - lr: 3.8974e-05 gnorm: 1.05 [19:27:18< 2:32:30] +[titan] 2025-07-10 09:02:25,491 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:02:26,278 - root - INFO - step: 88450 loss: 16.0413 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.6034 +[titan] 2025-07-10 09:02:26,278 - root - INFO - lr: 3.8966e-05 gnorm: 1.02 [19:27:22< 2:32:26] +[titan] 2025-07-10 09:02:30,181 - root - INFO - step: 88455 loss: 16.0679 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4477 global_avg_mtp_loss: 13.6202 +[titan] 2025-07-10 09:02:30,181 - root - INFO - lr: 3.8959e-05 gnorm: 1.00 [19:27:26< 2:32:22] +[titan] 2025-07-10 09:02:34,081 - root - INFO - step: 88460 loss: 16.0786 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4565 global_avg_mtp_loss: 13.6222 +[titan] 2025-07-10 09:02:34,081 - root - INFO - lr: 3.8951e-05 gnorm: 0.95 [19:27:30< 2:32:18] +[titan] 2025-07-10 09:02:37,971 - root - INFO - step: 88465 loss: 16.1811 memory: 44.58GiB(31.99%) tps: 84,235 tflops: 290.71 mfu: 29.39% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.7150 +[titan] 2025-07-10 09:02:37,972 - root - INFO - lr: 3.8943e-05 gnorm: 1.00 [19:27:34< 2:32:14] +[titan] 2025-07-10 09:02:41,869 - root - INFO - step: 88470 loss: 16.3435 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8464 +[titan] 2025-07-10 09:02:41,869 - root - INFO - lr: 3.8936e-05 gnorm: 0.99 [19:27:38< 2:32:10] +[titan] 2025-07-10 09:02:45,776 - root - INFO - step: 88475 loss: 16.1731 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.7161 +[titan] 2025-07-10 09:02:45,776 - root - INFO - lr: 3.8928e-05 gnorm: 1.03 [19:27:42< 2:32:06] +[titan] 2025-07-10 09:02:49,670 - root - INFO - step: 88480 loss: 16.3699 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8599 +[titan] 2025-07-10 09:02:49,671 - root - INFO - lr: 3.8920e-05 gnorm: 0.99 [19:27:46< 2:32:02] +[titan] 2025-07-10 09:02:53,568 - root - INFO - step: 88485 loss: 16.0042 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.4343 global_avg_mtp_loss: 13.5698 +[titan] 2025-07-10 09:02:53,568 - root - INFO - lr: 3.8913e-05 gnorm: 0.99 [19:27:50< 2:31:58] +[titan] 2025-07-10 09:02:57,502 - root - INFO - step: 88490 loss: 16.1122 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6644 +[titan] 2025-07-10 09:02:57,502 - root - INFO - lr: 3.8905e-05 gnorm: 1.00 [19:27:53< 2:31:54] +[titan] 2025-07-10 09:03:01,405 - root - INFO - step: 88495 loss: 16.3599 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 09:03:01,406 - root - INFO - lr: 3.8898e-05 gnorm: 1.01 [19:27:57< 2:31:50] +[titan] 2025-07-10 09:03:04,551 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:03:05,353 - root - INFO - step: 88500 loss: 16.3195 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.8337 +[titan] 2025-07-10 09:03:05,353 - root - INFO - lr: 3.8890e-05 gnorm: 0.98 [19:28:01< 2:31:46] +[titan] 2025-07-10 09:03:09,330 - root - INFO - step: 88505 loss: 16.3863 memory: 44.58GiB(31.99%) tps: 82,386 tflops: 284.33 mfu: 28.75% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.8804 +[titan] 2025-07-10 09:03:09,331 - root - INFO - lr: 3.8882e-05 gnorm: 1.03 [19:28:05< 2:31:42] +[titan] 2025-07-10 09:03:13,263 - root - INFO - step: 88510 loss: 15.9650 memory: 44.58GiB(31.99%) tps: 83,338 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.5330 +[titan] 2025-07-10 09:03:13,263 - root - INFO - lr: 3.8875e-05 gnorm: 0.99 [19:28:09< 2:31:38] +[titan] 2025-07-10 09:03:17,185 - root - INFO - step: 88515 loss: 16.3631 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8603 +[titan] 2025-07-10 09:03:17,185 - root - INFO - lr: 3.8867e-05 gnorm: 0.99 [19:28:13< 2:31:34] +[titan] 2025-07-10 09:03:21,120 - root - INFO - step: 88520 loss: 16.0046 memory: 44.58GiB(31.99%) tps: 83,276 tflops: 287.40 mfu: 29.06% global_avg_ntp_loss: 2.4363 global_avg_mtp_loss: 13.5683 +[titan] 2025-07-10 09:03:21,120 - root - INFO - lr: 3.8859e-05 gnorm: 1.08 [19:28:17< 2:31:30] +[titan] 2025-07-10 09:03:25,102 - root - INFO - step: 88525 loss: 16.0872 memory: 44.58GiB(31.99%) tps: 82,293 tflops: 284.01 mfu: 28.72% global_avg_ntp_loss: 2.4408 global_avg_mtp_loss: 13.6465 +[titan] 2025-07-10 09:03:25,103 - root - INFO - lr: 3.8852e-05 gnorm: 1.02 [19:28:21< 2:31:26] +[titan] 2025-07-10 09:03:29,040 - root - INFO - step: 88530 loss: 16.2268 memory: 44.58GiB(31.99%) tps: 83,219 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 09:03:29,041 - root - INFO - lr: 3.8844e-05 gnorm: 1.05 [19:28:25< 2:31:22] +[titan] 2025-07-10 09:03:32,999 - root - INFO - step: 88535 loss: 16.1978 memory: 44.58GiB(31.99%) tps: 82,778 tflops: 285.68 mfu: 28.89% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.7061 +[titan] 2025-07-10 09:03:32,999 - root - INFO - lr: 3.8836e-05 gnorm: 1.00 [19:28:29< 2:31:18] +[titan] 2025-07-10 09:03:36,923 - root - INFO - step: 88540 loss: 16.1701 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.6979 +[titan] 2025-07-10 09:03:36,923 - root - INFO - lr: 3.8829e-05 gnorm: 1.04 [19:28:33< 2:31:14] +[titan] 2025-07-10 09:03:40,813 - root - INFO - step: 88545 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 84,234 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6627 +[titan] 2025-07-10 09:03:40,813 - root - INFO - lr: 3.8821e-05 gnorm: 1.01 [19:28:37< 2:31:11] +[titan] 2025-07-10 09:03:43,931 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:03:44,725 - root - INFO - step: 88550 loss: 16.2786 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.8097 +[titan] 2025-07-10 09:03:44,725 - root - INFO - lr: 3.8814e-05 gnorm: 1.07 [19:28:41< 2:31:07] +[titan] 2025-07-10 09:03:48,630 - root - INFO - step: 88555 loss: 16.1675 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4557 global_avg_mtp_loss: 13.7118 +[titan] 2025-07-10 09:03:48,630 - root - INFO - lr: 3.8806e-05 gnorm: 1.03 [19:28:45< 2:31:03] +[titan] 2025-07-10 09:03:52,552 - root - INFO - step: 88560 loss: 16.3748 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 09:03:52,552 - root - INFO - lr: 3.8798e-05 gnorm: 1.00 [19:28:48< 2:30:59] +[titan] 2025-07-10 09:03:56,455 - root - INFO - step: 88565 loss: 16.2202 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7301 +[titan] 2025-07-10 09:03:56,455 - root - INFO - lr: 3.8791e-05 gnorm: 1.00 [19:28:52< 2:30:55] +[titan] 2025-07-10 09:04:00,371 - root - INFO - step: 88570 loss: 16.2658 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 09:04:00,372 - root - INFO - lr: 3.8783e-05 gnorm: 1.04 [19:28:56< 2:30:51] +[titan] 2025-07-10 09:04:04,371 - root - INFO - step: 88575 loss: 16.1533 memory: 44.58GiB(31.99%) tps: 81,935 tflops: 282.77 mfu: 28.59% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6884 +[titan] 2025-07-10 09:04:04,371 - root - INFO - lr: 3.8776e-05 gnorm: 1.01 [19:29:00< 2:30:47] +[titan] 2025-07-10 09:04:05,315 - root - INFO - Dumping profiler traces at step 88576 +[titan] 2025-07-10 09:04:05,349 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:04:08,479 - root - INFO - step: 88580 loss: 16.2494 memory: 44.58GiB(31.99%) tps: 79,783 tflops: 275.34 mfu: 27.84% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7613 +[titan] 2025-07-10 09:04:08,479 - root - INFO - lr: 3.8768e-05 gnorm: 0.99 [19:29:04< 2:30:43] +[titan] 2025-07-10 09:04:12,433 - root - INFO - step: 88585 loss: 16.3654 memory: 44.58GiB(31.99%) tps: 82,870 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5084 global_avg_mtp_loss: 13.8570 +[titan] 2025-07-10 09:04:12,433 - root - INFO - lr: 3.8760e-05 gnorm: 0.99 [19:29:08< 2:30:39] +[titan] 2025-07-10 09:04:16,339 - root - INFO - step: 88590 loss: 16.1293 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6673 +[titan] 2025-07-10 09:04:16,339 - root - INFO - lr: 3.8753e-05 gnorm: 1.08 [19:29:12< 2:30:35] +[titan] 2025-07-10 09:04:20,281 - root - INFO - step: 88595 loss: 16.1588 memory: 44.58GiB(31.99%) tps: 83,116 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.6850 +[titan] 2025-07-10 09:04:20,282 - root - INFO - lr: 3.8745e-05 gnorm: 1.02 [19:29:16< 2:30:31] +[titan] 2025-07-10 09:04:23,411 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:04:24,216 - root - INFO - step: 88600 loss: 16.1397 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6760 +[titan] 2025-07-10 09:04:24,216 - root - INFO - lr: 3.8738e-05 gnorm: 1.00 [19:29:20< 2:30:27] +[titan] 2025-07-10 09:04:28,174 - root - INFO - step: 88605 loss: 16.2223 memory: 44.58GiB(31.99%) tps: 82,800 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7441 +[titan] 2025-07-10 09:04:28,174 - root - INFO - lr: 3.8730e-05 gnorm: 1.03 [19:29:24< 2:30:23] +[titan] 2025-07-10 09:04:32,080 - root - INFO - step: 88610 loss: 16.1515 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6895 +[titan] 2025-07-10 09:04:32,081 - root - INFO - lr: 3.8722e-05 gnorm: 1.00 [19:29:28< 2:30:19] +[titan] 2025-07-10 09:04:35,982 - root - INFO - step: 88615 loss: 16.0711 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.6116 +[titan] 2025-07-10 09:04:35,982 - root - INFO - lr: 3.8715e-05 gnorm: 1.03 [19:29:32< 2:30:15] +[titan] 2025-07-10 09:04:39,889 - root - INFO - step: 88620 loss: 16.2574 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7790 +[titan] 2025-07-10 09:04:39,889 - root - INFO - lr: 3.8707e-05 gnorm: 1.02 [19:29:36< 2:30:11] +[titan] 2025-07-10 09:04:43,791 - root - INFO - step: 88625 loss: 16.3165 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8246 +[titan] 2025-07-10 09:04:43,791 - root - INFO - lr: 3.8700e-05 gnorm: 1.05 [19:29:40< 2:30:07] +[titan] 2025-07-10 09:04:47,732 - root - INFO - step: 88630 loss: 16.2379 memory: 44.58GiB(31.99%) tps: 83,153 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.7413 +[titan] 2025-07-10 09:04:47,733 - root - INFO - lr: 3.8692e-05 gnorm: 1.02 [19:29:44< 2:30:03] +[titan] 2025-07-10 09:04:51,641 - root - INFO - step: 88635 loss: 16.2412 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.7500 +[titan] 2025-07-10 09:04:51,642 - root - INFO - lr: 3.8685e-05 gnorm: 1.02 [19:29:48< 2:29:59] +[titan] 2025-07-10 09:04:55,603 - root - INFO - step: 88640 loss: 16.2538 memory: 44.58GiB(31.99%) tps: 82,726 tflops: 285.50 mfu: 28.87% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 09:04:55,603 - root - INFO - lr: 3.8677e-05 gnorm: 1.01 [19:29:52< 2:29:55] +[titan] 2025-07-10 09:04:59,537 - root - INFO - step: 88645 loss: 16.3897 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 09:04:59,537 - root - INFO - lr: 3.8670e-05 gnorm: 0.98 [19:29:55< 2:29:51] +[titan] 2025-07-10 09:05:02,647 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:05:03,452 - root - INFO - step: 88650 loss: 16.1314 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4531 global_avg_mtp_loss: 13.6783 +[titan] 2025-07-10 09:05:03,453 - root - INFO - lr: 3.8662e-05 gnorm: 1.00 [19:29:59< 2:29:47] +[titan] 2025-07-10 09:05:07,357 - root - INFO - step: 88655 loss: 16.3589 memory: 44.58GiB(31.99%) tps: 83,937 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.8503 +[titan] 2025-07-10 09:05:07,357 - root - INFO - lr: 3.8654e-05 gnorm: 0.96 [19:30:03< 2:29:43] +[titan] 2025-07-10 09:05:11,312 - root - INFO - step: 88660 loss: 16.3840 memory: 44.58GiB(31.99%) tps: 82,859 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 09:05:11,312 - root - INFO - lr: 3.8647e-05 gnorm: 1.01 [19:30:07< 2:29:39] +[titan] 2025-07-10 09:05:15,218 - root - INFO - step: 88665 loss: 16.3164 memory: 44.58GiB(31.99%) tps: 83,895 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8215 +[titan] 2025-07-10 09:05:15,218 - root - INFO - lr: 3.8639e-05 gnorm: 1.04 [19:30:11< 2:29:35] +[titan] 2025-07-10 09:05:19,197 - root - INFO - step: 88670 loss: 16.2153 memory: 44.58GiB(31.99%) tps: 82,355 tflops: 284.22 mfu: 28.74% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7330 +[titan] 2025-07-10 09:05:19,198 - root - INFO - lr: 3.8632e-05 gnorm: 1.00 [19:30:15< 2:29:31] +[titan] 2025-07-10 09:05:23,143 - root - INFO - step: 88675 loss: 16.0952 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.6530 +[titan] 2025-07-10 09:05:23,144 - root - INFO - lr: 3.8624e-05 gnorm: 1.06 [19:30:19< 2:29:27] +[titan] 2025-07-10 09:05:27,043 - root - INFO - step: 88680 loss: 16.1413 memory: 44.58GiB(31.99%) tps: 84,039 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6748 +[titan] 2025-07-10 09:05:27,043 - root - INFO - lr: 3.8617e-05 gnorm: 0.99 [19:30:23< 2:29:24] +[titan] 2025-07-10 09:05:30,943 - root - INFO - step: 88685 loss: 16.0199 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.5726 +[titan] 2025-07-10 09:05:30,943 - root - INFO - lr: 3.8609e-05 gnorm: 0.98 [19:30:27< 2:29:20] +[titan] 2025-07-10 09:05:34,860 - root - INFO - step: 88690 loss: 16.0154 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4352 global_avg_mtp_loss: 13.5802 +[titan] 2025-07-10 09:05:34,860 - root - INFO - lr: 3.8602e-05 gnorm: 1.02 [19:30:31< 2:29:16] +[titan] 2025-07-10 09:05:38,764 - root - INFO - step: 88695 loss: 16.4129 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.9076 +[titan] 2025-07-10 09:05:38,764 - root - INFO - lr: 3.8594e-05 gnorm: 1.04 [19:30:35< 2:29:12] +[titan] 2025-07-10 09:05:41,920 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:05:42,712 - root - INFO - step: 88700 loss: 16.0833 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4541 global_avg_mtp_loss: 13.6292 +[titan] 2025-07-10 09:05:42,712 - root - INFO - lr: 3.8587e-05 gnorm: 1.03 [19:30:39< 2:29:08] +[titan] 2025-07-10 09:05:46,672 - root - INFO - step: 88705 loss: 16.2807 memory: 44.58GiB(31.99%) tps: 82,749 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7995 +[titan] 2025-07-10 09:05:46,672 - root - INFO - lr: 3.8579e-05 gnorm: 0.98 [19:30:43< 2:29:04] +[titan] 2025-07-10 09:05:50,588 - root - INFO - step: 88710 loss: 16.4864 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5278 global_avg_mtp_loss: 13.9586 +[titan] 2025-07-10 09:05:50,588 - root - INFO - lr: 3.8572e-05 gnorm: 0.99 [19:30:47< 2:29:00] +[titan] 2025-07-10 09:05:54,507 - root - INFO - step: 88715 loss: 16.3409 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8358 +[titan] 2025-07-10 09:05:54,508 - root - INFO - lr: 3.8564e-05 gnorm: 0.97 [19:30:50< 2:28:56] +[titan] 2025-07-10 09:05:58,408 - root - INFO - step: 88720 loss: 16.3031 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8055 +[titan] 2025-07-10 09:05:58,408 - root - INFO - lr: 3.8557e-05 gnorm: 1.03 [19:30:54< 2:28:52] +[titan] 2025-07-10 09:06:02,311 - root - INFO - step: 88725 loss: 16.4952 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.5229 global_avg_mtp_loss: 13.9723 +[titan] 2025-07-10 09:06:02,312 - root - INFO - lr: 3.8549e-05 gnorm: 1.01 [19:30:58< 2:28:48] +[titan] 2025-07-10 09:06:06,227 - root - INFO - step: 88730 loss: 16.0842 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6260 +[titan] 2025-07-10 09:06:06,227 - root - INFO - lr: 3.8542e-05 gnorm: 1.01 [19:31:02< 2:28:44] +[titan] 2025-07-10 09:06:10,139 - root - INFO - step: 88735 loss: 16.2894 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.8035 +[titan] 2025-07-10 09:06:10,139 - root - INFO - lr: 3.8534e-05 gnorm: 1.01 [19:31:06< 2:28:40] +[titan] 2025-07-10 09:06:14,102 - root - INFO - step: 88740 loss: 16.2658 memory: 44.58GiB(31.99%) tps: 82,681 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 09:06:14,103 - root - INFO - lr: 3.8527e-05 gnorm: 1.01 [19:31:10< 2:28:36] +[titan] 2025-07-10 09:06:18,010 - root - INFO - step: 88745 loss: 16.1483 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6892 +[titan] 2025-07-10 09:06:18,011 - root - INFO - lr: 3.8519e-05 gnorm: 0.99 [19:31:14< 2:28:32] +[titan] 2025-07-10 09:06:21,139 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:06:21,922 - root - INFO - step: 88750 loss: 16.3445 memory: 44.58GiB(31.99%) tps: 83,772 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.8524 +[titan] 2025-07-10 09:06:21,923 - root - INFO - lr: 3.8512e-05 gnorm: 1.01 [19:31:18< 2:28:28] +[titan] 2025-07-10 09:06:25,837 - root - INFO - step: 88755 loss: 16.2398 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7642 +[titan] 2025-07-10 09:06:25,837 - root - INFO - lr: 3.8504e-05 gnorm: 0.97 [19:31:22< 2:28:24] +[titan] 2025-07-10 09:06:29,754 - root - INFO - step: 88760 loss: 16.1069 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.6579 +[titan] 2025-07-10 09:06:29,754 - root - INFO - lr: 3.8497e-05 gnorm: 1.04 [19:31:26< 2:28:20] +[titan] 2025-07-10 09:06:33,699 - root - INFO - step: 88765 loss: 16.3748 memory: 44.58GiB(31.99%) tps: 83,074 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8679 +[titan] 2025-07-10 09:06:33,699 - root - INFO - lr: 3.8489e-05 gnorm: 1.01 [19:31:30< 2:28:16] +[titan] 2025-07-10 09:06:37,603 - root - INFO - step: 88770 loss: 16.0195 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.5675 +[titan] 2025-07-10 09:06:37,603 - root - INFO - lr: 3.8482e-05 gnorm: 1.01 [19:31:34< 2:28:12] +[titan] 2025-07-10 09:06:41,535 - root - INFO - step: 88775 loss: 16.3033 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.8180 +[titan] 2025-07-10 09:06:41,535 - root - INFO - lr: 3.8474e-05 gnorm: 1.04 [19:31:37< 2:28:08] +[titan] 2025-07-10 09:06:45,439 - root - INFO - step: 88780 loss: 16.1258 memory: 44.58GiB(31.99%) tps: 83,936 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4605 global_avg_mtp_loss: 13.6653 +[titan] 2025-07-10 09:06:45,439 - root - INFO - lr: 3.8467e-05 gnorm: 1.03 [19:31:41< 2:28:04] +[titan] 2025-07-10 09:06:49,401 - root - INFO - step: 88785 loss: 16.2058 memory: 44.58GiB(31.99%) tps: 82,714 tflops: 285.46 mfu: 28.86% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7174 +[titan] 2025-07-10 09:06:49,401 - root - INFO - lr: 3.8459e-05 gnorm: 0.99 [19:31:45< 2:28:00] +[titan] 2025-07-10 09:06:53,354 - root - INFO - step: 88790 loss: 16.0080 memory: 44.58GiB(31.99%) tps: 82,892 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.4400 global_avg_mtp_loss: 13.5681 +[titan] 2025-07-10 09:06:53,355 - root - INFO - lr: 3.8452e-05 gnorm: 1.03 [19:31:49< 2:27:56] +[titan] 2025-07-10 09:06:57,276 - root - INFO - step: 88795 loss: 16.2806 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7987 +[titan] 2025-07-10 09:06:57,276 - root - INFO - lr: 3.8444e-05 gnorm: 1.01 [19:31:53< 2:27:52] +[titan] 2025-07-10 09:07:00,434 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:07:01,225 - root - INFO - step: 88800 loss: 16.0686 memory: 44.58GiB(31.99%) tps: 82,995 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.6225 +[titan] 2025-07-10 09:07:01,225 - root - INFO - lr: 3.8437e-05 gnorm: 1.01 [19:31:57< 2:27:48] +[titan] 2025-07-10 09:07:05,146 - root - INFO - step: 88805 loss: 16.1035 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6488 +[titan] 2025-07-10 09:07:05,147 - root - INFO - lr: 3.8429e-05 gnorm: 1.03 [19:32:01< 2:27:44] +[titan] 2025-07-10 09:07:09,070 - root - INFO - step: 88810 loss: 16.1972 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7196 +[titan] 2025-07-10 09:07:09,070 - root - INFO - lr: 3.8422e-05 gnorm: 1.00 [19:32:05< 2:27:40] +[titan] 2025-07-10 09:07:12,971 - root - INFO - step: 88815 loss: 16.3866 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.8777 +[titan] 2025-07-10 09:07:12,971 - root - INFO - lr: 3.8415e-05 gnorm: 1.07 [19:32:09< 2:27:36] +[titan] 2025-07-10 09:07:16,901 - root - INFO - step: 88820 loss: 16.1279 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.6592 +[titan] 2025-07-10 09:07:16,901 - root - INFO - lr: 3.8407e-05 gnorm: 1.01 [19:32:13< 2:27:33] +[titan] 2025-07-10 09:07:20,798 - root - INFO - step: 88825 loss: 16.0684 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4416 global_avg_mtp_loss: 13.6268 +[titan] 2025-07-10 09:07:20,798 - root - INFO - lr: 3.8400e-05 gnorm: 1.03 [19:32:17< 2:27:29] +[titan] 2025-07-10 09:07:24,698 - root - INFO - step: 88830 loss: 16.2361 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7675 +[titan] 2025-07-10 09:07:24,698 - root - INFO - lr: 3.8392e-05 gnorm: 1.05 [19:32:21< 2:27:25] +[titan] 2025-07-10 09:07:28,606 - root - INFO - step: 88835 loss: 16.2382 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7678 +[titan] 2025-07-10 09:07:28,606 - root - INFO - lr: 3.8385e-05 gnorm: 1.03 [19:32:25< 2:27:21] +[titan] 2025-07-10 09:07:32,515 - root - INFO - step: 88840 loss: 15.8462 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4200 global_avg_mtp_loss: 13.4262 +[titan] 2025-07-10 09:07:32,515 - root - INFO - lr: 3.8377e-05 gnorm: 1.08 [19:32:28< 2:27:17] +[titan] 2025-07-10 09:07:36,433 - root - INFO - step: 88845 loss: 16.1447 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 09:07:36,433 - root - INFO - lr: 3.8370e-05 gnorm: 1.01 [19:32:32< 2:27:13] +[titan] 2025-07-10 09:07:39,558 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:07:40,353 - root - INFO - step: 88850 loss: 15.9396 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4220 global_avg_mtp_loss: 13.5176 +[titan] 2025-07-10 09:07:40,353 - root - INFO - lr: 3.8363e-05 gnorm: 1.06 [19:32:36< 2:27:09] +[titan] 2025-07-10 09:07:44,290 - root - INFO - step: 88855 loss: 16.2563 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7810 +[titan] 2025-07-10 09:07:44,291 - root - INFO - lr: 3.8355e-05 gnorm: 1.03 [19:32:40< 2:27:05] +[titan] 2025-07-10 09:07:48,248 - root - INFO - step: 88860 loss: 16.5146 memory: 44.58GiB(31.99%) tps: 82,813 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.5308 global_avg_mtp_loss: 13.9839 +[titan] 2025-07-10 09:07:48,248 - root - INFO - lr: 3.8348e-05 gnorm: 0.99 [19:32:44< 2:27:01] +[titan] 2025-07-10 09:07:52,221 - root - INFO - step: 88865 loss: 16.0305 memory: 44.58GiB(31.99%) tps: 82,465 tflops: 284.60 mfu: 28.78% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5813 +[titan] 2025-07-10 09:07:52,222 - root - INFO - lr: 3.8340e-05 gnorm: 1.09 [19:32:48< 2:26:57] +[titan] 2025-07-10 09:07:56,152 - root - INFO - step: 88870 loss: 15.9231 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4199 global_avg_mtp_loss: 13.5032 +[titan] 2025-07-10 09:07:56,153 - root - INFO - lr: 3.8333e-05 gnorm: 1.07 [19:32:52< 2:26:53] +[titan] 2025-07-10 09:08:00,081 - root - INFO - step: 88875 loss: 16.2164 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7397 +[titan] 2025-07-10 09:08:00,081 - root - INFO - lr: 3.8325e-05 gnorm: 1.00 [19:32:56< 2:26:49] +[titan] 2025-07-10 09:08:03,984 - root - INFO - step: 88880 loss: 16.2350 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4930 global_avg_mtp_loss: 13.7419 +[titan] 2025-07-10 09:08:03,984 - root - INFO - lr: 3.8318e-05 gnorm: 1.04 [19:33:00< 2:26:45] +[titan] 2025-07-10 09:08:07,903 - root - INFO - step: 88885 loss: 16.1746 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.6988 +[titan] 2025-07-10 09:08:07,904 - root - INFO - lr: 3.8311e-05 gnorm: 1.04 [19:33:04< 2:26:41] +[titan] 2025-07-10 09:08:11,860 - root - INFO - step: 88890 loss: 16.3658 memory: 44.58GiB(31.99%) tps: 82,832 tflops: 285.87 mfu: 28.90% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8734 +[titan] 2025-07-10 09:08:11,860 - root - INFO - lr: 3.8303e-05 gnorm: 1.00 [19:33:08< 2:26:37] +[titan] 2025-07-10 09:08:15,815 - root - INFO - step: 88895 loss: 16.3484 memory: 44.58GiB(31.99%) tps: 82,844 tflops: 285.91 mfu: 28.91% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.8403 +[titan] 2025-07-10 09:08:15,816 - root - INFO - lr: 3.8296e-05 gnorm: 1.06 [19:33:12< 2:26:33] +[titan] 2025-07-10 09:08:18,959 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:08:19,746 - root - INFO - step: 88900 loss: 16.3682 memory: 44.58GiB(31.99%) tps: 83,386 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8629 +[titan] 2025-07-10 09:08:19,746 - root - INFO - lr: 3.8288e-05 gnorm: 1.01 [19:33:16< 2:26:29] +[titan] 2025-07-10 09:08:23,680 - root - INFO - step: 88905 loss: 16.2340 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7554 +[titan] 2025-07-10 09:08:23,680 - root - INFO - lr: 3.8281e-05 gnorm: 1.02 [19:33:20< 2:26:25] +[titan] 2025-07-10 09:08:27,590 - root - INFO - step: 88910 loss: 16.1355 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6738 +[titan] 2025-07-10 09:08:27,591 - root - INFO - lr: 3.8274e-05 gnorm: 1.00 [19:33:23< 2:26:21] +[titan] 2025-07-10 09:08:31,496 - root - INFO - step: 88915 loss: 16.1250 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6811 +[titan] 2025-07-10 09:08:31,496 - root - INFO - lr: 3.8266e-05 gnorm: 1.01 [19:33:27< 2:26:17] +[titan] 2025-07-10 09:08:35,397 - root - INFO - step: 88920 loss: 15.9958 memory: 44.58GiB(31.99%) tps: 83,989 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.5620 +[titan] 2025-07-10 09:08:35,398 - root - INFO - lr: 3.8259e-05 gnorm: 1.00 [19:33:31< 2:26:13] +[titan] 2025-07-10 09:08:39,345 - root - INFO - step: 88925 loss: 16.2695 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7877 +[titan] 2025-07-10 09:08:39,345 - root - INFO - lr: 3.8252e-05 gnorm: 1.00 [19:33:35< 2:26:09] +[titan] 2025-07-10 09:08:43,254 - root - INFO - step: 88930 loss: 16.2416 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7652 +[titan] 2025-07-10 09:08:43,254 - root - INFO - lr: 3.8244e-05 gnorm: 0.98 [19:33:39< 2:26:05] +[titan] 2025-07-10 09:08:47,150 - root - INFO - step: 88935 loss: 16.2116 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4749 global_avg_mtp_loss: 13.7367 +[titan] 2025-07-10 09:08:47,151 - root - INFO - lr: 3.8237e-05 gnorm: 1.01 [19:33:43< 2:26:01] +[titan] 2025-07-10 09:08:51,062 - root - INFO - step: 88940 loss: 16.2254 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7417 +[titan] 2025-07-10 09:08:51,063 - root - INFO - lr: 3.8229e-05 gnorm: 1.05 [19:33:47< 2:25:57] +[titan] 2025-07-10 09:08:54,976 - root - INFO - step: 88945 loss: 16.1488 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.6523 +[titan] 2025-07-10 09:08:54,977 - root - INFO - lr: 3.8222e-05 gnorm: 1.07 [19:33:51< 2:25:53] +[titan] 2025-07-10 09:08:58,113 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:08:58,917 - root - INFO - step: 88950 loss: 16.0338 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.5837 +[titan] 2025-07-10 09:08:58,917 - root - INFO - lr: 3.8215e-05 gnorm: 0.99 [19:33:55< 2:25:49] +[titan] 2025-07-10 09:09:02,810 - root - INFO - step: 88955 loss: 15.9180 memory: 44.58GiB(31.99%) tps: 84,178 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.4269 global_avg_mtp_loss: 13.4911 +[titan] 2025-07-10 09:09:02,810 - root - INFO - lr: 3.8207e-05 gnorm: 1.05 [19:33:59< 2:25:46] +[titan] 2025-07-10 09:09:06,750 - root - INFO - step: 88960 loss: 16.1368 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.6784 +[titan] 2025-07-10 09:09:06,751 - root - INFO - lr: 3.8200e-05 gnorm: 1.00 [19:34:03< 2:25:42] +[titan] 2025-07-10 09:09:10,638 - root - INFO - step: 88965 loss: 16.1994 memory: 44.58GiB(31.99%) tps: 84,287 tflops: 290.89 mfu: 29.41% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7283 +[titan] 2025-07-10 09:09:10,639 - root - INFO - lr: 3.8193e-05 gnorm: 1.09 [19:34:07< 2:25:38] +[titan] 2025-07-10 09:09:14,544 - root - INFO - step: 88970 loss: 16.0420 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4479 global_avg_mtp_loss: 13.5941 +[titan] 2025-07-10 09:09:14,545 - root - INFO - lr: 3.8185e-05 gnorm: 0.98 [19:34:10< 2:25:34] +[titan] 2025-07-10 09:09:18,451 - root - INFO - step: 88975 loss: 15.9646 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4283 global_avg_mtp_loss: 13.5363 +[titan] 2025-07-10 09:09:18,451 - root - INFO - lr: 3.8178e-05 gnorm: 1.00 [19:34:14< 2:25:30] +[titan] 2025-07-10 09:09:22,372 - root - INFO - step: 88980 loss: 16.3692 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5007 global_avg_mtp_loss: 13.8685 +[titan] 2025-07-10 09:09:22,372 - root - INFO - lr: 3.8171e-05 gnorm: 1.02 [19:34:18< 2:25:26] +[titan] 2025-07-10 09:09:26,314 - root - INFO - step: 88985 loss: 15.9874 memory: 44.58GiB(31.99%) tps: 83,126 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.5493 +[titan] 2025-07-10 09:09:26,315 - root - INFO - lr: 3.8163e-05 gnorm: 1.09 [19:34:22< 2:25:22] +[titan] 2025-07-10 09:09:30,223 - root - INFO - step: 88990 loss: 16.3490 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5158 global_avg_mtp_loss: 13.8332 +[titan] 2025-07-10 09:09:30,223 - root - INFO - lr: 3.8156e-05 gnorm: 1.02 [19:34:26< 2:25:18] +[titan] 2025-07-10 09:09:34,135 - root - INFO - step: 88995 loss: 16.3126 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.8190 +[titan] 2025-07-10 09:09:34,136 - root - INFO - lr: 3.8149e-05 gnorm: 1.04 [19:34:30< 2:25:14] +[titan] 2025-07-10 09:09:37,250 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:09:38,039 - root - INFO - step: 89000 loss: 16.2075 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7315 +[titan] 2025-07-10 09:09:38,040 - root - INFO - lr: 3.8141e-05 gnorm: 1.03 [19:34:34< 2:25:10] +[titan] 2025-07-10 09:09:41,956 - root - INFO - step: 89005 loss: 16.1502 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6840 +[titan] 2025-07-10 09:09:41,956 - root - INFO - lr: 3.8134e-05 gnorm: 1.03 [19:34:38< 2:25:06] +[titan] 2025-07-10 09:09:45,853 - root - INFO - step: 89010 loss: 15.7442 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.4015 global_avg_mtp_loss: 13.3427 +[titan] 2025-07-10 09:09:45,853 - root - INFO - lr: 3.8127e-05 gnorm: 1.00 [19:34:42< 2:25:02] +[titan] 2025-07-10 09:09:49,765 - root - INFO - step: 89015 loss: 16.2502 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7704 +[titan] 2025-07-10 09:09:49,765 - root - INFO - lr: 3.8119e-05 gnorm: 0.95 [19:34:46< 2:24:58] +[titan] 2025-07-10 09:09:53,710 - root - INFO - step: 89020 loss: 15.8546 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4259 global_avg_mtp_loss: 13.4287 +[titan] 2025-07-10 09:09:53,710 - root - INFO - lr: 3.8112e-05 gnorm: 1.00 [19:34:50< 2:24:54] +[titan] 2025-07-10 09:09:57,615 - root - INFO - step: 89025 loss: 16.3723 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8693 +[titan] 2025-07-10 09:09:57,616 - root - INFO - lr: 3.8105e-05 gnorm: 1.04 [19:34:54< 2:24:50] +[titan] 2025-07-10 09:10:01,520 - root - INFO - step: 89030 loss: 16.0767 memory: 44.58GiB(31.99%) tps: 83,941 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.6123 +[titan] 2025-07-10 09:10:01,520 - root - INFO - lr: 3.8097e-05 gnorm: 0.99 [19:34:57< 2:24:46] +[titan] 2025-07-10 09:10:05,432 - root - INFO - step: 89035 loss: 16.0807 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4412 global_avg_mtp_loss: 13.6395 +[titan] 2025-07-10 09:10:05,433 - root - INFO - lr: 3.8090e-05 gnorm: 0.99 [19:35:01< 2:24:42] +[titan] 2025-07-10 09:10:09,354 - root - INFO - step: 89040 loss: 16.1114 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6571 +[titan] 2025-07-10 09:10:09,354 - root - INFO - lr: 3.8083e-05 gnorm: 0.96 [19:35:05< 2:24:38] +[titan] 2025-07-10 09:10:13,262 - root - INFO - step: 89045 loss: 16.2481 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7622 +[titan] 2025-07-10 09:10:13,262 - root - INFO - lr: 3.8076e-05 gnorm: 1.00 [19:35:09< 2:24:34] +[titan] 2025-07-10 09:10:16,372 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:10:17,162 - root - INFO - step: 89050 loss: 16.4125 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.9059 +[titan] 2025-07-10 09:10:17,162 - root - INFO - lr: 3.8068e-05 gnorm: 0.99 [19:35:13< 2:24:30] +[titan] 2025-07-10 09:10:21,085 - root - INFO - step: 89055 loss: 16.0580 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.6071 +[titan] 2025-07-10 09:10:21,085 - root - INFO - lr: 3.8061e-05 gnorm: 0.96 [19:35:17< 2:24:26] +[titan] 2025-07-10 09:10:25,005 - root - INFO - step: 89060 loss: 16.2128 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7315 +[titan] 2025-07-10 09:10:25,005 - root - INFO - lr: 3.8054e-05 gnorm: 1.02 [19:35:21< 2:24:22] +[titan] 2025-07-10 09:10:28,923 - root - INFO - step: 89065 loss: 16.3357 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.8443 +[titan] 2025-07-10 09:10:28,923 - root - INFO - lr: 3.8046e-05 gnorm: 0.98 [19:35:25< 2:24:18] +[titan] 2025-07-10 09:10:32,826 - root - INFO - step: 89070 loss: 16.0087 memory: 44.58GiB(31.99%) tps: 83,969 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4364 global_avg_mtp_loss: 13.5723 +[titan] 2025-07-10 09:10:32,826 - root - INFO - lr: 3.8039e-05 gnorm: 1.00 [19:35:29< 2:24:14] +[titan] 2025-07-10 09:10:36,745 - root - INFO - step: 89075 loss: 16.0523 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4420 global_avg_mtp_loss: 13.6103 +[titan] 2025-07-10 09:10:36,746 - root - INFO - lr: 3.8032e-05 gnorm: 1.03 [19:35:33< 2:24:10] +[titan] 2025-07-10 09:10:40,651 - root - INFO - step: 89080 loss: 16.1431 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.6879 +[titan] 2025-07-10 09:10:40,652 - root - INFO - lr: 3.8025e-05 gnorm: 0.99 [19:35:37< 2:24:06] +[titan] 2025-07-10 09:10:44,664 - root - INFO - step: 89085 loss: 15.9888 memory: 44.58GiB(31.99%) tps: 81,677 tflops: 281.88 mfu: 28.50% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.5281 +[titan] 2025-07-10 09:10:44,664 - root - INFO - lr: 3.8017e-05 gnorm: 1.00 [19:35:41< 2:24:02] +[titan] 2025-07-10 09:10:47,184 - root - INFO - Dumping profiler traces at step 89088 +[titan] 2025-07-10 09:10:47,217 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:10:48,794 - root - INFO - step: 89090 loss: 15.9271 memory: 44.58GiB(31.99%) tps: 79,339 tflops: 273.81 mfu: 27.69% global_avg_ntp_loss: 2.4193 global_avg_mtp_loss: 13.5077 +[titan] 2025-07-10 09:10:48,795 - root - INFO - lr: 3.8010e-05 gnorm: 1.01 [19:35:45< 2:23:58] +[titan] 2025-07-10 09:10:52,748 - root - INFO - step: 89095 loss: 16.2928 memory: 44.58GiB(31.99%) tps: 82,886 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4939 global_avg_mtp_loss: 13.7989 +[titan] 2025-07-10 09:10:52,748 - root - INFO - lr: 3.8003e-05 gnorm: 1.01 [19:35:49< 2:23:55] +[titan] 2025-07-10 09:10:55,874 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:10:56,702 - root - INFO - step: 89100 loss: 16.1357 memory: 44.58GiB(31.99%) tps: 82,887 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6658 +[titan] 2025-07-10 09:10:56,702 - root - INFO - lr: 3.7995e-05 gnorm: 1.04 [19:35:53< 2:23:51] +[titan] 2025-07-10 09:11:00,641 - root - INFO - step: 89105 loss: 16.1891 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4706 global_avg_mtp_loss: 13.7185 +[titan] 2025-07-10 09:11:00,642 - root - INFO - lr: 3.7988e-05 gnorm: 1.01 [19:35:57< 2:23:47] +[titan] 2025-07-10 09:11:04,577 - root - INFO - step: 89110 loss: 16.2835 memory: 44.58GiB(31.99%) tps: 83,279 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7941 +[titan] 2025-07-10 09:11:04,577 - root - INFO - lr: 3.7981e-05 gnorm: 0.98 [19:36:00< 2:23:43] +[titan] 2025-07-10 09:11:08,523 - root - INFO - step: 89115 loss: 16.1691 memory: 44.58GiB(31.99%) tps: 83,030 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 09:11:08,524 - root - INFO - lr: 3.7974e-05 gnorm: 0.97 [19:36:04< 2:23:39] +[titan] 2025-07-10 09:11:12,452 - root - INFO - step: 89120 loss: 16.2106 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.7274 +[titan] 2025-07-10 09:11:12,453 - root - INFO - lr: 3.7966e-05 gnorm: 1.09 [19:36:08< 2:23:35] +[titan] 2025-07-10 09:11:16,373 - root - INFO - step: 89125 loss: 16.1935 memory: 44.58GiB(31.99%) tps: 83,588 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.7299 +[titan] 2025-07-10 09:11:16,373 - root - INFO - lr: 3.7959e-05 gnorm: 1.02 [19:36:12< 2:23:31] +[titan] 2025-07-10 09:11:20,317 - root - INFO - step: 89130 loss: 16.1352 memory: 44.58GiB(31.99%) tps: 83,092 tflops: 286.76 mfu: 29.00% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6754 +[titan] 2025-07-10 09:11:20,317 - root - INFO - lr: 3.7952e-05 gnorm: 1.03 [19:36:16< 2:23:27] +[titan] 2025-07-10 09:11:24,238 - root - INFO - step: 89135 loss: 16.1115 memory: 44.58GiB(31.99%) tps: 83,570 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6423 +[titan] 2025-07-10 09:11:24,239 - root - INFO - lr: 3.7945e-05 gnorm: 1.08 [19:36:20< 2:23:23] +[titan] 2025-07-10 09:11:28,163 - root - INFO - step: 89140 loss: 16.0316 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4333 global_avg_mtp_loss: 13.5983 +[titan] 2025-07-10 09:11:28,164 - root - INFO - lr: 3.7937e-05 gnorm: 1.05 [19:36:24< 2:23:19] +[titan] 2025-07-10 09:11:32,070 - root - INFO - step: 89145 loss: 16.1942 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7248 +[titan] 2025-07-10 09:11:32,070 - root - INFO - lr: 3.7930e-05 gnorm: 0.99 [19:36:28< 2:23:15] +[titan] 2025-07-10 09:11:35,203 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:11:35,992 - root - INFO - step: 89150 loss: 16.2652 memory: 44.58GiB(31.99%) tps: 83,550 tflops: 288.34 mfu: 29.16% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7780 +[titan] 2025-07-10 09:11:35,992 - root - INFO - lr: 3.7923e-05 gnorm: 1.02 [19:36:32< 2:23:11] +[titan] 2025-07-10 09:11:39,909 - root - INFO - step: 89155 loss: 15.9188 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4214 global_avg_mtp_loss: 13.4973 +[titan] 2025-07-10 09:11:39,909 - root - INFO - lr: 3.7916e-05 gnorm: 1.03 [19:36:36< 2:23:07] +[titan] 2025-07-10 09:11:43,844 - root - INFO - step: 89160 loss: 16.0976 memory: 44.58GiB(31.99%) tps: 83,273 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6442 +[titan] 2025-07-10 09:11:43,844 - root - INFO - lr: 3.7909e-05 gnorm: 0.99 [19:36:40< 2:23:03] +[titan] 2025-07-10 09:11:47,764 - root - INFO - step: 89165 loss: 15.8901 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4158 global_avg_mtp_loss: 13.4743 +[titan] 2025-07-10 09:11:47,764 - root - INFO - lr: 3.7901e-05 gnorm: 1.03 [19:36:44< 2:22:59] +[titan] 2025-07-10 09:11:51,661 - root - INFO - step: 89170 loss: 16.0771 memory: 44.58GiB(31.99%) tps: 84,106 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4514 global_avg_mtp_loss: 13.6257 +[titan] 2025-07-10 09:11:51,661 - root - INFO - lr: 3.7894e-05 gnorm: 0.98 [19:36:48< 2:22:55] +[titan] 2025-07-10 09:11:55,563 - root - INFO - step: 89175 loss: 16.1121 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4543 global_avg_mtp_loss: 13.6577 +[titan] 2025-07-10 09:11:55,563 - root - INFO - lr: 3.7887e-05 gnorm: 1.05 [19:36:51< 2:22:51] +[titan] 2025-07-10 09:11:59,479 - root - INFO - step: 89180 loss: 16.2411 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7555 +[titan] 2025-07-10 09:11:59,479 - root - INFO - lr: 3.7880e-05 gnorm: 1.00 [19:36:55< 2:22:47] +[titan] 2025-07-10 09:12:03,392 - root - INFO - step: 89185 loss: 16.3247 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8217 +[titan] 2025-07-10 09:12:03,392 - root - INFO - lr: 3.7872e-05 gnorm: 1.00 [19:36:59< 2:22:43] +[titan] 2025-07-10 09:12:07,316 - root - INFO - step: 89190 loss: 16.4307 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.5126 global_avg_mtp_loss: 13.9181 +[titan] 2025-07-10 09:12:07,316 - root - INFO - lr: 3.7865e-05 gnorm: 0.97 [19:37:03< 2:22:39] +[titan] 2025-07-10 09:12:11,234 - root - INFO - step: 89195 loss: 15.9777 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4369 global_avg_mtp_loss: 13.5408 +[titan] 2025-07-10 09:12:11,234 - root - INFO - lr: 3.7858e-05 gnorm: 1.01 [19:37:07< 2:22:35] +[titan] 2025-07-10 09:12:14,348 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:12:15,135 - root - INFO - step: 89200 loss: 15.8841 memory: 44.58GiB(31.99%) tps: 84,005 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4146 global_avg_mtp_loss: 13.4696 +[titan] 2025-07-10 09:12:15,135 - root - INFO - lr: 3.7851e-05 gnorm: 1.02 [19:37:11< 2:22:31] +[titan] 2025-07-10 09:12:19,048 - root - INFO - step: 89205 loss: 16.0441 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.5932 +[titan] 2025-07-10 09:12:19,049 - root - INFO - lr: 3.7844e-05 gnorm: 1.06 [19:37:15< 2:22:27] +[titan] 2025-07-10 09:12:23,002 - root - INFO - step: 89210 loss: 16.2567 memory: 44.58GiB(31.99%) tps: 82,879 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7719 +[titan] 2025-07-10 09:12:23,003 - root - INFO - lr: 3.7836e-05 gnorm: 0.97 [19:37:19< 2:22:23] +[titan] 2025-07-10 09:12:26,944 - root - INFO - step: 89215 loss: 16.1391 memory: 44.58GiB(31.99%) tps: 83,134 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.6932 +[titan] 2025-07-10 09:12:26,945 - root - INFO - lr: 3.7829e-05 gnorm: 1.02 [19:37:23< 2:22:19] +[titan] 2025-07-10 09:12:30,858 - root - INFO - step: 89220 loss: 16.5699 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5418 global_avg_mtp_loss: 14.0280 +[titan] 2025-07-10 09:12:30,859 - root - INFO - lr: 3.7822e-05 gnorm: 1.04 [19:37:27< 2:22:15] +[titan] 2025-07-10 09:12:34,765 - root - INFO - step: 89225 loss: 16.2402 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.7591 +[titan] 2025-07-10 09:12:34,766 - root - INFO - lr: 3.7815e-05 gnorm: 1.02 [19:37:31< 2:22:11] +[titan] 2025-07-10 09:12:38,723 - root - INFO - step: 89230 loss: 16.3364 memory: 44.58GiB(31.99%) tps: 82,813 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.8472 +[titan] 2025-07-10 09:12:38,723 - root - INFO - lr: 3.7808e-05 gnorm: 0.97 [19:37:35< 2:22:08] +[titan] 2025-07-10 09:12:42,631 - root - INFO - step: 89235 loss: 16.0530 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4474 global_avg_mtp_loss: 13.6057 +[titan] 2025-07-10 09:12:42,632 - root - INFO - lr: 3.7801e-05 gnorm: 1.06 [19:37:38< 2:22:04] +[titan] 2025-07-10 09:12:46,551 - root - INFO - step: 89240 loss: 16.2949 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.8052 +[titan] 2025-07-10 09:12:46,551 - root - INFO - lr: 3.7793e-05 gnorm: 0.98 [19:37:42< 2:22:00] +[titan] 2025-07-10 09:12:50,469 - root - INFO - step: 89245 loss: 16.2804 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.8011 +[titan] 2025-07-10 09:12:50,470 - root - INFO - lr: 3.7786e-05 gnorm: 0.98 [19:37:46< 2:21:56] +[titan] 2025-07-10 09:12:53,603 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:12:54,399 - root - INFO - step: 89250 loss: 16.1249 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6598 +[titan] 2025-07-10 09:12:54,399 - root - INFO - lr: 3.7779e-05 gnorm: 1.04 [19:37:50< 2:21:52] +[titan] 2025-07-10 09:12:58,357 - root - INFO - step: 89255 loss: 16.1598 memory: 44.58GiB(31.99%) tps: 82,797 tflops: 285.75 mfu: 28.89% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.6904 +[titan] 2025-07-10 09:12:58,357 - root - INFO - lr: 3.7772e-05 gnorm: 1.05 [19:37:54< 2:21:48] +[titan] 2025-07-10 09:13:02,304 - root - INFO - step: 89260 loss: 16.2163 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7324 +[titan] 2025-07-10 09:13:02,304 - root - INFO - lr: 3.7765e-05 gnorm: 1.04 [19:37:58< 2:21:44] +[titan] 2025-07-10 09:13:06,241 - root - INFO - step: 89265 loss: 16.3560 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4937 global_avg_mtp_loss: 13.8623 +[titan] 2025-07-10 09:13:06,241 - root - INFO - lr: 3.7758e-05 gnorm: 1.00 [19:38:02< 2:21:40] +[titan] 2025-07-10 09:13:10,193 - root - INFO - step: 89270 loss: 16.2329 memory: 44.58GiB(31.99%) tps: 82,929 tflops: 286.20 mfu: 28.94% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.7419 +[titan] 2025-07-10 09:13:10,193 - root - INFO - lr: 3.7750e-05 gnorm: 0.98 [19:38:06< 2:21:36] +[titan] 2025-07-10 09:13:14,131 - root - INFO - step: 89275 loss: 15.9920 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4389 global_avg_mtp_loss: 13.5530 +[titan] 2025-07-10 09:13:14,132 - root - INFO - lr: 3.7743e-05 gnorm: 1.03 [19:38:10< 2:21:32] +[titan] 2025-07-10 09:13:18,041 - root - INFO - step: 89280 loss: 15.9641 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4328 global_avg_mtp_loss: 13.5314 +[titan] 2025-07-10 09:13:18,041 - root - INFO - lr: 3.7736e-05 gnorm: 1.02 [19:38:14< 2:21:28] +[titan] 2025-07-10 09:13:21,966 - root - INFO - step: 89285 loss: 15.9702 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.5269 +[titan] 2025-07-10 09:13:21,967 - root - INFO - lr: 3.7729e-05 gnorm: 1.06 [19:38:18< 2:21:24] +[titan] 2025-07-10 09:13:25,898 - root - INFO - step: 89290 loss: 16.0917 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.67 mfu: 29.09% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6384 +[titan] 2025-07-10 09:13:25,898 - root - INFO - lr: 3.7722e-05 gnorm: 1.02 [19:38:22< 2:21:20] +[titan] 2025-07-10 09:13:29,815 - root - INFO - step: 89295 loss: 16.2914 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4832 global_avg_mtp_loss: 13.8082 +[titan] 2025-07-10 09:13:29,815 - root - INFO - lr: 3.7715e-05 gnorm: 1.03 [19:38:26< 2:21:16] +[titan] 2025-07-10 09:13:32,943 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:13:33,737 - root - INFO - step: 89300 loss: 16.1725 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7027 +[titan] 2025-07-10 09:13:33,737 - root - INFO - lr: 3.7708e-05 gnorm: 0.99 [19:38:30< 2:21:12] +[titan] 2025-07-10 09:13:37,641 - root - INFO - step: 89305 loss: 16.1966 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7164 +[titan] 2025-07-10 09:13:37,642 - root - INFO - lr: 3.7700e-05 gnorm: 1.01 [19:38:34< 2:21:08] +[titan] 2025-07-10 09:13:41,568 - root - INFO - step: 89310 loss: 16.2298 memory: 44.58GiB(31.99%) tps: 83,462 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7530 +[titan] 2025-07-10 09:13:41,568 - root - INFO - lr: 3.7693e-05 gnorm: 1.02 [19:38:37< 2:21:04] +[titan] 2025-07-10 09:13:45,507 - root - INFO - step: 89315 loss: 16.2498 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7750 +[titan] 2025-07-10 09:13:45,507 - root - INFO - lr: 3.7686e-05 gnorm: 1.01 [19:38:41< 2:21:00] +[titan] 2025-07-10 09:13:49,434 - root - INFO - step: 89320 loss: 16.2079 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7371 +[titan] 2025-07-10 09:13:49,434 - root - INFO - lr: 3.7679e-05 gnorm: 1.00 [19:38:45< 2:20:56] +[titan] 2025-07-10 09:13:53,381 - root - INFO - step: 89325 loss: 16.4734 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9534 +[titan] 2025-07-10 09:13:53,381 - root - INFO - lr: 3.7672e-05 gnorm: 1.04 [19:38:49< 2:20:52] +[titan] 2025-07-10 09:13:57,288 - root - INFO - step: 89330 loss: 16.2153 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4764 global_avg_mtp_loss: 13.7388 +[titan] 2025-07-10 09:13:57,288 - root - INFO - lr: 3.7665e-05 gnorm: 0.99 [19:38:53< 2:20:48] +[titan] 2025-07-10 09:14:01,200 - root - INFO - step: 89335 loss: 16.2259 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7479 +[titan] 2025-07-10 09:14:01,201 - root - INFO - lr: 3.7658e-05 gnorm: 0.99 [19:38:57< 2:20:44] +[titan] 2025-07-10 09:14:05,120 - root - INFO - step: 89340 loss: 16.3776 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8746 +[titan] 2025-07-10 09:14:05,120 - root - INFO - lr: 3.7651e-05 gnorm: 0.97 [19:39:01< 2:20:40] +[titan] 2025-07-10 09:14:09,035 - root - INFO - step: 89345 loss: 15.9166 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4173 global_avg_mtp_loss: 13.4993 +[titan] 2025-07-10 09:14:09,036 - root - INFO - lr: 3.7643e-05 gnorm: 0.99 [19:39:05< 2:20:36] +[titan] 2025-07-10 09:14:12,188 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:14:12,979 - root - INFO - step: 89350 loss: 16.1840 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7107 +[titan] 2025-07-10 09:14:12,979 - root - INFO - lr: 3.7636e-05 gnorm: 1.08 [19:39:09< 2:20:32] +[titan] 2025-07-10 09:14:16,898 - root - INFO - step: 89355 loss: 16.0193 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.5855 +[titan] 2025-07-10 09:14:16,898 - root - INFO - lr: 3.7629e-05 gnorm: 0.99 [19:39:13< 2:20:28] +[titan] 2025-07-10 09:14:20,818 - root - INFO - step: 89360 loss: 16.1000 memory: 44.58GiB(31.99%) tps: 83,599 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4565 global_avg_mtp_loss: 13.6435 +[titan] 2025-07-10 09:14:20,818 - root - INFO - lr: 3.7622e-05 gnorm: 1.04 [19:39:17< 2:20:24] +[titan] 2025-07-10 09:14:24,744 - root - INFO - step: 89365 loss: 16.4898 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.5416 global_avg_mtp_loss: 13.9482 +[titan] 2025-07-10 09:14:24,745 - root - INFO - lr: 3.7615e-05 gnorm: 1.06 [19:39:21< 2:20:21] +[titan] 2025-07-10 09:14:28,727 - root - INFO - step: 89370 loss: 16.0229 memory: 44.58GiB(31.99%) tps: 82,298 tflops: 284.02 mfu: 28.72% global_avg_ntp_loss: 2.4411 global_avg_mtp_loss: 13.5818 +[titan] 2025-07-10 09:14:28,727 - root - INFO - lr: 3.7608e-05 gnorm: 0.99 [19:39:25< 2:20:17] +[titan] 2025-07-10 09:14:32,694 - root - INFO - step: 89375 loss: 16.1391 memory: 44.58GiB(31.99%) tps: 82,609 tflops: 285.10 mfu: 28.83% global_avg_ntp_loss: 2.4613 global_avg_mtp_loss: 13.6778 +[titan] 2025-07-10 09:14:32,694 - root - INFO - lr: 3.7601e-05 gnorm: 1.13 [19:39:29< 2:20:13] +[titan] 2025-07-10 09:14:36,689 - root - INFO - step: 89380 loss: 16.0462 memory: 44.58GiB(31.99%) tps: 82,021 tflops: 283.07 mfu: 28.62% global_avg_ntp_loss: 2.4369 global_avg_mtp_loss: 13.6093 +[titan] 2025-07-10 09:14:36,689 - root - INFO - lr: 3.7594e-05 gnorm: 1.03 [19:39:33< 2:20:09] +[titan] 2025-07-10 09:14:40,606 - root - INFO - step: 89385 loss: 16.2826 memory: 44.58GiB(31.99%) tps: 83,656 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.7864 +[titan] 2025-07-10 09:14:40,607 - root - INFO - lr: 3.7587e-05 gnorm: 0.98 [19:39:36< 2:20:05] +[titan] 2025-07-10 09:14:44,528 - root - INFO - step: 89390 loss: 16.0692 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4483 global_avg_mtp_loss: 13.6209 +[titan] 2025-07-10 09:14:44,528 - root - INFO - lr: 3.7580e-05 gnorm: 1.06 [19:39:40< 2:20:01] +[titan] 2025-07-10 09:14:48,442 - root - INFO - step: 89395 loss: 16.3345 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4889 global_avg_mtp_loss: 13.8456 +[titan] 2025-07-10 09:14:48,442 - root - INFO - lr: 3.7573e-05 gnorm: 1.03 [19:39:44< 2:19:57] +[titan] 2025-07-10 09:14:51,578 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:14:52,367 - root - INFO - step: 89400 loss: 16.2210 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.7298 +[titan] 2025-07-10 09:14:52,367 - root - INFO - lr: 3.7565e-05 gnorm: 0.99 [19:39:48< 2:19:53] +[titan] 2025-07-10 09:14:56,271 - root - INFO - step: 89405 loss: 16.1308 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6720 +[titan] 2025-07-10 09:14:56,272 - root - INFO - lr: 3.7558e-05 gnorm: 1.02 [19:39:52< 2:19:49] +[titan] 2025-07-10 09:15:00,200 - root - INFO - step: 89410 loss: 15.8152 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.3961 global_avg_mtp_loss: 13.4191 +[titan] 2025-07-10 09:15:00,200 - root - INFO - lr: 3.7551e-05 gnorm: 1.02 [19:39:56< 2:19:45] +[titan] 2025-07-10 09:15:04,157 - root - INFO - step: 89415 loss: 16.3013 memory: 44.58GiB(31.99%) tps: 82,818 tflops: 285.82 mfu: 28.90% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8071 +[titan] 2025-07-10 09:15:04,158 - root - INFO - lr: 3.7544e-05 gnorm: 1.04 [19:40:00< 2:19:41] +[titan] 2025-07-10 09:15:08,071 - root - INFO - step: 89420 loss: 16.2513 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7716 +[titan] 2025-07-10 09:15:08,072 - root - INFO - lr: 3.7537e-05 gnorm: 1.08 [19:40:04< 2:19:37] +[titan] 2025-07-10 09:15:11,971 - root - INFO - step: 89425 loss: 16.4177 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 289.99 mfu: 29.32% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.9203 +[titan] 2025-07-10 09:15:11,972 - root - INFO - lr: 3.7530e-05 gnorm: 0.99 [19:40:08< 2:19:33] +[titan] 2025-07-10 09:15:15,889 - root - INFO - step: 89430 loss: 16.2013 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7232 +[titan] 2025-07-10 09:15:15,889 - root - INFO - lr: 3.7523e-05 gnorm: 1.00 [19:40:12< 2:19:29] +[titan] 2025-07-10 09:15:19,851 - root - INFO - step: 89435 loss: 16.0591 memory: 44.58GiB(31.99%) tps: 82,711 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.4498 global_avg_mtp_loss: 13.6093 +[titan] 2025-07-10 09:15:19,852 - root - INFO - lr: 3.7516e-05 gnorm: 1.02 [19:40:16< 2:19:25] +[titan] 2025-07-10 09:15:23,773 - root - INFO - step: 89440 loss: 15.9931 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4291 global_avg_mtp_loss: 13.5640 +[titan] 2025-07-10 09:15:23,773 - root - INFO - lr: 3.7509e-05 gnorm: 0.98 [19:40:20< 2:19:21] +[titan] 2025-07-10 09:15:27,681 - root - INFO - step: 89445 loss: 16.1411 memory: 44.58GiB(31.99%) tps: 83,843 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.6690 +[titan] 2025-07-10 09:15:27,682 - root - INFO - lr: 3.7502e-05 gnorm: 1.01 [19:40:24< 2:19:17] +[titan] 2025-07-10 09:15:30,813 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:15:31,604 - root - INFO - step: 89450 loss: 16.2330 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4719 global_avg_mtp_loss: 13.7610 +[titan] 2025-07-10 09:15:31,604 - root - INFO - lr: 3.7495e-05 gnorm: 1.00 [19:40:27< 2:19:13] +[titan] 2025-07-10 09:15:35,526 - root - INFO - step: 89455 loss: 16.3487 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8476 +[titan] 2025-07-10 09:15:35,526 - root - INFO - lr: 3.7488e-05 gnorm: 1.00 [19:40:31< 2:19:09] +[titan] 2025-07-10 09:15:39,464 - root - INFO - step: 89460 loss: 16.4397 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.5219 global_avg_mtp_loss: 13.9177 +[titan] 2025-07-10 09:15:39,465 - root - INFO - lr: 3.7481e-05 gnorm: 0.99 [19:40:35< 2:19:05] +[titan] 2025-07-10 09:15:43,368 - root - INFO - step: 89465 loss: 16.3544 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8559 +[titan] 2025-07-10 09:15:43,368 - root - INFO - lr: 3.7474e-05 gnorm: 1.00 [19:40:39< 2:19:01] +[titan] 2025-07-10 09:15:47,283 - root - INFO - step: 89470 loss: 16.2150 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.7337 +[titan] 2025-07-10 09:15:47,283 - root - INFO - lr: 3.7467e-05 gnorm: 1.00 [19:40:43< 2:18:57] +[titan] 2025-07-10 09:15:51,193 - root - INFO - step: 89475 loss: 16.2976 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.8224 +[titan] 2025-07-10 09:15:51,193 - root - INFO - lr: 3.7460e-05 gnorm: 1.00 [19:40:47< 2:18:53] +[titan] 2025-07-10 09:15:55,110 - root - INFO - step: 89480 loss: 15.9563 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4373 global_avg_mtp_loss: 13.5190 +[titan] 2025-07-10 09:15:55,111 - root - INFO - lr: 3.7453e-05 gnorm: 1.15 [19:40:51< 2:18:49] +[titan] 2025-07-10 09:15:59,045 - root - INFO - step: 89485 loss: 15.9448 memory: 44.58GiB(31.99%) tps: 83,295 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4282 global_avg_mtp_loss: 13.5166 +[titan] 2025-07-10 09:15:59,045 - root - INFO - lr: 3.7446e-05 gnorm: 1.00 [19:40:55< 2:18:45] +[titan] 2025-07-10 09:16:02,960 - root - INFO - step: 89490 loss: 16.0142 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5707 +[titan] 2025-07-10 09:16:02,960 - root - INFO - lr: 3.7439e-05 gnorm: 1.02 [19:40:59< 2:18:41] +[titan] 2025-07-10 09:16:06,865 - root - INFO - step: 89495 loss: 16.4532 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5111 global_avg_mtp_loss: 13.9421 +[titan] 2025-07-10 09:16:06,865 - root - INFO - lr: 3.7432e-05 gnorm: 1.00 [19:41:03< 2:18:37] +[titan] 2025-07-10 09:16:10,001 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:16:10,792 - root - INFO - step: 89500 loss: 16.2012 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.7346 +[titan] 2025-07-10 09:16:10,793 - root - INFO - lr: 3.7425e-05 gnorm: 1.04 [19:41:07< 2:18:34] +[titan] 2025-07-10 09:16:14,727 - root - INFO - step: 89505 loss: 16.0897 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.6288 +[titan] 2025-07-10 09:16:14,727 - root - INFO - lr: 3.7418e-05 gnorm: 1.05 [19:41:11< 2:18:30] +[titan] 2025-07-10 09:16:18,656 - root - INFO - step: 89510 loss: 16.1625 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 09:16:18,657 - root - INFO - lr: 3.7411e-05 gnorm: 0.99 [19:41:14< 2:18:26] +[titan] 2025-07-10 09:16:22,560 - root - INFO - step: 89515 loss: 16.4253 memory: 44.58GiB(31.99%) tps: 83,956 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.8983 +[titan] 2025-07-10 09:16:22,560 - root - INFO - lr: 3.7404e-05 gnorm: 1.08 [19:41:18< 2:18:22] +[titan] 2025-07-10 09:16:26,477 - root - INFO - step: 89520 loss: 16.1857 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.7239 +[titan] 2025-07-10 09:16:26,477 - root - INFO - lr: 3.7397e-05 gnorm: 1.01 [19:41:22< 2:18:18] +[titan] 2025-07-10 09:16:30,386 - root - INFO - step: 89525 loss: 16.3864 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 09:16:30,386 - root - INFO - lr: 3.7390e-05 gnorm: 0.98 [19:41:26< 2:18:14] +[titan] 2025-07-10 09:16:34,307 - root - INFO - step: 89530 loss: 16.2550 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7714 +[titan] 2025-07-10 09:16:34,307 - root - INFO - lr: 3.7383e-05 gnorm: 0.99 [19:41:30< 2:18:10] +[titan] 2025-07-10 09:16:38,237 - root - INFO - step: 89535 loss: 16.4291 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.5034 global_avg_mtp_loss: 13.9257 +[titan] 2025-07-10 09:16:38,237 - root - INFO - lr: 3.7376e-05 gnorm: 0.99 [19:41:34< 2:18:06] +[titan] 2025-07-10 09:16:42,148 - root - INFO - step: 89540 loss: 15.9431 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4364 global_avg_mtp_loss: 13.5067 +[titan] 2025-07-10 09:16:42,148 - root - INFO - lr: 3.7369e-05 gnorm: 1.01 [19:41:38< 2:18:02] +[titan] 2025-07-10 09:16:46,051 - root - INFO - step: 89545 loss: 16.2979 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.8096 +[titan] 2025-07-10 09:16:46,051 - root - INFO - lr: 3.7362e-05 gnorm: 0.99 [19:41:42< 2:17:58] +[titan] 2025-07-10 09:16:49,173 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:16:49,960 - root - INFO - step: 89550 loss: 15.5858 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.3711 global_avg_mtp_loss: 13.2147 +[titan] 2025-07-10 09:16:49,961 - root - INFO - lr: 3.7355e-05 gnorm: 1.09 [19:41:46< 2:17:54] +[titan] 2025-07-10 09:16:53,882 - root - INFO - step: 89555 loss: 16.0468 memory: 44.58GiB(31.99%) tps: 83,564 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4458 global_avg_mtp_loss: 13.6010 +[titan] 2025-07-10 09:16:53,882 - root - INFO - lr: 3.7348e-05 gnorm: 1.02 [19:41:50< 2:17:50] +[titan] 2025-07-10 09:16:57,797 - root - INFO - step: 89560 loss: 16.1255 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4639 global_avg_mtp_loss: 13.6617 +[titan] 2025-07-10 09:16:57,797 - root - INFO - lr: 3.7341e-05 gnorm: 1.04 [19:41:54< 2:17:46] +[titan] 2025-07-10 09:17:01,703 - root - INFO - step: 89565 loss: 15.8810 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4261 global_avg_mtp_loss: 13.4549 +[titan] 2025-07-10 09:17:01,704 - root - INFO - lr: 3.7334e-05 gnorm: 1.06 [19:41:58< 2:17:42] +[titan] 2025-07-10 09:17:05,626 - root - INFO - step: 89570 loss: 16.4568 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.9432 +[titan] 2025-07-10 09:17:05,626 - root - INFO - lr: 3.7327e-05 gnorm: 1.00 [19:42:01< 2:17:38] +[titan] 2025-07-10 09:17:09,558 - root - INFO - step: 89575 loss: 16.1017 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4624 global_avg_mtp_loss: 13.6393 +[titan] 2025-07-10 09:17:09,558 - root - INFO - lr: 3.7320e-05 gnorm: 1.04 [19:42:05< 2:17:34] +[titan] 2025-07-10 09:17:13,477 - root - INFO - step: 89580 loss: 16.1557 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 09:17:13,477 - root - INFO - lr: 3.7313e-05 gnorm: 0.98 [19:42:09< 2:17:30] +[titan] 2025-07-10 09:17:17,373 - root - INFO - step: 89585 loss: 16.2545 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7769 +[titan] 2025-07-10 09:17:17,374 - root - INFO - lr: 3.7306e-05 gnorm: 1.00 [19:42:13< 2:17:26] +[titan] 2025-07-10 09:17:21,302 - root - INFO - step: 89590 loss: 16.4406 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.9297 +[titan] 2025-07-10 09:17:21,302 - root - INFO - lr: 3.7299e-05 gnorm: 1.01 [19:42:17< 2:17:22] +[titan] 2025-07-10 09:17:25,233 - root - INFO - step: 89595 loss: 15.9953 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4282 global_avg_mtp_loss: 13.5672 +[titan] 2025-07-10 09:17:25,234 - root - INFO - lr: 3.7292e-05 gnorm: 1.06 [19:42:21< 2:17:18] +[titan] 2025-07-10 09:17:28,450 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:17:29,252 - root - INFO - step: 89600 loss: 16.4324 memory: 44.58GiB(31.99%) tps: 81,549 tflops: 281.44 mfu: 28.46% global_avg_ntp_loss: 2.5141 global_avg_mtp_loss: 13.9183 +[titan] 2025-07-10 09:17:29,252 - root - INFO - lr: 3.7285e-05 gnorm: 1.01 [19:42:25< 2:17:14] +[titan] 2025-07-10 09:17:29,406 - root - INFO - Dumping profiler traces at step 89600 +[titan] 2025-07-10 09:17:29,440 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:17:33,358 - root - INFO - step: 89605 loss: 16.3118 memory: 44.58GiB(31.99%) tps: 79,816 tflops: 275.46 mfu: 27.85% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.8288 +[titan] 2025-07-10 09:17:33,358 - root - INFO - lr: 3.7278e-05 gnorm: 1.00 [19:42:29< 2:17:10] +[titan] 2025-07-10 09:17:37,275 - root - INFO - step: 89610 loss: 16.1118 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6600 +[titan] 2025-07-10 09:17:37,275 - root - INFO - lr: 3.7271e-05 gnorm: 1.04 [19:42:33< 2:17:06] +[titan] 2025-07-10 09:17:41,222 - root - INFO - step: 89615 loss: 16.1464 memory: 44.58GiB(31.99%) tps: 83,026 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6772 +[titan] 2025-07-10 09:17:41,222 - root - INFO - lr: 3.7264e-05 gnorm: 1.12 [19:42:37< 2:17:02] +[titan] 2025-07-10 09:17:45,169 - root - INFO - step: 89620 loss: 15.9646 memory: 44.58GiB(31.99%) tps: 83,029 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.4243 global_avg_mtp_loss: 13.5403 +[titan] 2025-07-10 09:17:45,169 - root - INFO - lr: 3.7258e-05 gnorm: 1.01 [19:42:41< 2:16:58] +[titan] 2025-07-10 09:17:49,084 - root - INFO - step: 89625 loss: 15.9897 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4317 global_avg_mtp_loss: 13.5580 +[titan] 2025-07-10 09:17:49,085 - root - INFO - lr: 3.7251e-05 gnorm: 1.03 [19:42:45< 2:16:54] +[titan] 2025-07-10 09:17:53,017 - root - INFO - step: 89630 loss: 16.1114 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4469 global_avg_mtp_loss: 13.6645 +[titan] 2025-07-10 09:17:53,018 - root - INFO - lr: 3.7244e-05 gnorm: 1.02 [19:42:49< 2:16:51] +[titan] 2025-07-10 09:17:56,970 - root - INFO - step: 89635 loss: 16.0587 memory: 44.58GiB(31.99%) tps: 82,906 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.4506 global_avg_mtp_loss: 13.6081 +[titan] 2025-07-10 09:17:56,970 - root - INFO - lr: 3.7237e-05 gnorm: 1.03 [19:42:53< 2:16:47] +[titan] 2025-07-10 09:18:00,887 - root - INFO - step: 89640 loss: 15.9981 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4416 global_avg_mtp_loss: 13.5565 +[titan] 2025-07-10 09:18:00,887 - root - INFO - lr: 3.7230e-05 gnorm: 1.06 [19:42:57< 2:16:43] +[titan] 2025-07-10 09:18:04,826 - root - INFO - step: 89645 loss: 16.5574 memory: 44.58GiB(31.99%) tps: 83,189 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 14.0200 +[titan] 2025-07-10 09:18:04,827 - root - INFO - lr: 3.7223e-05 gnorm: 0.99 [19:43:01< 2:16:39] +[titan] 2025-07-10 09:18:07,959 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:18:08,747 - root - INFO - step: 89650 loss: 16.0810 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6331 +[titan] 2025-07-10 09:18:08,747 - root - INFO - lr: 3.7216e-05 gnorm: 0.99 [19:43:05< 2:16:35] +[titan] 2025-07-10 09:18:12,710 - root - INFO - step: 89655 loss: 16.1685 memory: 44.58GiB(31.99%) tps: 82,688 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 09:18:12,711 - root - INFO - lr: 3.7209e-05 gnorm: 1.00 [19:43:09< 2:16:31] +[titan] 2025-07-10 09:18:16,640 - root - INFO - step: 89660 loss: 16.1075 memory: 44.58GiB(31.99%) tps: 83,401 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6428 +[titan] 2025-07-10 09:18:16,640 - root - INFO - lr: 3.7202e-05 gnorm: 1.04 [19:43:12< 2:16:27] +[titan] 2025-07-10 09:18:20,554 - root - INFO - step: 89665 loss: 16.3765 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-10 09:18:20,554 - root - INFO - lr: 3.7195e-05 gnorm: 1.08 [19:43:16< 2:16:23] +[titan] 2025-07-10 09:18:24,481 - root - INFO - step: 89670 loss: 16.2211 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.7307 +[titan] 2025-07-10 09:18:24,481 - root - INFO - lr: 3.7188e-05 gnorm: 0.99 [19:43:20< 2:16:19] +[titan] 2025-07-10 09:18:28,413 - root - INFO - step: 89675 loss: 16.6308 memory: 44.58GiB(31.99%) tps: 83,350 tflops: 287.65 mfu: 29.09% global_avg_ntp_loss: 2.5393 global_avg_mtp_loss: 14.0915 +[titan] 2025-07-10 09:18:28,413 - root - INFO - lr: 3.7182e-05 gnorm: 1.01 [19:43:24< 2:16:15] +[titan] 2025-07-10 09:18:32,308 - root - INFO - step: 89680 loss: 16.3429 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8509 +[titan] 2025-07-10 09:18:32,308 - root - INFO - lr: 3.7175e-05 gnorm: 1.01 [19:43:28< 2:16:11] +[titan] 2025-07-10 09:18:36,231 - root - INFO - step: 89685 loss: 16.2547 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4685 global_avg_mtp_loss: 13.7862 +[titan] 2025-07-10 09:18:36,231 - root - INFO - lr: 3.7168e-05 gnorm: 1.04 [19:43:32< 2:16:07] +[titan] 2025-07-10 09:18:40,150 - root - INFO - step: 89690 loss: 16.1813 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.7190 +[titan] 2025-07-10 09:18:40,150 - root - INFO - lr: 3.7161e-05 gnorm: 1.04 [19:43:36< 2:16:03] +[titan] 2025-07-10 09:18:44,063 - root - INFO - step: 89695 loss: 15.9806 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.5493 +[titan] 2025-07-10 09:18:44,064 - root - INFO - lr: 3.7154e-05 gnorm: 1.10 [19:43:40< 2:15:59] +[titan] 2025-07-10 09:18:47,182 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:18:47,982 - root - INFO - step: 89700 loss: 16.3962 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.5136 global_avg_mtp_loss: 13.8826 +[titan] 2025-07-10 09:18:47,982 - root - INFO - lr: 3.7147e-05 gnorm: 1.15 [19:43:44< 2:15:55] +[titan] 2025-07-10 09:18:51,890 - root - INFO - step: 89705 loss: 16.2252 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7390 +[titan] 2025-07-10 09:18:51,890 - root - INFO - lr: 3.7140e-05 gnorm: 1.01 [19:43:48< 2:15:51] +[titan] 2025-07-10 09:18:55,799 - root - INFO - step: 89710 loss: 16.3180 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.8263 +[titan] 2025-07-10 09:18:55,799 - root - INFO - lr: 3.7133e-05 gnorm: 1.03 [19:43:52< 2:15:47] +[titan] 2025-07-10 09:18:59,714 - root - INFO - step: 89715 loss: 16.2931 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.8086 +[titan] 2025-07-10 09:18:59,714 - root - INFO - lr: 3.7126e-05 gnorm: 1.01 [19:43:56< 2:15:43] +[titan] 2025-07-10 09:19:03,617 - root - INFO - step: 89720 loss: 16.0208 memory: 44.58GiB(31.99%) tps: 83,955 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4529 global_avg_mtp_loss: 13.5679 +[titan] 2025-07-10 09:19:03,618 - root - INFO - lr: 3.7120e-05 gnorm: 1.01 [19:43:59< 2:15:39] +[titan] 2025-07-10 09:19:07,537 - root - INFO - step: 89725 loss: 16.1741 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7004 +[titan] 2025-07-10 09:19:07,537 - root - INFO - lr: 3.7113e-05 gnorm: 1.04 [19:44:03< 2:15:35] +[titan] 2025-07-10 09:19:11,451 - root - INFO - step: 89730 loss: 15.6396 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.3586 global_avg_mtp_loss: 13.2809 +[titan] 2025-07-10 09:19:11,452 - root - INFO - lr: 3.7106e-05 gnorm: 1.09 [19:44:07< 2:15:31] +[titan] 2025-07-10 09:19:15,371 - root - INFO - step: 89735 loss: 16.3775 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8763 +[titan] 2025-07-10 09:19:15,372 - root - INFO - lr: 3.7099e-05 gnorm: 1.01 [19:44:11< 2:15:27] +[titan] 2025-07-10 09:19:19,288 - root - INFO - step: 89740 loss: 15.9869 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4494 global_avg_mtp_loss: 13.5375 +[titan] 2025-07-10 09:19:19,288 - root - INFO - lr: 3.7092e-05 gnorm: 1.05 [19:44:15< 2:15:23] +[titan] 2025-07-10 09:19:23,193 - root - INFO - step: 89745 loss: 16.4599 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5245 global_avg_mtp_loss: 13.9355 +[titan] 2025-07-10 09:19:23,194 - root - INFO - lr: 3.7085e-05 gnorm: 0.99 [19:44:19< 2:15:19] +[titan] 2025-07-10 09:19:26,308 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:19:27,095 - root - INFO - step: 89750 loss: 16.1408 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4517 global_avg_mtp_loss: 13.6891 +[titan] 2025-07-10 09:19:27,095 - root - INFO - lr: 3.7078e-05 gnorm: 1.04 [19:44:23< 2:15:15] +[titan] 2025-07-10 09:19:31,010 - root - INFO - step: 89755 loss: 16.4260 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.5298 global_avg_mtp_loss: 13.8961 +[titan] 2025-07-10 09:19:31,011 - root - INFO - lr: 3.7072e-05 gnorm: 1.01 [19:44:27< 2:15:11] +[titan] 2025-07-10 09:19:34,933 - root - INFO - step: 89760 loss: 15.8718 memory: 44.58GiB(31.99%) tps: 83,537 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4033 global_avg_mtp_loss: 13.4685 +[titan] 2025-07-10 09:19:34,934 - root - INFO - lr: 3.7065e-05 gnorm: 1.00 [19:44:31< 2:15:07] +[titan] 2025-07-10 09:19:38,850 - root - INFO - step: 89765 loss: 16.1292 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6551 +[titan] 2025-07-10 09:19:38,851 - root - INFO - lr: 3.7058e-05 gnorm: 1.01 [19:44:35< 2:15:03] +[titan] 2025-07-10 09:19:42,765 - root - INFO - step: 89770 loss: 16.0348 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4423 global_avg_mtp_loss: 13.5925 +[titan] 2025-07-10 09:19:42,765 - root - INFO - lr: 3.7051e-05 gnorm: 1.04 [19:44:39< 2:15:00] +[titan] 2025-07-10 09:19:46,696 - root - INFO - step: 89775 loss: 15.8765 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4289 global_avg_mtp_loss: 13.4476 +[titan] 2025-07-10 09:19:46,696 - root - INFO - lr: 3.7044e-05 gnorm: 1.01 [19:44:43< 2:14:56] +[titan] 2025-07-10 09:19:50,607 - root - INFO - step: 89780 loss: 16.3682 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8634 +[titan] 2025-07-10 09:19:50,607 - root - INFO - lr: 3.7037e-05 gnorm: 0.97 [19:44:46< 2:14:52] +[titan] 2025-07-10 09:19:54,517 - root - INFO - step: 89785 loss: 16.3605 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8647 +[titan] 2025-07-10 09:19:54,517 - root - INFO - lr: 3.7031e-05 gnorm: 0.95 [19:44:50< 2:14:48] +[titan] 2025-07-10 09:19:58,456 - root - INFO - step: 89790 loss: 16.1681 memory: 44.58GiB(31.99%) tps: 83,192 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.6927 +[titan] 2025-07-10 09:19:58,457 - root - INFO - lr: 3.7024e-05 gnorm: 1.03 [19:44:54< 2:14:44] +[titan] 2025-07-10 09:20:02,361 - root - INFO - step: 89795 loss: 16.4020 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.9054 +[titan] 2025-07-10 09:20:02,361 - root - INFO - lr: 3.7017e-05 gnorm: 0.98 [19:44:58< 2:14:40] +[titan] 2025-07-10 09:20:05,479 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:20:06,266 - root - INFO - step: 89800 loss: 16.4342 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5180 global_avg_mtp_loss: 13.9162 +[titan] 2025-07-10 09:20:06,267 - root - INFO - lr: 3.7010e-05 gnorm: 0.99 [19:45:02< 2:14:36] +[titan] 2025-07-10 09:20:10,197 - root - INFO - step: 89805 loss: 16.1608 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6967 +[titan] 2025-07-10 09:20:10,197 - root - INFO - lr: 3.7003e-05 gnorm: 1.02 [19:45:06< 2:14:32] +[titan] 2025-07-10 09:20:14,103 - root - INFO - step: 89810 loss: 16.4337 memory: 44.58GiB(31.99%) tps: 83,893 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.5121 global_avg_mtp_loss: 13.9216 +[titan] 2025-07-10 09:20:14,103 - root - INFO - lr: 3.6997e-05 gnorm: 1.00 [19:45:10< 2:14:28] +[titan] 2025-07-10 09:20:18,026 - root - INFO - step: 89815 loss: 16.1469 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.6833 +[titan] 2025-07-10 09:20:18,027 - root - INFO - lr: 3.6990e-05 gnorm: 1.00 [19:45:14< 2:14:24] +[titan] 2025-07-10 09:20:21,970 - root - INFO - step: 89820 loss: 15.9655 memory: 44.58GiB(31.99%) tps: 83,100 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4303 global_avg_mtp_loss: 13.5352 +[titan] 2025-07-10 09:20:21,970 - root - INFO - lr: 3.6983e-05 gnorm: 1.04 [19:45:18< 2:14:20] +[titan] 2025-07-10 09:20:25,903 - root - INFO - step: 89825 loss: 16.2813 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.7890 +[titan] 2025-07-10 09:20:25,903 - root - INFO - lr: 3.6976e-05 gnorm: 1.03 [19:45:22< 2:14:16] +[titan] 2025-07-10 09:20:29,847 - root - INFO - step: 89830 loss: 16.0145 memory: 44.58GiB(31.99%) tps: 83,089 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4336 global_avg_mtp_loss: 13.5809 +[titan] 2025-07-10 09:20:29,847 - root - INFO - lr: 3.6969e-05 gnorm: 0.98 [19:45:26< 2:14:12] +[titan] 2025-07-10 09:20:33,745 - root - INFO - step: 89835 loss: 16.3200 memory: 44.58GiB(31.99%) tps: 84,072 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.8380 +[titan] 2025-07-10 09:20:33,745 - root - INFO - lr: 3.6963e-05 gnorm: 1.00 [19:45:30< 2:14:08] +[titan] 2025-07-10 09:20:37,649 - root - INFO - step: 89840 loss: 16.2532 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7796 +[titan] 2025-07-10 09:20:37,649 - root - INFO - lr: 3.6956e-05 gnorm: 1.04 [19:45:33< 2:14:04] +[titan] 2025-07-10 09:20:41,554 - root - INFO - step: 89845 loss: 16.2154 memory: 44.58GiB(31.99%) tps: 83,921 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.7438 +[titan] 2025-07-10 09:20:41,554 - root - INFO - lr: 3.6949e-05 gnorm: 1.03 [19:45:37< 2:14:00] +[titan] 2025-07-10 09:20:44,723 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:20:45,512 - root - INFO - step: 89850 loss: 16.1913 memory: 44.58GiB(31.99%) tps: 82,789 tflops: 285.72 mfu: 28.89% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.7272 +[titan] 2025-07-10 09:20:45,512 - root - INFO - lr: 3.6942e-05 gnorm: 1.06 [19:45:41< 2:13:56] +[titan] 2025-07-10 09:20:49,450 - root - INFO - step: 89855 loss: 16.1265 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6620 +[titan] 2025-07-10 09:20:49,450 - root - INFO - lr: 3.6935e-05 gnorm: 1.05 [19:45:45< 2:13:52] +[titan] 2025-07-10 09:20:53,392 - root - INFO - step: 89860 loss: 16.1651 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.6917 +[titan] 2025-07-10 09:20:53,392 - root - INFO - lr: 3.6929e-05 gnorm: 1.00 [19:45:49< 2:13:48] +[titan] 2025-07-10 09:20:57,329 - root - INFO - step: 89865 loss: 16.4070 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.5074 global_avg_mtp_loss: 13.8996 +[titan] 2025-07-10 09:20:57,329 - root - INFO - lr: 3.6922e-05 gnorm: 1.02 [19:45:53< 2:13:44] +[titan] 2025-07-10 09:21:01,248 - root - INFO - step: 89870 loss: 16.4110 memory: 44.58GiB(31.99%) tps: 83,621 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.9060 +[titan] 2025-07-10 09:21:01,248 - root - INFO - lr: 3.6915e-05 gnorm: 1.02 [19:45:57< 2:13:40] +[titan] 2025-07-10 09:21:05,183 - root - INFO - step: 89875 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.7205 +[titan] 2025-07-10 09:21:05,184 - root - INFO - lr: 3.6908e-05 gnorm: 1.00 [19:46:01< 2:13:36] +[titan] 2025-07-10 09:21:09,128 - root - INFO - step: 89880 loss: 16.0137 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4471 global_avg_mtp_loss: 13.5666 +[titan] 2025-07-10 09:21:09,129 - root - INFO - lr: 3.6902e-05 gnorm: 1.01 [19:46:05< 2:13:32] +[titan] 2025-07-10 09:21:13,061 - root - INFO - step: 89885 loss: 16.2087 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7303 +[titan] 2025-07-10 09:21:13,062 - root - INFO - lr: 3.6895e-05 gnorm: 1.07 [19:46:09< 2:13:28] +[titan] 2025-07-10 09:21:17,001 - root - INFO - step: 89890 loss: 16.0238 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4500 global_avg_mtp_loss: 13.5738 +[titan] 2025-07-10 09:21:17,002 - root - INFO - lr: 3.6888e-05 gnorm: 0.99 [19:46:13< 2:13:24] +[titan] 2025-07-10 09:21:20,908 - root - INFO - step: 89895 loss: 16.0561 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.6047 +[titan] 2025-07-10 09:21:20,909 - root - INFO - lr: 3.6881e-05 gnorm: 1.08 [19:46:17< 2:13:20] +[titan] 2025-07-10 09:21:24,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:21:24,834 - root - INFO - step: 89900 loss: 16.1021 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6468 +[titan] 2025-07-10 09:21:24,835 - root - INFO - lr: 3.6875e-05 gnorm: 1.07 [19:46:21< 2:13:16] +[titan] 2025-07-10 09:21:28,751 - root - INFO - step: 89905 loss: 16.1944 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4926 global_avg_mtp_loss: 13.7018 +[titan] 2025-07-10 09:21:28,752 - root - INFO - lr: 3.6868e-05 gnorm: 1.02 [19:46:25< 2:13:13] +[titan] 2025-07-10 09:21:32,680 - root - INFO - step: 89910 loss: 16.2857 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.8033 +[titan] 2025-07-10 09:21:32,681 - root - INFO - lr: 3.6861e-05 gnorm: 1.03 [19:46:28< 2:13:09] +[titan] 2025-07-10 09:21:36,629 - root - INFO - step: 89915 loss: 16.2613 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4830 global_avg_mtp_loss: 13.7783 +[titan] 2025-07-10 09:21:36,630 - root - INFO - lr: 3.6854e-05 gnorm: 1.02 [19:46:32< 2:13:05] +[titan] 2025-07-10 09:21:40,551 - root - INFO - step: 89920 loss: 15.8158 memory: 44.58GiB(31.99%) tps: 83,558 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4065 global_avg_mtp_loss: 13.4094 +[titan] 2025-07-10 09:21:40,552 - root - INFO - lr: 3.6848e-05 gnorm: 1.02 [19:46:36< 2:13:01] +[titan] 2025-07-10 09:21:44,485 - root - INFO - step: 89925 loss: 16.2800 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7963 +[titan] 2025-07-10 09:21:44,486 - root - INFO - lr: 3.6841e-05 gnorm: 1.03 [19:46:40< 2:12:57] +[titan] 2025-07-10 09:21:48,418 - root - INFO - step: 89930 loss: 15.9697 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4258 global_avg_mtp_loss: 13.5440 +[titan] 2025-07-10 09:21:48,419 - root - INFO - lr: 3.6834e-05 gnorm: 1.01 [19:46:44< 2:12:53] +[titan] 2025-07-10 09:21:52,339 - root - INFO - step: 89935 loss: 16.4091 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.8936 +[titan] 2025-07-10 09:21:52,339 - root - INFO - lr: 3.6827e-05 gnorm: 1.03 [19:46:48< 2:12:49] +[titan] 2025-07-10 09:21:56,257 - root - INFO - step: 89940 loss: 16.2661 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.7714 +[titan] 2025-07-10 09:21:56,257 - root - INFO - lr: 3.6821e-05 gnorm: 1.20 [19:46:52< 2:12:45] +[titan] 2025-07-10 09:22:00,208 - root - INFO - step: 89945 loss: 16.0571 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4356 global_avg_mtp_loss: 13.6215 +[titan] 2025-07-10 09:22:00,209 - root - INFO - lr: 3.6814e-05 gnorm: 1.02 [19:46:56< 2:12:41] +[titan] 2025-07-10 09:22:03,345 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:22:04,134 - root - INFO - step: 89950 loss: 15.9038 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4265 global_avg_mtp_loss: 13.4774 +[titan] 2025-07-10 09:22:04,135 - root - INFO - lr: 3.6807e-05 gnorm: 1.05 [19:47:00< 2:12:37] +[titan] 2025-07-10 09:22:08,055 - root - INFO - step: 89955 loss: 15.9817 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4270 global_avg_mtp_loss: 13.5547 +[titan] 2025-07-10 09:22:08,055 - root - INFO - lr: 3.6801e-05 gnorm: 1.05 [19:47:04< 2:12:33] +[titan] 2025-07-10 09:22:11,965 - root - INFO - step: 89960 loss: 16.3397 memory: 44.58GiB(31.99%) tps: 83,816 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-10 09:22:11,965 - root - INFO - lr: 3.6794e-05 gnorm: 1.04 [19:47:08< 2:12:29] +[titan] 2025-07-10 09:22:15,867 - root - INFO - step: 89965 loss: 16.1738 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7027 +[titan] 2025-07-10 09:22:15,868 - root - INFO - lr: 3.6787e-05 gnorm: 1.00 [19:47:12< 2:12:25] +[titan] 2025-07-10 09:22:19,767 - root - INFO - step: 89970 loss: 16.4087 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 09:22:19,767 - root - INFO - lr: 3.6780e-05 gnorm: 1.07 [19:47:16< 2:12:21] +[titan] 2025-07-10 09:22:23,672 - root - INFO - step: 89975 loss: 16.2367 memory: 44.58GiB(31.99%) tps: 83,906 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.7421 +[titan] 2025-07-10 09:22:23,673 - root - INFO - lr: 3.6774e-05 gnorm: 1.03 [19:47:19< 2:12:17] +[titan] 2025-07-10 09:22:27,610 - root - INFO - step: 89980 loss: 16.1217 memory: 44.58GiB(31.99%) tps: 83,218 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.4542 global_avg_mtp_loss: 13.6675 +[titan] 2025-07-10 09:22:27,611 - root - INFO - lr: 3.6767e-05 gnorm: 0.95 [19:47:23< 2:12:13] +[titan] 2025-07-10 09:22:31,518 - root - INFO - step: 89985 loss: 16.1617 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6940 +[titan] 2025-07-10 09:22:31,518 - root - INFO - lr: 3.6760e-05 gnorm: 1.05 [19:47:27< 2:12:09] +[titan] 2025-07-10 09:22:35,433 - root - INFO - step: 89990 loss: 16.3394 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8438 +[titan] 2025-07-10 09:22:35,434 - root - INFO - lr: 3.6754e-05 gnorm: 1.04 [19:47:31< 2:12:05] +[titan] 2025-07-10 09:22:39,403 - root - INFO - step: 89995 loss: 16.1562 memory: 44.58GiB(31.99%) tps: 82,553 tflops: 284.90 mfu: 28.81% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6897 +[titan] 2025-07-10 09:22:39,403 - root - INFO - lr: 3.6747e-05 gnorm: 1.01 [19:47:35< 2:12:01] +[titan] 2025-07-10 09:22:42,524 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:22:43,327 - root - INFO - step: 90000 loss: 16.1041 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.6431 +[titan] 2025-07-10 09:22:43,328 - root - INFO - lr: 3.6740e-05 gnorm: 0.99 [19:47:39< 2:11:57] +[titan] 2025-07-10 09:22:43,328 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 09:22:45,792 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 09:22:45,793 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.46 seconds. +[titan] 2025-07-10 09:24:19,580 - root - INFO - step: 90005 loss: 16.2192 memory: 44.58GiB(31.99%) tps: 3,404 tflops: 11.75 mfu: 1.19% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7467 +[titan] 2025-07-10 09:24:19,580 - root - INFO - lr: 3.6734e-05 gnorm: 0.95 [19:49:15< 2:12:04] +[titan] 2025-07-10 09:24:23,547 - root - INFO - step: 90010 loss: 16.2935 memory: 44.58GiB(31.99%) tps: 82,593 tflops: 285.04 mfu: 28.82% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8011 +[titan] 2025-07-10 09:24:23,548 - root - INFO - lr: 3.6727e-05 gnorm: 0.99 [19:49:19< 2:12:00] +[titan] 2025-07-10 09:24:27,462 - root - INFO - step: 90015 loss: 16.1897 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.7196 +[titan] 2025-07-10 09:24:27,462 - root - INFO - lr: 3.6720e-05 gnorm: 1.00 [19:49:23< 2:11:56] +[titan] 2025-07-10 09:24:31,437 - root - INFO - step: 90020 loss: 16.3269 memory: 44.58GiB(31.99%) tps: 82,423 tflops: 284.46 mfu: 28.76% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.8414 +[titan] 2025-07-10 09:24:31,438 - root - INFO - lr: 3.6714e-05 gnorm: 1.01 [19:49:27< 2:11:52] +[titan] 2025-07-10 09:24:35,372 - root - INFO - step: 90025 loss: 16.1586 memory: 44.58GiB(31.99%) tps: 83,297 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.6918 +[titan] 2025-07-10 09:24:35,372 - root - INFO - lr: 3.6707e-05 gnorm: 1.00 [19:49:31< 2:11:48] +[titan] 2025-07-10 09:24:39,333 - root - INFO - step: 90030 loss: 15.9124 memory: 44.58GiB(31.99%) tps: 82,736 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.4498 global_avg_mtp_loss: 13.4627 +[titan] 2025-07-10 09:24:39,333 - root - INFO - lr: 3.6700e-05 gnorm: 1.10 [19:49:35< 2:11:44] +[titan] 2025-07-10 09:24:43,264 - root - INFO - step: 90035 loss: 16.2791 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.7792 +[titan] 2025-07-10 09:24:43,264 - root - INFO - lr: 3.6694e-05 gnorm: 1.01 [19:49:39< 2:11:40] +[titan] 2025-07-10 09:24:47,190 - root - INFO - step: 90040 loss: 16.2386 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 09:24:47,191 - root - INFO - lr: 3.6687e-05 gnorm: 1.06 [19:49:43< 2:11:36] +[titan] 2025-07-10 09:24:51,133 - root - INFO - step: 90045 loss: 16.2164 memory: 44.58GiB(31.99%) tps: 83,118 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4664 global_avg_mtp_loss: 13.7500 +[titan] 2025-07-10 09:24:51,133 - root - INFO - lr: 3.6680e-05 gnorm: 1.02 [19:49:47< 2:11:32] +[titan] 2025-07-10 09:24:54,249 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:24:55,041 - root - INFO - step: 90050 loss: 16.2251 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7323 +[titan] 2025-07-10 09:24:55,041 - root - INFO - lr: 3.6674e-05 gnorm: 1.04 [19:49:51< 2:11:28] +[titan] 2025-07-10 09:24:58,993 - root - INFO - step: 90055 loss: 16.2691 memory: 44.58GiB(31.99%) tps: 82,937 tflops: 286.23 mfu: 28.94% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7739 +[titan] 2025-07-10 09:24:58,993 - root - INFO - lr: 3.6667e-05 gnorm: 1.06 [19:49:55< 2:11:24] +[titan] 2025-07-10 09:25:02,955 - root - INFO - step: 90060 loss: 16.2505 memory: 44.58GiB(31.99%) tps: 82,712 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7737 +[titan] 2025-07-10 09:25:02,955 - root - INFO - lr: 3.6660e-05 gnorm: 1.03 [19:49:59< 2:11:20] +[titan] 2025-07-10 09:25:06,993 - root - INFO - step: 90065 loss: 16.1902 memory: 44.58GiB(31.99%) tps: 81,145 tflops: 280.04 mfu: 28.32% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7171 +[titan] 2025-07-10 09:25:06,994 - root - INFO - lr: 3.6654e-05 gnorm: 1.04 [19:50:03< 2:11:16] +[titan] 2025-07-10 09:25:11,041 - root - INFO - step: 90070 loss: 16.0827 memory: 44.58GiB(31.99%) tps: 80,956 tflops: 279.39 mfu: 28.25% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6307 +[titan] 2025-07-10 09:25:11,042 - root - INFO - lr: 3.6647e-05 gnorm: 1.02 [19:50:07< 2:11:12] +[titan] 2025-07-10 09:25:15,006 - root - INFO - step: 90075 loss: 16.0234 memory: 44.58GiB(31.99%) tps: 82,656 tflops: 285.26 mfu: 28.84% global_avg_ntp_loss: 2.4356 global_avg_mtp_loss: 13.5879 +[titan] 2025-07-10 09:25:15,006 - root - INFO - lr: 3.6640e-05 gnorm: 1.04 [19:50:11< 2:11:08] +[titan] 2025-07-10 09:25:18,993 - root - INFO - step: 90080 loss: 16.2748 memory: 44.58GiB(31.99%) tps: 82,208 tflops: 283.71 mfu: 28.69% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.7850 +[titan] 2025-07-10 09:25:18,993 - root - INFO - lr: 3.6634e-05 gnorm: 1.00 [19:50:15< 2:11:04] +[titan] 2025-07-10 09:25:22,927 - root - INFO - step: 90085 loss: 16.1651 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.7034 +[titan] 2025-07-10 09:25:22,927 - root - INFO - lr: 3.6627e-05 gnorm: 1.04 [19:50:19< 2:11:00] +[titan] 2025-07-10 09:25:26,854 - root - INFO - step: 90090 loss: 16.2494 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 09:25:26,855 - root - INFO - lr: 3.6620e-05 gnorm: 1.00 [19:50:23< 2:10:56] +[titan] 2025-07-10 09:25:30,866 - root - INFO - step: 90095 loss: 16.1551 memory: 44.58GiB(31.99%) tps: 81,690 tflops: 281.92 mfu: 28.51% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6947 +[titan] 2025-07-10 09:25:30,866 - root - INFO - lr: 3.6614e-05 gnorm: 1.06 [19:50:27< 2:10:52] +[titan] 2025-07-10 09:25:34,048 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:25:34,844 - root - INFO - step: 90100 loss: 16.2199 memory: 44.58GiB(31.99%) tps: 82,391 tflops: 284.34 mfu: 28.75% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7534 +[titan] 2025-07-10 09:25:34,844 - root - INFO - lr: 3.6607e-05 gnorm: 1.05 [19:50:31< 2:10:48] +[titan] 2025-07-10 09:25:38,813 - root - INFO - step: 90105 loss: 16.2854 memory: 44.58GiB(31.99%) tps: 82,555 tflops: 284.91 mfu: 28.81% global_avg_ntp_loss: 2.4895 global_avg_mtp_loss: 13.7959 +[titan] 2025-07-10 09:25:38,814 - root - INFO - lr: 3.6601e-05 gnorm: 1.04 [19:50:35< 2:10:44] +[titan] 2025-07-10 09:25:42,812 - root - INFO - step: 90110 loss: 16.1969 memory: 44.58GiB(31.99%) tps: 81,954 tflops: 282.84 mfu: 28.60% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7193 +[titan] 2025-07-10 09:25:42,812 - root - INFO - lr: 3.6594e-05 gnorm: 0.99 [19:50:39< 2:10:40] +[titan] 2025-07-10 09:25:44,532 - root - INFO - Dumping profiler traces at step 90112 +[titan] 2025-07-10 09:25:44,565 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:25:46,919 - root - INFO - step: 90115 loss: 16.2239 memory: 44.58GiB(31.99%) tps: 79,791 tflops: 275.37 mfu: 27.84% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7493 +[titan] 2025-07-10 09:25:46,920 - root - INFO - lr: 3.6587e-05 gnorm: 1.05 [19:50:43< 2:10:36] +[titan] 2025-07-10 09:25:50,864 - root - INFO - step: 90120 loss: 16.1248 memory: 44.58GiB(31.99%) tps: 83,075 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4542 global_avg_mtp_loss: 13.6706 +[titan] 2025-07-10 09:25:50,865 - root - INFO - lr: 3.6581e-05 gnorm: 0.99 [19:50:47< 2:10:32] +[titan] 2025-07-10 09:25:54,790 - root - INFO - step: 90125 loss: 16.2381 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7491 +[titan] 2025-07-10 09:25:54,790 - root - INFO - lr: 3.6574e-05 gnorm: 1.04 [19:50:51< 2:10:28] +[titan] 2025-07-10 09:25:58,714 - root - INFO - step: 90130 loss: 15.9729 memory: 44.58GiB(31.99%) tps: 83,505 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4443 global_avg_mtp_loss: 13.5286 +[titan] 2025-07-10 09:25:58,714 - root - INFO - lr: 3.6568e-05 gnorm: 1.03 [19:50:54< 2:10:24] +[titan] 2025-07-10 09:26:02,639 - root - INFO - step: 90135 loss: 16.3831 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5101 global_avg_mtp_loss: 13.8730 +[titan] 2025-07-10 09:26:02,639 - root - INFO - lr: 3.6561e-05 gnorm: 0.97 [19:50:58< 2:10:20] +[titan] 2025-07-10 09:26:06,545 - root - INFO - step: 90140 loss: 16.2944 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.8167 +[titan] 2025-07-10 09:26:06,545 - root - INFO - lr: 3.6554e-05 gnorm: 1.06 [19:51:02< 2:10:16] +[titan] 2025-07-10 09:26:10,463 - root - INFO - step: 90145 loss: 16.3956 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.8780 +[titan] 2025-07-10 09:26:10,463 - root - INFO - lr: 3.6548e-05 gnorm: 1.02 [19:51:06< 2:10:13] +[titan] 2025-07-10 09:26:13,615 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:26:14,401 - root - INFO - step: 90150 loss: 16.2170 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4824 global_avg_mtp_loss: 13.7346 +[titan] 2025-07-10 09:26:14,401 - root - INFO - lr: 3.6541e-05 gnorm: 1.04 [19:51:10< 2:10:09] +[titan] 2025-07-10 09:26:18,320 - root - INFO - step: 90155 loss: 16.3090 memory: 44.58GiB(31.99%) tps: 83,613 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.8074 +[titan] 2025-07-10 09:26:18,321 - root - INFO - lr: 3.6535e-05 gnorm: 1.02 [19:51:14< 2:10:05] +[titan] 2025-07-10 09:26:22,242 - root - INFO - step: 90160 loss: 16.1402 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.6816 +[titan] 2025-07-10 09:26:22,243 - root - INFO - lr: 3.6528e-05 gnorm: 1.07 [19:51:18< 2:10:01] +[titan] 2025-07-10 09:26:26,189 - root - INFO - step: 90165 loss: 16.2197 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7442 +[titan] 2025-07-10 09:26:26,189 - root - INFO - lr: 3.6521e-05 gnorm: 1.02 [19:51:22< 2:09:57] +[titan] 2025-07-10 09:26:30,143 - root - INFO - step: 90170 loss: 16.1222 memory: 44.58GiB(31.99%) tps: 82,875 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6669 +[titan] 2025-07-10 09:26:30,144 - root - INFO - lr: 3.6515e-05 gnorm: 0.97 [19:51:26< 2:09:53] +[titan] 2025-07-10 09:26:34,054 - root - INFO - step: 90175 loss: 16.2514 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7775 +[titan] 2025-07-10 09:26:34,055 - root - INFO - lr: 3.6508e-05 gnorm: 1.01 [19:51:30< 2:09:49] +[titan] 2025-07-10 09:26:37,972 - root - INFO - step: 90180 loss: 16.4675 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.5374 global_avg_mtp_loss: 13.9300 +[titan] 2025-07-10 09:26:37,972 - root - INFO - lr: 3.6502e-05 gnorm: 1.02 [19:51:34< 2:09:45] +[titan] 2025-07-10 09:26:41,878 - root - INFO - step: 90185 loss: 16.1721 memory: 44.58GiB(31.99%) tps: 83,901 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.6948 +[titan] 2025-07-10 09:26:41,878 - root - INFO - lr: 3.6495e-05 gnorm: 1.01 [19:51:38< 2:09:41] +[titan] 2025-07-10 09:26:45,791 - root - INFO - step: 90190 loss: 16.2235 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.7503 +[titan] 2025-07-10 09:26:45,791 - root - INFO - lr: 3.6489e-05 gnorm: 1.03 [19:51:42< 2:09:37] +[titan] 2025-07-10 09:26:49,711 - root - INFO - step: 90195 loss: 16.3259 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.8340 +[titan] 2025-07-10 09:26:49,711 - root - INFO - lr: 3.6482e-05 gnorm: 1.02 [19:51:45< 2:09:33] +[titan] 2025-07-10 09:26:52,811 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:26:53,616 - root - INFO - step: 90200 loss: 16.3300 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.8377 +[titan] 2025-07-10 09:26:53,616 - root - INFO - lr: 3.6476e-05 gnorm: 1.00 [19:51:49< 2:09:29] +[titan] 2025-07-10 09:26:57,527 - root - INFO - step: 90205 loss: 16.1519 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6874 +[titan] 2025-07-10 09:26:57,528 - root - INFO - lr: 3.6469e-05 gnorm: 1.03 [19:51:53< 2:09:25] +[titan] 2025-07-10 09:27:01,440 - root - INFO - step: 90210 loss: 16.3357 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.8363 +[titan] 2025-07-10 09:27:01,440 - root - INFO - lr: 3.6462e-05 gnorm: 0.98 [19:51:57< 2:09:21] +[titan] 2025-07-10 09:27:05,343 - root - INFO - step: 90215 loss: 16.2941 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7986 +[titan] 2025-07-10 09:27:05,344 - root - INFO - lr: 3.6456e-05 gnorm: 1.01 [19:52:01< 2:09:17] +[titan] 2025-07-10 09:27:09,296 - root - INFO - step: 90220 loss: 15.7782 memory: 44.58GiB(31.99%) tps: 82,913 tflops: 286.15 mfu: 28.93% global_avg_ntp_loss: 2.4077 global_avg_mtp_loss: 13.3705 +[titan] 2025-07-10 09:27:09,296 - root - INFO - lr: 3.6449e-05 gnorm: 1.07 [19:52:05< 2:09:13] +[titan] 2025-07-10 09:27:13,246 - root - INFO - step: 90225 loss: 15.8960 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.27 mfu: 28.95% global_avg_ntp_loss: 2.4309 global_avg_mtp_loss: 13.4651 +[titan] 2025-07-10 09:27:13,247 - root - INFO - lr: 3.6443e-05 gnorm: 1.02 [19:52:09< 2:09:09] +[titan] 2025-07-10 09:27:17,177 - root - INFO - step: 90230 loss: 16.0704 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6265 +[titan] 2025-07-10 09:27:17,177 - root - INFO - lr: 3.6436e-05 gnorm: 1.04 [19:52:13< 2:09:05] +[titan] 2025-07-10 09:27:21,117 - root - INFO - step: 90235 loss: 16.1650 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.6958 +[titan] 2025-07-10 09:27:21,118 - root - INFO - lr: 3.6430e-05 gnorm: 1.02 [19:52:17< 2:09:01] +[titan] 2025-07-10 09:27:25,071 - root - INFO - step: 90240 loss: 16.2525 memory: 44.58GiB(31.99%) tps: 82,899 tflops: 286.10 mfu: 28.93% global_avg_ntp_loss: 2.4906 global_avg_mtp_loss: 13.7619 +[titan] 2025-07-10 09:27:25,071 - root - INFO - lr: 3.6423e-05 gnorm: 1.06 [19:52:21< 2:08:57] +[titan] 2025-07-10 09:27:29,009 - root - INFO - step: 90245 loss: 16.3011 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4913 global_avg_mtp_loss: 13.8097 +[titan] 2025-07-10 09:27:29,010 - root - INFO - lr: 3.6417e-05 gnorm: 1.07 [19:52:25< 2:08:53] +[titan] 2025-07-10 09:27:32,145 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:27:32,933 - root - INFO - step: 90250 loss: 16.3187 memory: 44.58GiB(31.99%) tps: 83,529 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8254 +[titan] 2025-07-10 09:27:32,933 - root - INFO - lr: 3.6410e-05 gnorm: 1.02 [19:52:29< 2:08:49] +[titan] 2025-07-10 09:27:36,835 - root - INFO - step: 90255 loss: 16.1296 memory: 44.58GiB(31.99%) tps: 83,985 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6618 +[titan] 2025-07-10 09:27:36,835 - root - INFO - lr: 3.6404e-05 gnorm: 1.03 [19:52:33< 2:08:45] +[titan] 2025-07-10 09:27:40,734 - root - INFO - step: 90260 loss: 16.3060 memory: 44.58GiB(31.99%) tps: 84,037 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.8239 +[titan] 2025-07-10 09:27:40,735 - root - INFO - lr: 3.6397e-05 gnorm: 1.08 [19:52:37< 2:08:41] +[titan] 2025-07-10 09:27:44,657 - root - INFO - step: 90265 loss: 16.2194 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7500 +[titan] 2025-07-10 09:27:44,657 - root - INFO - lr: 3.6391e-05 gnorm: 0.98 [19:52:40< 2:08:37] +[titan] 2025-07-10 09:27:48,578 - root - INFO - step: 90270 loss: 16.2920 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.8135 +[titan] 2025-07-10 09:27:48,578 - root - INFO - lr: 3.6384e-05 gnorm: 0.99 [19:52:44< 2:08:33] +[titan] 2025-07-10 09:27:52,483 - root - INFO - step: 90275 loss: 15.5468 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.3800 global_avg_mtp_loss: 13.1668 +[titan] 2025-07-10 09:27:52,483 - root - INFO - lr: 3.6378e-05 gnorm: 1.03 [19:52:48< 2:08:29] +[titan] 2025-07-10 09:27:56,394 - root - INFO - step: 90280 loss: 15.8461 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4214 global_avg_mtp_loss: 13.4247 +[titan] 2025-07-10 09:27:56,395 - root - INFO - lr: 3.6371e-05 gnorm: 1.06 [19:52:52< 2:08:25] +[titan] 2025-07-10 09:28:00,333 - root - INFO - step: 90285 loss: 16.3198 memory: 44.58GiB(31.99%) tps: 83,201 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.5067 global_avg_mtp_loss: 13.8131 +[titan] 2025-07-10 09:28:00,333 - root - INFO - lr: 3.6365e-05 gnorm: 1.00 [19:52:56< 2:08:21] +[titan] 2025-07-10 09:28:04,254 - root - INFO - step: 90290 loss: 16.2331 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.7542 +[titan] 2025-07-10 09:28:04,254 - root - INFO - lr: 3.6358e-05 gnorm: 1.00 [19:53:00< 2:08:17] +[titan] 2025-07-10 09:28:08,172 - root - INFO - step: 90295 loss: 16.3923 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 09:28:08,173 - root - INFO - lr: 3.6352e-05 gnorm: 1.02 [19:53:04< 2:08:13] +[titan] 2025-07-10 09:28:11,286 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:28:12,077 - root - INFO - step: 90300 loss: 15.7124 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.3817 global_avg_mtp_loss: 13.3307 +[titan] 2025-07-10 09:28:12,077 - root - INFO - lr: 3.6345e-05 gnorm: 1.08 [19:53:08< 2:08:09] +[titan] 2025-07-10 09:28:15,990 - root - INFO - step: 90305 loss: 16.2990 memory: 44.58GiB(31.99%) tps: 83,752 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.8094 +[titan] 2025-07-10 09:28:15,990 - root - INFO - lr: 3.6339e-05 gnorm: 0.98 [19:53:12< 2:08:06] +[titan] 2025-07-10 09:28:19,961 - root - INFO - step: 90310 loss: 16.1285 memory: 44.58GiB(31.99%) tps: 82,529 tflops: 284.82 mfu: 28.80% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6721 +[titan] 2025-07-10 09:28:19,961 - root - INFO - lr: 3.6332e-05 gnorm: 1.03 [19:53:16< 2:08:02] +[titan] 2025-07-10 09:28:23,852 - root - INFO - step: 90315 loss: 16.0449 memory: 44.58GiB(31.99%) tps: 84,210 tflops: 290.62 mfu: 29.39% global_avg_ntp_loss: 2.4344 global_avg_mtp_loss: 13.6105 +[titan] 2025-07-10 09:28:23,853 - root - INFO - lr: 3.6326e-05 gnorm: 1.03 [19:53:20< 2:07:58] +[titan] 2025-07-10 09:28:27,760 - root - INFO - step: 90320 loss: 16.1993 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 09:28:27,760 - root - INFO - lr: 3.6319e-05 gnorm: 1.02 [19:53:24< 2:07:54] +[titan] 2025-07-10 09:28:31,679 - root - INFO - step: 90325 loss: 16.3082 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.8203 +[titan] 2025-07-10 09:28:31,679 - root - INFO - lr: 3.6313e-05 gnorm: 1.02 [19:53:27< 2:07:50] +[titan] 2025-07-10 09:28:35,618 - root - INFO - step: 90330 loss: 16.1035 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4542 global_avg_mtp_loss: 13.6494 +[titan] 2025-07-10 09:28:35,618 - root - INFO - lr: 3.6306e-05 gnorm: 0.98 [19:53:31< 2:07:46] +[titan] 2025-07-10 09:28:39,532 - root - INFO - step: 90335 loss: 16.2695 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-10 09:28:39,532 - root - INFO - lr: 3.6300e-05 gnorm: 1.01 [19:53:35< 2:07:42] +[titan] 2025-07-10 09:28:43,429 - root - INFO - step: 90340 loss: 15.8801 memory: 44.58GiB(31.99%) tps: 84,087 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4120 global_avg_mtp_loss: 13.4681 +[titan] 2025-07-10 09:28:43,429 - root - INFO - lr: 3.6293e-05 gnorm: 1.02 [19:53:39< 2:07:38] +[titan] 2025-07-10 09:28:47,343 - root - INFO - step: 90345 loss: 16.0932 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4498 global_avg_mtp_loss: 13.6434 +[titan] 2025-07-10 09:28:47,344 - root - INFO - lr: 3.6287e-05 gnorm: 1.03 [19:53:43< 2:07:34] +[titan] 2025-07-10 09:28:50,476 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:28:51,281 - root - INFO - step: 90350 loss: 15.9288 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4257 global_avg_mtp_loss: 13.5031 +[titan] 2025-07-10 09:28:51,281 - root - INFO - lr: 3.6280e-05 gnorm: 1.03 [19:53:47< 2:07:30] +[titan] 2025-07-10 09:28:55,194 - root - INFO - step: 90355 loss: 16.3042 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8127 +[titan] 2025-07-10 09:28:55,194 - root - INFO - lr: 3.6274e-05 gnorm: 0.96 [19:53:51< 2:07:26] +[titan] 2025-07-10 09:28:59,149 - root - INFO - step: 90360 loss: 16.0702 memory: 44.58GiB(31.99%) tps: 82,840 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6182 +[titan] 2025-07-10 09:28:59,150 - root - INFO - lr: 3.6267e-05 gnorm: 1.02 [19:53:55< 2:07:22] +[titan] 2025-07-10 09:29:03,053 - root - INFO - step: 90365 loss: 16.2233 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4621 global_avg_mtp_loss: 13.7612 +[titan] 2025-07-10 09:29:03,054 - root - INFO - lr: 3.6261e-05 gnorm: 1.08 [19:53:59< 2:07:18] +[titan] 2025-07-10 09:29:06,975 - root - INFO - step: 90370 loss: 16.2669 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-10 09:29:06,975 - root - INFO - lr: 3.6255e-05 gnorm: 1.02 [19:54:03< 2:07:14] +[titan] 2025-07-10 09:29:10,906 - root - INFO - step: 90375 loss: 16.0036 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4442 global_avg_mtp_loss: 13.5594 +[titan] 2025-07-10 09:29:10,906 - root - INFO - lr: 3.6248e-05 gnorm: 1.00 [19:54:07< 2:07:10] +[titan] 2025-07-10 09:29:14,851 - root - INFO - step: 90380 loss: 16.2305 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4766 global_avg_mtp_loss: 13.7538 +[titan] 2025-07-10 09:29:14,852 - root - INFO - lr: 3.6242e-05 gnorm: 1.01 [19:54:11< 2:07:06] +[titan] 2025-07-10 09:29:18,798 - root - INFO - step: 90385 loss: 16.2916 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.8041 +[titan] 2025-07-10 09:29:18,798 - root - INFO - lr: 3.6235e-05 gnorm: 0.98 [19:54:15< 2:07:02] +[titan] 2025-07-10 09:29:22,747 - root - INFO - step: 90390 loss: 16.1251 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.40 mfu: 28.96% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6655 +[titan] 2025-07-10 09:29:22,747 - root - INFO - lr: 3.6229e-05 gnorm: 1.03 [19:54:19< 2:06:58] +[titan] 2025-07-10 09:29:26,668 - root - INFO - step: 90395 loss: 16.0638 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.5965 +[titan] 2025-07-10 09:29:26,669 - root - INFO - lr: 3.6222e-05 gnorm: 1.02 [19:54:22< 2:06:54] +[titan] 2025-07-10 09:29:29,798 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:29:30,592 - root - INFO - step: 90400 loss: 16.1684 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4535 global_avg_mtp_loss: 13.7149 +[titan] 2025-07-10 09:29:30,593 - root - INFO - lr: 3.6216e-05 gnorm: 1.07 [19:54:26< 2:06:50] +[titan] 2025-07-10 09:29:34,529 - root - INFO - step: 90405 loss: 16.3903 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.8785 +[titan] 2025-07-10 09:29:34,529 - root - INFO - lr: 3.6209e-05 gnorm: 1.02 [19:54:30< 2:06:46] +[titan] 2025-07-10 09:29:38,480 - root - INFO - step: 90410 loss: 15.9995 memory: 44.58GiB(31.99%) tps: 82,945 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4270 global_avg_mtp_loss: 13.5726 +[titan] 2025-07-10 09:29:38,480 - root - INFO - lr: 3.6203e-05 gnorm: 1.01 [19:54:34< 2:06:42] +[titan] 2025-07-10 09:29:42,389 - root - INFO - step: 90415 loss: 16.1740 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7020 +[titan] 2025-07-10 09:29:42,389 - root - INFO - lr: 3.6197e-05 gnorm: 1.00 [19:54:38< 2:06:38] +[titan] 2025-07-10 09:29:46,324 - root - INFO - step: 90420 loss: 16.3443 memory: 44.58GiB(31.99%) tps: 83,283 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5051 global_avg_mtp_loss: 13.8392 +[titan] 2025-07-10 09:29:46,324 - root - INFO - lr: 3.6190e-05 gnorm: 1.00 [19:54:42< 2:06:34] +[titan] 2025-07-10 09:29:50,240 - root - INFO - step: 90425 loss: 16.1885 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7104 +[titan] 2025-07-10 09:29:50,240 - root - INFO - lr: 3.6184e-05 gnorm: 1.05 [19:54:46< 2:06:30] +[titan] 2025-07-10 09:29:54,197 - root - INFO - step: 90430 loss: 16.3651 memory: 44.58GiB(31.99%) tps: 82,811 tflops: 285.80 mfu: 28.90% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8736 +[titan] 2025-07-10 09:29:54,197 - root - INFO - lr: 3.6177e-05 gnorm: 1.00 [19:54:50< 2:06:26] +[titan] 2025-07-10 09:29:58,125 - root - INFO - step: 90435 loss: 16.1847 memory: 44.58GiB(31.99%) tps: 83,432 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.7129 +[titan] 2025-07-10 09:29:58,125 - root - INFO - lr: 3.6171e-05 gnorm: 0.99 [19:54:54< 2:06:22] +[titan] 2025-07-10 09:30:02,039 - root - INFO - step: 90440 loss: 16.1012 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4499 global_avg_mtp_loss: 13.6513 +[titan] 2025-07-10 09:30:02,040 - root - INFO - lr: 3.6165e-05 gnorm: 1.07 [19:54:58< 2:06:18] +[titan] 2025-07-10 09:30:05,958 - root - INFO - step: 90445 loss: 16.1203 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6621 +[titan] 2025-07-10 09:30:05,958 - root - INFO - lr: 3.6158e-05 gnorm: 1.09 [19:55:02< 2:06:14] +[titan] 2025-07-10 09:30:09,086 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:30:09,879 - root - INFO - step: 90450 loss: 15.8840 memory: 44.58GiB(31.99%) tps: 83,572 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4120 global_avg_mtp_loss: 13.4720 +[titan] 2025-07-10 09:30:09,879 - root - INFO - lr: 3.6152e-05 gnorm: 1.06 [19:55:06< 2:06:10] +[titan] 2025-07-10 09:30:13,806 - root - INFO - step: 90455 loss: 16.0811 memory: 44.58GiB(31.99%) tps: 83,446 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.6361 +[titan] 2025-07-10 09:30:13,807 - root - INFO - lr: 3.6145e-05 gnorm: 1.03 [19:55:10< 2:06:06] +[titan] 2025-07-10 09:30:17,765 - root - INFO - step: 90460 loss: 15.9566 memory: 44.58GiB(31.99%) tps: 82,788 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.4199 global_avg_mtp_loss: 13.5368 +[titan] 2025-07-10 09:30:17,765 - root - INFO - lr: 3.6139e-05 gnorm: 1.01 [19:55:14< 2:06:03] +[titan] 2025-07-10 09:30:21,669 - root - INFO - step: 90465 loss: 16.0695 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4486 global_avg_mtp_loss: 13.6209 +[titan] 2025-07-10 09:30:21,669 - root - INFO - lr: 3.6133e-05 gnorm: 1.03 [19:55:17< 2:05:59] +[titan] 2025-07-10 09:30:25,577 - root - INFO - step: 90470 loss: 16.3200 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.8357 +[titan] 2025-07-10 09:30:25,578 - root - INFO - lr: 3.6126e-05 gnorm: 1.00 [19:55:21< 2:05:55] +[titan] 2025-07-10 09:30:29,494 - root - INFO - step: 90475 loss: 16.1581 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6937 +[titan] 2025-07-10 09:30:29,494 - root - INFO - lr: 3.6120e-05 gnorm: 1.05 [19:55:25< 2:05:51] +[titan] 2025-07-10 09:30:33,495 - root - INFO - step: 90480 loss: 16.0931 memory: 44.58GiB(31.99%) tps: 81,894 tflops: 282.63 mfu: 28.58% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.6467 +[titan] 2025-07-10 09:30:33,496 - root - INFO - lr: 3.6114e-05 gnorm: 1.00 [19:55:29< 2:05:47] +[titan] 2025-07-10 09:30:37,399 - root - INFO - step: 90485 loss: 16.2871 memory: 44.58GiB(31.99%) tps: 83,943 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.8039 +[titan] 2025-07-10 09:30:37,400 - root - INFO - lr: 3.6107e-05 gnorm: 0.98 [19:55:33< 2:05:43] +[titan] 2025-07-10 09:30:41,361 - root - INFO - step: 90490 loss: 16.0669 memory: 44.58GiB(31.99%) tps: 82,724 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6142 +[titan] 2025-07-10 09:30:41,361 - root - INFO - lr: 3.6101e-05 gnorm: 1.09 [19:55:37< 2:05:39] +[titan] 2025-07-10 09:30:45,280 - root - INFO - step: 90495 loss: 16.0396 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.5975 +[titan] 2025-07-10 09:30:45,280 - root - INFO - lr: 3.6094e-05 gnorm: 1.03 [19:55:41< 2:05:35] +[titan] 2025-07-10 09:30:48,430 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:30:49,217 - root - INFO - step: 90500 loss: 16.1696 memory: 44.58GiB(31.99%) tps: 83,233 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4626 global_avg_mtp_loss: 13.7070 +[titan] 2025-07-10 09:30:49,217 - root - INFO - lr: 3.6088e-05 gnorm: 1.03 [19:55:45< 2:05:31] +[titan] 2025-07-10 09:30:53,176 - root - INFO - step: 90505 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 82,785 tflops: 285.71 mfu: 28.89% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.6476 +[titan] 2025-07-10 09:30:53,176 - root - INFO - lr: 3.6082e-05 gnorm: 1.00 [19:55:49< 2:05:27] +[titan] 2025-07-10 09:30:57,094 - root - INFO - step: 90510 loss: 16.2908 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.8041 +[titan] 2025-07-10 09:30:57,095 - root - INFO - lr: 3.6075e-05 gnorm: 1.01 [19:55:53< 2:05:23] +[titan] 2025-07-10 09:31:01,002 - root - INFO - step: 90515 loss: 16.0929 memory: 44.58GiB(31.99%) tps: 83,856 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6216 +[titan] 2025-07-10 09:31:01,003 - root - INFO - lr: 3.6069e-05 gnorm: 1.07 [19:55:57< 2:05:19] +[titan] 2025-07-10 09:31:04,927 - root - INFO - step: 90520 loss: 16.3909 memory: 44.58GiB(31.99%) tps: 83,497 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.5087 global_avg_mtp_loss: 13.8821 +[titan] 2025-07-10 09:31:04,928 - root - INFO - lr: 3.6063e-05 gnorm: 0.99 [19:56:01< 2:05:15] +[titan] 2025-07-10 09:31:08,862 - root - INFO - step: 90525 loss: 16.5209 memory: 44.58GiB(31.99%) tps: 83,294 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9840 +[titan] 2025-07-10 09:31:08,862 - root - INFO - lr: 3.6056e-05 gnorm: 1.01 [19:56:05< 2:05:11] +[titan] 2025-07-10 09:31:12,758 - root - INFO - step: 90530 loss: 16.1402 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6736 +[titan] 2025-07-10 09:31:12,758 - root - INFO - lr: 3.6050e-05 gnorm: 1.04 [19:56:08< 2:05:07] +[titan] 2025-07-10 09:31:16,674 - root - INFO - step: 90535 loss: 16.1920 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4595 global_avg_mtp_loss: 13.7324 +[titan] 2025-07-10 09:31:16,674 - root - INFO - lr: 3.6044e-05 gnorm: 1.06 [19:56:12< 2:05:03] +[titan] 2025-07-10 09:31:20,578 - root - INFO - step: 90540 loss: 16.0453 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.5866 +[titan] 2025-07-10 09:31:20,578 - root - INFO - lr: 3.6037e-05 gnorm: 1.08 [19:56:16< 2:04:59] +[titan] 2025-07-10 09:31:24,493 - root - INFO - step: 90545 loss: 16.1043 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4562 global_avg_mtp_loss: 13.6481 +[titan] 2025-07-10 09:31:24,494 - root - INFO - lr: 3.6031e-05 gnorm: 1.07 [19:56:20< 2:04:55] +[titan] 2025-07-10 09:31:27,616 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:31:28,410 - root - INFO - step: 90550 loss: 16.2164 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7408 +[titan] 2025-07-10 09:31:28,410 - root - INFO - lr: 3.6025e-05 gnorm: 1.03 [19:56:24< 2:04:51] +[titan] 2025-07-10 09:31:32,337 - root - INFO - step: 90555 loss: 16.1600 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6972 +[titan] 2025-07-10 09:31:32,337 - root - INFO - lr: 3.6018e-05 gnorm: 1.01 [19:56:28< 2:04:47] +[titan] 2025-07-10 09:31:36,273 - root - INFO - step: 90560 loss: 16.3783 memory: 44.58GiB(31.99%) tps: 83,259 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8743 +[titan] 2025-07-10 09:31:36,274 - root - INFO - lr: 3.6012e-05 gnorm: 1.00 [19:56:32< 2:04:43] +[titan] 2025-07-10 09:31:40,201 - root - INFO - step: 90565 loss: 15.8768 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4175 global_avg_mtp_loss: 13.4593 +[titan] 2025-07-10 09:31:40,202 - root - INFO - lr: 3.6006e-05 gnorm: 1.01 [19:56:36< 2:04:39] +[titan] 2025-07-10 09:31:44,114 - root - INFO - step: 90570 loss: 16.2205 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.7481 +[titan] 2025-07-10 09:31:44,114 - root - INFO - lr: 3.5999e-05 gnorm: 1.02 [19:56:40< 2:04:35] +[titan] 2025-07-10 09:31:48,025 - root - INFO - step: 90575 loss: 16.0661 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6057 +[titan] 2025-07-10 09:31:48,025 - root - INFO - lr: 3.5993e-05 gnorm: 1.02 [19:56:44< 2:04:31] +[titan] 2025-07-10 09:31:51,926 - root - INFO - step: 90580 loss: 16.1748 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7010 +[titan] 2025-07-10 09:31:51,927 - root - INFO - lr: 3.5987e-05 gnorm: 1.07 [19:56:48< 2:04:27] +[titan] 2025-07-10 09:31:55,825 - root - INFO - step: 90585 loss: 16.2524 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7793 +[titan] 2025-07-10 09:31:55,825 - root - INFO - lr: 3.5980e-05 gnorm: 0.98 [19:56:52< 2:04:23] +[titan] 2025-07-10 09:31:59,731 - root - INFO - step: 90590 loss: 16.1721 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7037 +[titan] 2025-07-10 09:31:59,732 - root - INFO - lr: 3.5974e-05 gnorm: 1.03 [19:56:55< 2:04:19] +[titan] 2025-07-10 09:32:03,645 - root - INFO - step: 90595 loss: 16.0568 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4432 global_avg_mtp_loss: 13.6136 +[titan] 2025-07-10 09:32:03,646 - root - INFO - lr: 3.5968e-05 gnorm: 1.03 [19:56:59< 2:04:15] +[titan] 2025-07-10 09:32:06,787 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:32:07,577 - root - INFO - step: 90600 loss: 16.1871 memory: 44.58GiB(31.99%) tps: 83,344 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7184 +[titan] 2025-07-10 09:32:07,578 - root - INFO - lr: 3.5962e-05 gnorm: 1.05 [19:57:03< 2:04:11] +[titan] 2025-07-10 09:32:11,478 - root - INFO - step: 90605 loss: 16.0245 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.5871 +[titan] 2025-07-10 09:32:11,478 - root - INFO - lr: 3.5955e-05 gnorm: 1.08 [19:57:07< 2:04:07] +[titan] 2025-07-10 09:32:15,398 - root - INFO - step: 90610 loss: 16.3469 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8515 +[titan] 2025-07-10 09:32:15,398 - root - INFO - lr: 3.5949e-05 gnorm: 0.99 [19:57:11< 2:04:03] +[titan] 2025-07-10 09:32:19,309 - root - INFO - step: 90615 loss: 16.3577 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8644 +[titan] 2025-07-10 09:32:19,310 - root - INFO - lr: 3.5943e-05 gnorm: 1.01 [19:57:15< 2:04:00] +[titan] 2025-07-10 09:32:23,218 - root - INFO - step: 90620 loss: 16.0800 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4505 global_avg_mtp_loss: 13.6295 +[titan] 2025-07-10 09:32:23,219 - root - INFO - lr: 3.5936e-05 gnorm: 1.01 [19:57:19< 2:03:56] +[titan] 2025-07-10 09:32:26,611 - root - INFO - Dumping profiler traces at step 90624 +[titan] 2025-07-10 09:32:26,644 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:32:27,467 - root - INFO - step: 90625 loss: 16.2354 memory: 44.58GiB(31.99%) tps: 77,136 tflops: 266.21 mfu: 26.92% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7598 +[titan] 2025-07-10 09:32:27,467 - root - INFO - lr: 3.5930e-05 gnorm: 1.03 [19:57:23< 2:03:52] +[titan] 2025-07-10 09:32:31,395 - root - INFO - step: 90630 loss: 16.3671 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.8502 +[titan] 2025-07-10 09:32:31,396 - root - INFO - lr: 3.5924e-05 gnorm: 1.03 [19:57:27< 2:03:48] +[titan] 2025-07-10 09:32:35,306 - root - INFO - step: 90635 loss: 16.3215 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8220 +[titan] 2025-07-10 09:32:35,307 - root - INFO - lr: 3.5918e-05 gnorm: 1.05 [19:57:31< 2:03:44] +[titan] 2025-07-10 09:32:39,261 - root - INFO - step: 90640 loss: 16.4782 memory: 44.58GiB(31.99%) tps: 82,871 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.5234 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-10 09:32:39,261 - root - INFO - lr: 3.5911e-05 gnorm: 1.03 [19:57:35< 2:03:40] +[titan] 2025-07-10 09:32:43,178 - root - INFO - step: 90645 loss: 15.9466 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4186 global_avg_mtp_loss: 13.5280 +[titan] 2025-07-10 09:32:43,178 - root - INFO - lr: 3.5905e-05 gnorm: 1.00 [19:57:39< 2:03:36] +[titan] 2025-07-10 09:32:46,303 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:32:47,090 - root - INFO - step: 90650 loss: 15.8865 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4260 global_avg_mtp_loss: 13.4605 +[titan] 2025-07-10 09:32:47,091 - root - INFO - lr: 3.5899e-05 gnorm: 1.02 [19:57:43< 2:03:32] +[titan] 2025-07-10 09:32:51,005 - root - INFO - step: 90655 loss: 16.1658 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4643 global_avg_mtp_loss: 13.7014 +[titan] 2025-07-10 09:32:51,005 - root - INFO - lr: 3.5892e-05 gnorm: 1.03 [19:57:47< 2:03:28] +[titan] 2025-07-10 09:32:54,907 - root - INFO - step: 90660 loss: 16.4440 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5173 global_avg_mtp_loss: 13.9267 +[titan] 2025-07-10 09:32:54,907 - root - INFO - lr: 3.5886e-05 gnorm: 1.02 [19:57:51< 2:03:24] +[titan] 2025-07-10 09:32:58,811 - root - INFO - step: 90665 loss: 15.9791 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4255 global_avg_mtp_loss: 13.5536 +[titan] 2025-07-10 09:32:58,811 - root - INFO - lr: 3.5880e-05 gnorm: 0.99 [19:57:55< 2:03:20] +[titan] 2025-07-10 09:33:02,712 - root - INFO - step: 90670 loss: 16.0500 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6060 +[titan] 2025-07-10 09:33:02,712 - root - INFO - lr: 3.5874e-05 gnorm: 1.02 [19:57:58< 2:03:16] +[titan] 2025-07-10 09:33:06,634 - root - INFO - step: 90675 loss: 16.1552 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.6834 +[titan] 2025-07-10 09:33:06,634 - root - INFO - lr: 3.5867e-05 gnorm: 1.06 [19:58:02< 2:03:12] +[titan] 2025-07-10 09:33:10,540 - root - INFO - step: 90680 loss: 16.1475 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 09:33:10,541 - root - INFO - lr: 3.5861e-05 gnorm: 0.96 [19:58:06< 2:03:08] +[titan] 2025-07-10 09:33:14,489 - root - INFO - step: 90685 loss: 16.2140 memory: 44.58GiB(31.99%) tps: 82,998 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.4901 global_avg_mtp_loss: 13.7239 +[titan] 2025-07-10 09:33:14,489 - root - INFO - lr: 3.5855e-05 gnorm: 1.03 [19:58:10< 2:03:04] +[titan] 2025-07-10 09:33:18,402 - root - INFO - step: 90690 loss: 15.9335 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4165 global_avg_mtp_loss: 13.5171 +[titan] 2025-07-10 09:33:18,403 - root - INFO - lr: 3.5849e-05 gnorm: 1.02 [19:58:14< 2:03:00] +[titan] 2025-07-10 09:33:22,318 - root - INFO - step: 90695 loss: 16.2739 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.7797 +[titan] 2025-07-10 09:33:22,319 - root - INFO - lr: 3.5843e-05 gnorm: 1.07 [19:58:18< 2:02:56] +[titan] 2025-07-10 09:33:25,445 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:33:26,230 - root - INFO - step: 90700 loss: 16.2569 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7722 +[titan] 2025-07-10 09:33:26,231 - root - INFO - lr: 3.5836e-05 gnorm: 1.03 [19:58:22< 2:02:52] +[titan] 2025-07-10 09:33:30,194 - root - INFO - step: 90705 loss: 16.0077 memory: 44.58GiB(31.99%) tps: 82,689 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.4442 global_avg_mtp_loss: 13.5635 +[titan] 2025-07-10 09:33:30,194 - root - INFO - lr: 3.5830e-05 gnorm: 1.02 [19:58:26< 2:02:48] +[titan] 2025-07-10 09:33:34,099 - root - INFO - step: 90710 loss: 16.2474 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7674 +[titan] 2025-07-10 09:33:34,099 - root - INFO - lr: 3.5824e-05 gnorm: 1.00 [19:58:30< 2:02:44] +[titan] 2025-07-10 09:33:38,007 - root - INFO - step: 90715 loss: 16.2957 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.8110 +[titan] 2025-07-10 09:33:38,007 - root - INFO - lr: 3.5818e-05 gnorm: 1.04 [19:58:34< 2:02:40] +[titan] 2025-07-10 09:33:41,914 - root - INFO - step: 90720 loss: 16.1493 memory: 44.58GiB(31.99%) tps: 83,872 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4607 global_avg_mtp_loss: 13.6887 +[titan] 2025-07-10 09:33:41,914 - root - INFO - lr: 3.5811e-05 gnorm: 1.02 [19:58:38< 2:02:36] +[titan] 2025-07-10 09:33:45,812 - root - INFO - step: 90725 loss: 16.3509 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5068 global_avg_mtp_loss: 13.8441 +[titan] 2025-07-10 09:33:45,813 - root - INFO - lr: 3.5805e-05 gnorm: 1.01 [19:58:42< 2:02:32] +[titan] 2025-07-10 09:33:49,737 - root - INFO - step: 90730 loss: 16.2673 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7836 +[titan] 2025-07-10 09:33:49,738 - root - INFO - lr: 3.5799e-05 gnorm: 1.03 [19:58:45< 2:02:28] +[titan] 2025-07-10 09:33:53,652 - root - INFO - step: 90735 loss: 16.1815 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7127 +[titan] 2025-07-10 09:33:53,652 - root - INFO - lr: 3.5793e-05 gnorm: 1.00 [19:58:49< 2:02:24] +[titan] 2025-07-10 09:33:57,606 - root - INFO - step: 90740 loss: 16.6062 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.5438 global_avg_mtp_loss: 14.0624 +[titan] 2025-07-10 09:33:57,606 - root - INFO - lr: 3.5787e-05 gnorm: 1.02 [19:58:53< 2:02:20] +[titan] 2025-07-10 09:34:01,512 - root - INFO - step: 90745 loss: 16.2289 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 09:34:01,512 - root - INFO - lr: 3.5780e-05 gnorm: 1.01 [19:58:57< 2:02:16] +[titan] 2025-07-10 09:34:04,645 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:34:05,440 - root - INFO - step: 90750 loss: 16.0549 memory: 44.58GiB(31.99%) tps: 83,428 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4408 global_avg_mtp_loss: 13.6140 +[titan] 2025-07-10 09:34:05,440 - root - INFO - lr: 3.5774e-05 gnorm: 0.99 [19:59:01< 2:02:12] +[titan] 2025-07-10 09:34:09,356 - root - INFO - step: 90755 loss: 16.4047 memory: 44.58GiB(31.99%) tps: 83,692 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.5228 global_avg_mtp_loss: 13.8819 +[titan] 2025-07-10 09:34:09,356 - root - INFO - lr: 3.5768e-05 gnorm: 1.02 [19:59:05< 2:02:08] +[titan] 2025-07-10 09:34:13,297 - root - INFO - step: 90760 loss: 15.8736 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4218 global_avg_mtp_loss: 13.4518 +[titan] 2025-07-10 09:34:13,297 - root - INFO - lr: 3.5762e-05 gnorm: 1.10 [19:59:09< 2:02:04] +[titan] 2025-07-10 09:34:17,206 - root - INFO - step: 90765 loss: 15.9381 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4237 global_avg_mtp_loss: 13.5144 +[titan] 2025-07-10 09:34:17,207 - root - INFO - lr: 3.5756e-05 gnorm: 1.01 [19:59:13< 2:02:00] +[titan] 2025-07-10 09:34:21,118 - root - INFO - step: 90770 loss: 16.2466 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.7790 +[titan] 2025-07-10 09:34:21,119 - root - INFO - lr: 3.5749e-05 gnorm: 1.08 [19:59:17< 2:01:57] +[titan] 2025-07-10 09:34:25,068 - root - INFO - step: 90775 loss: 16.2520 memory: 44.58GiB(31.99%) tps: 82,967 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7712 +[titan] 2025-07-10 09:34:25,069 - root - INFO - lr: 3.5743e-05 gnorm: 1.08 [19:59:21< 2:01:53] +[titan] 2025-07-10 09:34:29,009 - root - INFO - step: 90780 loss: 16.3567 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4997 global_avg_mtp_loss: 13.8570 +[titan] 2025-07-10 09:34:29,009 - root - INFO - lr: 3.5737e-05 gnorm: 1.08 [19:59:25< 2:01:49] +[titan] 2025-07-10 09:34:32,911 - root - INFO - step: 90785 loss: 16.1926 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7159 +[titan] 2025-07-10 09:34:32,911 - root - INFO - lr: 3.5731e-05 gnorm: 1.05 [19:59:29< 2:01:45] +[titan] 2025-07-10 09:34:36,805 - root - INFO - step: 90790 loss: 15.8268 memory: 44.58GiB(31.99%) tps: 84,145 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.4161 global_avg_mtp_loss: 13.4107 +[titan] 2025-07-10 09:34:36,806 - root - INFO - lr: 3.5725e-05 gnorm: 1.09 [19:59:33< 2:01:41] +[titan] 2025-07-10 09:34:40,745 - root - INFO - step: 90795 loss: 16.2773 memory: 44.58GiB(31.99%) tps: 83,180 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4888 global_avg_mtp_loss: 13.7884 +[titan] 2025-07-10 09:34:40,745 - root - INFO - lr: 3.5718e-05 gnorm: 1.00 [19:59:36< 2:01:37] +[titan] 2025-07-10 09:34:43,878 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:34:44,673 - root - INFO - step: 90800 loss: 16.1198 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6632 +[titan] 2025-07-10 09:34:44,673 - root - INFO - lr: 3.5712e-05 gnorm: 1.01 [19:59:40< 2:01:33] +[titan] 2025-07-10 09:34:48,592 - root - INFO - step: 90805 loss: 16.3893 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8897 +[titan] 2025-07-10 09:34:48,593 - root - INFO - lr: 3.5706e-05 gnorm: 1.03 [19:59:44< 2:01:29] +[titan] 2025-07-10 09:34:52,513 - root - INFO - step: 90810 loss: 16.1524 memory: 44.58GiB(31.99%) tps: 83,592 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6916 +[titan] 2025-07-10 09:34:52,513 - root - INFO - lr: 3.5700e-05 gnorm: 1.00 [19:59:48< 2:01:25] +[titan] 2025-07-10 09:34:56,415 - root - INFO - step: 90815 loss: 16.2156 memory: 44.58GiB(31.99%) tps: 83,973 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7326 +[titan] 2025-07-10 09:34:56,416 - root - INFO - lr: 3.5694e-05 gnorm: 1.05 [19:59:52< 2:01:21] +[titan] 2025-07-10 09:35:00,331 - root - INFO - step: 90820 loss: 15.8430 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4072 global_avg_mtp_loss: 13.4358 +[titan] 2025-07-10 09:35:00,332 - root - INFO - lr: 3.5688e-05 gnorm: 1.00 [19:59:56< 2:01:17] +[titan] 2025-07-10 09:35:04,237 - root - INFO - step: 90825 loss: 16.1357 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6741 +[titan] 2025-07-10 09:35:04,238 - root - INFO - lr: 3.5682e-05 gnorm: 0.98 [20:00:00< 2:01:13] +[titan] 2025-07-10 09:35:08,163 - root - INFO - step: 90830 loss: 16.1407 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4599 global_avg_mtp_loss: 13.6808 +[titan] 2025-07-10 09:35:08,164 - root - INFO - lr: 3.5675e-05 gnorm: 1.00 [20:00:04< 2:01:09] +[titan] 2025-07-10 09:35:12,103 - root - INFO - step: 90835 loss: 16.1401 memory: 44.58GiB(31.99%) tps: 83,186 tflops: 287.09 mfu: 29.03% global_avg_ntp_loss: 2.4710 global_avg_mtp_loss: 13.6691 +[titan] 2025-07-10 09:35:12,103 - root - INFO - lr: 3.5669e-05 gnorm: 1.03 [20:00:08< 2:01:05] +[titan] 2025-07-10 09:35:16,002 - root - INFO - step: 90840 loss: 16.1743 memory: 44.58GiB(31.99%) tps: 84,061 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4735 global_avg_mtp_loss: 13.7008 +[titan] 2025-07-10 09:35:16,002 - root - INFO - lr: 3.5663e-05 gnorm: 1.05 [20:00:12< 2:01:01] +[titan] 2025-07-10 09:35:19,912 - root - INFO - step: 90845 loss: 16.0108 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5706 +[titan] 2025-07-10 09:35:19,912 - root - INFO - lr: 3.5657e-05 gnorm: 1.00 [20:00:16< 2:00:57] +[titan] 2025-07-10 09:35:23,028 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:35:23,827 - root - INFO - step: 90850 loss: 15.8627 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4022 global_avg_mtp_loss: 13.4605 +[titan] 2025-07-10 09:35:23,827 - root - INFO - lr: 3.5651e-05 gnorm: 1.03 [20:00:20< 2:00:53] +[titan] 2025-07-10 09:35:27,733 - root - INFO - step: 90855 loss: 15.9599 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.5287 +[titan] 2025-07-10 09:35:27,734 - root - INFO - lr: 3.5645e-05 gnorm: 1.02 [20:00:23< 2:00:49] +[titan] 2025-07-10 09:35:31,652 - root - INFO - step: 90860 loss: 16.1932 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.7319 +[titan] 2025-07-10 09:35:31,652 - root - INFO - lr: 3.5639e-05 gnorm: 0.99 [20:00:27< 2:00:45] +[titan] 2025-07-10 09:35:35,567 - root - INFO - step: 90865 loss: 16.0664 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6131 +[titan] 2025-07-10 09:35:35,568 - root - INFO - lr: 3.5632e-05 gnorm: 1.00 [20:00:31< 2:00:41] +[titan] 2025-07-10 09:35:39,509 - root - INFO - step: 90870 loss: 16.2749 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7969 +[titan] 2025-07-10 09:35:39,509 - root - INFO - lr: 3.5626e-05 gnorm: 1.00 [20:00:35< 2:00:37] +[titan] 2025-07-10 09:35:43,426 - root - INFO - step: 90875 loss: 16.3089 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.8207 +[titan] 2025-07-10 09:35:43,426 - root - INFO - lr: 3.5620e-05 gnorm: 1.05 [20:00:39< 2:00:33] +[titan] 2025-07-10 09:35:47,341 - root - INFO - step: 90880 loss: 16.3904 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.5022 global_avg_mtp_loss: 13.8883 +[titan] 2025-07-10 09:35:47,342 - root - INFO - lr: 3.5614e-05 gnorm: 1.05 [20:00:43< 2:00:29] +[titan] 2025-07-10 09:35:51,249 - root - INFO - step: 90885 loss: 16.4350 memory: 44.58GiB(31.99%) tps: 83,868 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.9145 +[titan] 2025-07-10 09:35:51,249 - root - INFO - lr: 3.5608e-05 gnorm: 0.99 [20:00:47< 2:00:25] +[titan] 2025-07-10 09:35:55,175 - root - INFO - step: 90890 loss: 16.2601 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 09:35:55,175 - root - INFO - lr: 3.5602e-05 gnorm: 1.00 [20:00:51< 2:00:21] +[titan] 2025-07-10 09:35:59,119 - root - INFO - step: 90895 loss: 16.1404 memory: 44.58GiB(31.99%) tps: 83,093 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4510 global_avg_mtp_loss: 13.6894 +[titan] 2025-07-10 09:35:59,119 - root - INFO - lr: 3.5596e-05 gnorm: 1.05 [20:00:55< 2:00:17] +[titan] 2025-07-10 09:36:02,253 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:36:03,043 - root - INFO - step: 90900 loss: 16.3764 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.8588 +[titan] 2025-07-10 09:36:03,044 - root - INFO - lr: 3.5590e-05 gnorm: 1.02 [20:00:59< 2:00:13] +[titan] 2025-07-10 09:36:06,953 - root - INFO - step: 90905 loss: 16.2539 memory: 44.58GiB(31.99%) tps: 83,819 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7735 +[titan] 2025-07-10 09:36:06,953 - root - INFO - lr: 3.5584e-05 gnorm: 1.03 [20:01:03< 2:00:09] +[titan] 2025-07-10 09:36:10,889 - root - INFO - step: 90910 loss: 16.1667 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.7000 +[titan] 2025-07-10 09:36:10,889 - root - INFO - lr: 3.5577e-05 gnorm: 0.97 [20:01:07< 2:00:05] +[titan] 2025-07-10 09:36:14,838 - root - INFO - step: 90915 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 82,982 tflops: 286.38 mfu: 28.96% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7998 +[titan] 2025-07-10 09:36:14,838 - root - INFO - lr: 3.5571e-05 gnorm: 1.00 [20:01:11< 2:00:01] +[titan] 2025-07-10 09:36:18,789 - root - INFO - step: 90920 loss: 16.0854 memory: 44.58GiB(31.99%) tps: 82,939 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4471 global_avg_mtp_loss: 13.6383 +[titan] 2025-07-10 09:36:18,790 - root - INFO - lr: 3.5565e-05 gnorm: 1.03 [20:01:14< 1:59:57] +[titan] 2025-07-10 09:36:22,695 - root - INFO - step: 90925 loss: 16.4874 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9488 +[titan] 2025-07-10 09:36:22,695 - root - INFO - lr: 3.5559e-05 gnorm: 1.01 [20:01:18< 1:59:54] +[titan] 2025-07-10 09:36:26,650 - root - INFO - step: 90930 loss: 16.5491 memory: 44.58GiB(31.99%) tps: 82,864 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.5572 global_avg_mtp_loss: 13.9919 +[titan] 2025-07-10 09:36:26,650 - root - INFO - lr: 3.5553e-05 gnorm: 1.06 [20:01:22< 1:59:50] +[titan] 2025-07-10 09:36:30,613 - root - INFO - step: 90935 loss: 16.4376 memory: 44.58GiB(31.99%) tps: 82,685 tflops: 285.36 mfu: 28.85% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9247 +[titan] 2025-07-10 09:36:30,613 - root - INFO - lr: 3.5547e-05 gnorm: 1.00 [20:01:26< 1:59:46] +[titan] 2025-07-10 09:36:34,550 - root - INFO - step: 90940 loss: 16.1724 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.6980 +[titan] 2025-07-10 09:36:34,550 - root - INFO - lr: 3.5541e-05 gnorm: 1.05 [20:01:30< 1:59:42] +[titan] 2025-07-10 09:36:38,470 - root - INFO - step: 90945 loss: 16.3700 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.8726 +[titan] 2025-07-10 09:36:38,470 - root - INFO - lr: 3.5535e-05 gnorm: 1.05 [20:01:34< 1:59:38] +[titan] 2025-07-10 09:36:41,620 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:36:42,421 - root - INFO - step: 90950 loss: 16.2823 memory: 44.58GiB(31.99%) tps: 82,955 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.7746 +[titan] 2025-07-10 09:36:42,421 - root - INFO - lr: 3.5529e-05 gnorm: 1.08 [20:01:38< 1:59:34] +[titan] 2025-07-10 09:36:46,329 - root - INFO - step: 90955 loss: 16.4022 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.8810 +[titan] 2025-07-10 09:36:46,329 - root - INFO - lr: 3.5523e-05 gnorm: 1.18 [20:01:42< 1:59:30] +[titan] 2025-07-10 09:36:50,260 - root - INFO - step: 90960 loss: 16.2727 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.5019 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 09:36:50,260 - root - INFO - lr: 3.5517e-05 gnorm: 1.07 [20:01:46< 1:59:26] +[titan] 2025-07-10 09:36:54,187 - root - INFO - step: 90965 loss: 16.4372 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.9296 +[titan] 2025-07-10 09:36:54,187 - root - INFO - lr: 3.5511e-05 gnorm: 1.05 [20:01:50< 1:59:22] +[titan] 2025-07-10 09:36:58,112 - root - INFO - step: 90970 loss: 16.0768 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4522 global_avg_mtp_loss: 13.6246 +[titan] 2025-07-10 09:36:58,112 - root - INFO - lr: 3.5505e-05 gnorm: 1.02 [20:01:54< 1:59:18] +[titan] 2025-07-10 09:37:02,047 - root - INFO - step: 90975 loss: 15.8562 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4049 global_avg_mtp_loss: 13.4513 +[titan] 2025-07-10 09:37:02,048 - root - INFO - lr: 3.5499e-05 gnorm: 1.01 [20:01:58< 1:59:14] +[titan] 2025-07-10 09:37:05,989 - root - INFO - step: 90980 loss: 16.1088 memory: 44.58GiB(31.99%) tps: 83,139 tflops: 286.93 mfu: 29.01% global_avg_ntp_loss: 2.4546 global_avg_mtp_loss: 13.6541 +[titan] 2025-07-10 09:37:05,989 - root - INFO - lr: 3.5493e-05 gnorm: 1.00 [20:02:02< 1:59:10] +[titan] 2025-07-10 09:37:09,898 - root - INFO - step: 90985 loss: 15.9212 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4199 global_avg_mtp_loss: 13.5013 +[titan] 2025-07-10 09:37:09,898 - root - INFO - lr: 3.5486e-05 gnorm: 1.03 [20:02:06< 1:59:06] +[titan] 2025-07-10 09:37:13,805 - root - INFO - step: 90990 loss: 16.2327 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7526 +[titan] 2025-07-10 09:37:13,805 - root - INFO - lr: 3.5480e-05 gnorm: 1.07 [20:02:09< 1:59:02] +[titan] 2025-07-10 09:37:17,748 - root - INFO - step: 90995 loss: 16.4494 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.9256 +[titan] 2025-07-10 09:37:17,748 - root - INFO - lr: 3.5474e-05 gnorm: 1.00 [20:02:13< 1:58:58] +[titan] 2025-07-10 09:37:20,913 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:37:21,711 - root - INFO - step: 91000 loss: 16.2143 memory: 44.58GiB(31.99%) tps: 82,690 tflops: 285.38 mfu: 28.85% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7313 +[titan] 2025-07-10 09:37:21,711 - root - INFO - lr: 3.5468e-05 gnorm: 1.02 [20:02:17< 1:58:54] +[titan] 2025-07-10 09:37:25,667 - root - INFO - step: 91005 loss: 16.2879 memory: 44.58GiB(31.99%) tps: 82,840 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.8071 +[titan] 2025-07-10 09:37:25,667 - root - INFO - lr: 3.5462e-05 gnorm: 1.10 [20:02:21< 1:58:50] +[titan] 2025-07-10 09:37:29,591 - root - INFO - step: 91010 loss: 16.0708 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4532 global_avg_mtp_loss: 13.6176 +[titan] 2025-07-10 09:37:29,591 - root - INFO - lr: 3.5456e-05 gnorm: 1.02 [20:02:25< 1:58:46] +[titan] 2025-07-10 09:37:33,522 - root - INFO - step: 91015 loss: 16.3064 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.8112 +[titan] 2025-07-10 09:37:33,523 - root - INFO - lr: 3.5450e-05 gnorm: 1.05 [20:02:29< 1:58:42] +[titan] 2025-07-10 09:37:37,466 - root - INFO - step: 91020 loss: 16.1775 memory: 44.58GiB(31.99%) tps: 83,094 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7055 +[titan] 2025-07-10 09:37:37,467 - root - INFO - lr: 3.5444e-05 gnorm: 1.05 [20:02:33< 1:58:38] +[titan] 2025-07-10 09:37:41,386 - root - INFO - step: 91025 loss: 16.2282 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4808 global_avg_mtp_loss: 13.7474 +[titan] 2025-07-10 09:37:41,386 - root - INFO - lr: 3.5438e-05 gnorm: 0.99 [20:02:37< 1:58:34] +[titan] 2025-07-10 09:37:45,302 - root - INFO - step: 91030 loss: 16.3604 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5073 global_avg_mtp_loss: 13.8531 +[titan] 2025-07-10 09:37:45,302 - root - INFO - lr: 3.5432e-05 gnorm: 0.98 [20:02:41< 1:58:30] +[titan] 2025-07-10 09:37:49,210 - root - INFO - step: 91035 loss: 16.2921 memory: 44.58GiB(31.99%) tps: 83,853 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.8127 +[titan] 2025-07-10 09:37:49,211 - root - INFO - lr: 3.5426e-05 gnorm: 0.98 [20:02:45< 1:58:26] +[titan] 2025-07-10 09:37:53,132 - root - INFO - step: 91040 loss: 16.1428 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.6668 +[titan] 2025-07-10 09:37:53,132 - root - INFO - lr: 3.5420e-05 gnorm: 1.01 [20:02:49< 1:58:22] +[titan] 2025-07-10 09:37:57,065 - root - INFO - step: 91045 loss: 16.1983 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7263 +[titan] 2025-07-10 09:37:57,065 - root - INFO - lr: 3.5414e-05 gnorm: 1.08 [20:02:53< 1:58:18] +[titan] 2025-07-10 09:38:00,182 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:38:00,975 - root - INFO - step: 91050 loss: 16.2147 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7368 +[titan] 2025-07-10 09:38:00,975 - root - INFO - lr: 3.5408e-05 gnorm: 1.01 [20:02:57< 1:58:14] +[titan] 2025-07-10 09:38:04,879 - root - INFO - step: 91055 loss: 16.1762 memory: 44.58GiB(31.99%) tps: 83,938 tflops: 289.68 mfu: 29.29% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.7112 +[titan] 2025-07-10 09:38:04,880 - root - INFO - lr: 3.5402e-05 gnorm: 1.03 [20:03:01< 1:58:10] +[titan] 2025-07-10 09:38:08,791 - root - INFO - step: 91060 loss: 15.9670 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4314 global_avg_mtp_loss: 13.5356 +[titan] 2025-07-10 09:38:08,791 - root - INFO - lr: 3.5396e-05 gnorm: 1.03 [20:03:04< 1:58:06] +[titan] 2025-07-10 09:38:12,722 - root - INFO - step: 91065 loss: 16.0860 memory: 44.58GiB(31.99%) tps: 83,363 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.6352 +[titan] 2025-07-10 09:38:12,723 - root - INFO - lr: 3.5390e-05 gnorm: 1.00 [20:03:08< 1:58:02] +[titan] 2025-07-10 09:38:16,633 - root - INFO - step: 91070 loss: 16.2077 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7218 +[titan] 2025-07-10 09:38:16,634 - root - INFO - lr: 3.5384e-05 gnorm: 1.03 [20:03:12< 1:57:58] +[titan] 2025-07-10 09:38:20,560 - root - INFO - step: 91075 loss: 16.1868 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7151 +[titan] 2025-07-10 09:38:20,561 - root - INFO - lr: 3.5378e-05 gnorm: 1.01 [20:03:16< 1:57:55] +[titan] 2025-07-10 09:38:24,456 - root - INFO - step: 91080 loss: 16.0876 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6234 +[titan] 2025-07-10 09:38:24,457 - root - INFO - lr: 3.5372e-05 gnorm: 1.07 [20:03:20< 1:57:51] +[titan] 2025-07-10 09:38:28,396 - root - INFO - step: 91085 loss: 16.4625 memory: 44.58GiB(31.99%) tps: 83,177 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.5338 global_avg_mtp_loss: 13.9287 +[titan] 2025-07-10 09:38:28,397 - root - INFO - lr: 3.5366e-05 gnorm: 1.02 [20:03:24< 1:57:47] +[titan] 2025-07-10 09:38:32,354 - root - INFO - step: 91090 loss: 16.0479 memory: 44.58GiB(31.99%) tps: 82,800 tflops: 285.76 mfu: 28.89% global_avg_ntp_loss: 2.4525 global_avg_mtp_loss: 13.5954 +[titan] 2025-07-10 09:38:32,354 - root - INFO - lr: 3.5360e-05 gnorm: 0.98 [20:03:28< 1:57:43] +[titan] 2025-07-10 09:38:36,252 - root - INFO - step: 91095 loss: 15.8061 memory: 44.58GiB(31.99%) tps: 84,067 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4030 global_avg_mtp_loss: 13.4031 +[titan] 2025-07-10 09:38:36,253 - root - INFO - lr: 3.5354e-05 gnorm: 1.06 [20:03:32< 1:57:39] +[titan] 2025-07-10 09:38:39,372 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:38:40,163 - root - INFO - step: 91100 loss: 16.1656 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.6901 +[titan] 2025-07-10 09:38:40,163 - root - INFO - lr: 3.5348e-05 gnorm: 1.01 [20:03:36< 1:57:35] +[titan] 2025-07-10 09:38:44,152 - root - INFO - step: 91105 loss: 16.3612 memory: 44.58GiB(31.99%) tps: 82,154 tflops: 283.53 mfu: 28.67% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.8490 +[titan] 2025-07-10 09:38:44,152 - root - INFO - lr: 3.5342e-05 gnorm: 1.04 [20:03:40< 1:57:31] +[titan] 2025-07-10 09:38:48,067 - root - INFO - step: 91110 loss: 16.2658 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7918 +[titan] 2025-07-10 09:38:48,067 - root - INFO - lr: 3.5336e-05 gnorm: 1.01 [20:03:44< 1:57:27] +[titan] 2025-07-10 09:38:51,990 - root - INFO - step: 91115 loss: 16.0625 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4382 global_avg_mtp_loss: 13.6243 +[titan] 2025-07-10 09:38:51,991 - root - INFO - lr: 3.5330e-05 gnorm: 1.06 [20:03:48< 1:57:23] +[titan] 2025-07-10 09:38:55,947 - root - INFO - step: 91120 loss: 16.1734 memory: 44.58GiB(31.99%) tps: 82,827 tflops: 285.85 mfu: 28.90% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.7115 +[titan] 2025-07-10 09:38:55,947 - root - INFO - lr: 3.5324e-05 gnorm: 1.02 [20:03:52< 1:57:19] +[titan] 2025-07-10 09:38:59,897 - root - INFO - step: 91125 loss: 16.1465 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.6794 +[titan] 2025-07-10 09:38:59,898 - root - INFO - lr: 3.5318e-05 gnorm: 0.99 [20:03:56< 1:57:15] +[titan] 2025-07-10 09:39:03,818 - root - INFO - step: 91130 loss: 16.3021 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.8200 +[titan] 2025-07-10 09:39:03,818 - root - INFO - lr: 3.5313e-05 gnorm: 0.99 [20:03:59< 1:57:11] +[titan] 2025-07-10 09:39:07,815 - root - INFO - step: 91135 loss: 16.1993 memory: 44.58GiB(31.99%) tps: 81,992 tflops: 282.97 mfu: 28.61% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.7222 +[titan] 2025-07-10 09:39:07,815 - root - INFO - lr: 3.5307e-05 gnorm: 1.03 [20:04:03< 1:57:07] +[titan] 2025-07-10 09:39:08,754 - root - INFO - Dumping profiler traces at step 91136 +[titan] 2025-07-10 09:39:08,787 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:39:11,930 - root - INFO - step: 91140 loss: 16.5019 memory: 44.58GiB(31.99%) tps: 79,647 tflops: 274.88 mfu: 27.79% global_avg_ntp_loss: 2.5368 global_avg_mtp_loss: 13.9651 +[titan] 2025-07-10 09:39:11,930 - root - INFO - lr: 3.5301e-05 gnorm: 1.03 [20:04:08< 1:57:03] +[titan] 2025-07-10 09:39:15,858 - root - INFO - step: 91145 loss: 16.4164 memory: 44.58GiB(31.99%) tps: 83,420 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5059 global_avg_mtp_loss: 13.9105 +[titan] 2025-07-10 09:39:15,858 - root - INFO - lr: 3.5295e-05 gnorm: 1.00 [20:04:12< 1:56:59] +[titan] 2025-07-10 09:39:19,005 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:39:19,793 - root - INFO - step: 91150 loss: 16.2683 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4782 global_avg_mtp_loss: 13.7901 +[titan] 2025-07-10 09:39:19,793 - root - INFO - lr: 3.5289e-05 gnorm: 1.00 [20:04:15< 1:56:55] +[titan] 2025-07-10 09:39:23,692 - root - INFO - step: 91155 loss: 16.1182 memory: 44.58GiB(31.99%) tps: 84,056 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4506 global_avg_mtp_loss: 13.6675 +[titan] 2025-07-10 09:39:23,692 - root - INFO - lr: 3.5283e-05 gnorm: 1.00 [20:04:19< 1:56:51] +[titan] 2025-07-10 09:39:27,616 - root - INFO - step: 91160 loss: 15.8650 memory: 44.58GiB(31.99%) tps: 83,507 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4251 global_avg_mtp_loss: 13.4398 +[titan] 2025-07-10 09:39:27,616 - root - INFO - lr: 3.5277e-05 gnorm: 1.03 [20:04:23< 1:56:47] +[titan] 2025-07-10 09:39:31,535 - root - INFO - step: 91165 loss: 16.0072 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4370 global_avg_mtp_loss: 13.5702 +[titan] 2025-07-10 09:39:31,535 - root - INFO - lr: 3.5271e-05 gnorm: 1.02 [20:04:27< 1:56:43] +[titan] 2025-07-10 09:39:35,437 - root - INFO - step: 91170 loss: 16.1270 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6625 +[titan] 2025-07-10 09:39:35,437 - root - INFO - lr: 3.5265e-05 gnorm: 1.08 [20:04:31< 1:56:39] +[titan] 2025-07-10 09:39:39,340 - root - INFO - step: 91175 loss: 16.2603 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7835 +[titan] 2025-07-10 09:39:39,340 - root - INFO - lr: 3.5259e-05 gnorm: 1.06 [20:04:35< 1:56:35] +[titan] 2025-07-10 09:39:43,281 - root - INFO - step: 91180 loss: 16.1868 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7182 +[titan] 2025-07-10 09:39:43,282 - root - INFO - lr: 3.5253e-05 gnorm: 1.10 [20:04:39< 1:56:31] +[titan] 2025-07-10 09:39:47,182 - root - INFO - step: 91185 loss: 16.3530 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8513 +[titan] 2025-07-10 09:39:47,182 - root - INFO - lr: 3.5247e-05 gnorm: 1.01 [20:04:43< 1:56:27] +[titan] 2025-07-10 09:39:51,080 - root - INFO - step: 91190 loss: 16.1343 memory: 44.58GiB(31.99%) tps: 84,059 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4557 global_avg_mtp_loss: 13.6785 +[titan] 2025-07-10 09:39:51,081 - root - INFO - lr: 3.5241e-05 gnorm: 1.04 [20:04:47< 1:56:23] +[titan] 2025-07-10 09:39:55,007 - root - INFO - step: 91195 loss: 15.9547 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4266 global_avg_mtp_loss: 13.5281 +[titan] 2025-07-10 09:39:55,008 - root - INFO - lr: 3.5235e-05 gnorm: 1.06 [20:04:51< 1:56:19] +[titan] 2025-07-10 09:39:58,115 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:39:58,907 - root - INFO - step: 91200 loss: 16.3536 memory: 44.58GiB(31.99%) tps: 84,038 tflops: 290.03 mfu: 29.33% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.8540 +[titan] 2025-07-10 09:39:58,907 - root - INFO - lr: 3.5230e-05 gnorm: 1.04 [20:04:55< 1:56:15] +[titan] 2025-07-10 09:40:02,811 - root - INFO - step: 91205 loss: 16.2048 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.7279 +[titan] 2025-07-10 09:40:02,811 - root - INFO - lr: 3.5224e-05 gnorm: 1.03 [20:04:58< 1:56:11] +[titan] 2025-07-10 09:40:06,741 - root - INFO - step: 91210 loss: 16.1861 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.7005 +[titan] 2025-07-10 09:40:06,741 - root - INFO - lr: 3.5218e-05 gnorm: 1.08 [20:05:02< 1:56:07] +[titan] 2025-07-10 09:40:10,667 - root - INFO - step: 91215 loss: 16.1547 memory: 44.58GiB(31.99%) tps: 83,457 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6929 +[titan] 2025-07-10 09:40:10,668 - root - INFO - lr: 3.5212e-05 gnorm: 1.03 [20:05:06< 1:56:03] +[titan] 2025-07-10 09:40:14,598 - root - INFO - step: 91220 loss: 16.1617 memory: 44.58GiB(31.99%) tps: 83,381 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.7019 +[titan] 2025-07-10 09:40:14,598 - root - INFO - lr: 3.5206e-05 gnorm: 1.05 [20:05:10< 1:55:59] +[titan] 2025-07-10 09:40:18,497 - root - INFO - step: 91225 loss: 16.3060 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.8206 +[titan] 2025-07-10 09:40:18,497 - root - INFO - lr: 3.5200e-05 gnorm: 1.02 [20:05:14< 1:55:56] +[titan] 2025-07-10 09:40:22,463 - root - INFO - step: 91230 loss: 15.9798 memory: 44.58GiB(31.99%) tps: 82,619 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.5237 +[titan] 2025-07-10 09:40:22,464 - root - INFO - lr: 3.5194e-05 gnorm: 1.04 [20:05:18< 1:55:52] +[titan] 2025-07-10 09:40:26,377 - root - INFO - step: 91235 loss: 15.8608 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4139 global_avg_mtp_loss: 13.4469 +[titan] 2025-07-10 09:40:26,377 - root - INFO - lr: 3.5188e-05 gnorm: 1.03 [20:05:22< 1:55:48] +[titan] 2025-07-10 09:40:30,314 - root - INFO - step: 91240 loss: 15.9919 memory: 44.58GiB(31.99%) tps: 83,243 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4300 global_avg_mtp_loss: 13.5619 +[titan] 2025-07-10 09:40:30,314 - root - INFO - lr: 3.5182e-05 gnorm: 1.03 [20:05:26< 1:55:44] +[titan] 2025-07-10 09:40:34,233 - root - INFO - step: 91245 loss: 16.0588 memory: 44.58GiB(31.99%) tps: 83,617 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.6039 +[titan] 2025-07-10 09:40:34,233 - root - INFO - lr: 3.5177e-05 gnorm: 0.96 [20:05:30< 1:55:40] +[titan] 2025-07-10 09:40:37,399 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:40:38,209 - root - INFO - step: 91250 loss: 16.4403 memory: 44.58GiB(31.99%) tps: 82,410 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.5080 global_avg_mtp_loss: 13.9322 +[titan] 2025-07-10 09:40:38,210 - root - INFO - lr: 3.5171e-05 gnorm: 1.02 [20:05:34< 1:55:36] +[titan] 2025-07-10 09:40:42,106 - root - INFO - step: 91255 loss: 16.0925 memory: 44.58GiB(31.99%) tps: 84,092 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.6281 +[titan] 2025-07-10 09:40:42,107 - root - INFO - lr: 3.5165e-05 gnorm: 1.04 [20:05:38< 1:55:32] +[titan] 2025-07-10 09:40:46,047 - root - INFO - step: 91260 loss: 16.1299 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.6881 +[titan] 2025-07-10 09:40:46,047 - root - INFO - lr: 3.5159e-05 gnorm: 1.00 [20:05:42< 1:55:28] +[titan] 2025-07-10 09:40:49,965 - root - INFO - step: 91265 loss: 16.0986 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4473 global_avg_mtp_loss: 13.6512 +[titan] 2025-07-10 09:40:49,965 - root - INFO - lr: 3.5153e-05 gnorm: 1.06 [20:05:46< 1:55:24] +[titan] 2025-07-10 09:40:53,898 - root - INFO - step: 91270 loss: 16.1695 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.6959 +[titan] 2025-07-10 09:40:53,899 - root - INFO - lr: 3.5147e-05 gnorm: 1.01 [20:05:50< 1:55:20] +[titan] 2025-07-10 09:40:57,835 - root - INFO - step: 91275 loss: 16.3696 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4990 global_avg_mtp_loss: 13.8706 +[titan] 2025-07-10 09:40:57,836 - root - INFO - lr: 3.5141e-05 gnorm: 0.98 [20:05:54< 1:55:16] +[titan] 2025-07-10 09:41:01,762 - root - INFO - step: 91280 loss: 16.2685 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.7655 +[titan] 2025-07-10 09:41:01,762 - root - INFO - lr: 3.5136e-05 gnorm: 1.03 [20:05:57< 1:55:12] +[titan] 2025-07-10 09:41:05,694 - root - INFO - step: 91285 loss: 16.4691 memory: 44.58GiB(31.99%) tps: 83,352 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5086 global_avg_mtp_loss: 13.9604 +[titan] 2025-07-10 09:41:05,694 - root - INFO - lr: 3.5130e-05 gnorm: 1.07 [20:06:01< 1:55:08] +[titan] 2025-07-10 09:41:09,620 - root - INFO - step: 91290 loss: 16.2926 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8011 +[titan] 2025-07-10 09:41:09,620 - root - INFO - lr: 3.5124e-05 gnorm: 1.13 [20:06:05< 1:55:04] +[titan] 2025-07-10 09:41:13,569 - root - INFO - step: 91295 loss: 15.9297 memory: 44.58GiB(31.99%) tps: 82,983 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4325 global_avg_mtp_loss: 13.4972 +[titan] 2025-07-10 09:41:13,569 - root - INFO - lr: 3.5118e-05 gnorm: 1.04 [20:06:09< 1:55:00] +[titan] 2025-07-10 09:41:16,704 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:41:17,491 - root - INFO - step: 91300 loss: 16.1640 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6961 +[titan] 2025-07-10 09:41:17,491 - root - INFO - lr: 3.5112e-05 gnorm: 1.07 [20:06:13< 1:54:56] +[titan] 2025-07-10 09:41:21,393 - root - INFO - step: 91305 loss: 15.7333 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.3981 global_avg_mtp_loss: 13.3352 +[titan] 2025-07-10 09:41:21,393 - root - INFO - lr: 3.5106e-05 gnorm: 1.14 [20:06:17< 1:54:52] +[titan] 2025-07-10 09:41:25,308 - root - INFO - step: 91310 loss: 16.1416 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6768 +[titan] 2025-07-10 09:41:25,308 - root - INFO - lr: 3.5100e-05 gnorm: 1.08 [20:06:21< 1:54:48] +[titan] 2025-07-10 09:41:29,296 - root - INFO - step: 91315 loss: 16.0321 memory: 44.58GiB(31.99%) tps: 82,174 tflops: 283.60 mfu: 28.68% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.5897 +[titan] 2025-07-10 09:41:29,297 - root - INFO - lr: 3.5095e-05 gnorm: 0.98 [20:06:25< 1:54:44] +[titan] 2025-07-10 09:41:33,194 - root - INFO - step: 91320 loss: 16.1524 memory: 44.58GiB(31.99%) tps: 84,076 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.6891 +[titan] 2025-07-10 09:41:33,194 - root - INFO - lr: 3.5089e-05 gnorm: 1.02 [20:06:29< 1:54:40] +[titan] 2025-07-10 09:41:37,095 - root - INFO - step: 91325 loss: 16.2583 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7798 +[titan] 2025-07-10 09:41:37,096 - root - INFO - lr: 3.5083e-05 gnorm: 1.03 [20:06:33< 1:54:36] +[titan] 2025-07-10 09:41:41,083 - root - INFO - step: 91330 loss: 16.2494 memory: 44.58GiB(31.99%) tps: 82,187 tflops: 283.64 mfu: 28.68% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7839 +[titan] 2025-07-10 09:41:41,083 - root - INFO - lr: 3.5077e-05 gnorm: 1.05 [20:06:37< 1:54:32] +[titan] 2025-07-10 09:41:44,999 - root - INFO - step: 91335 loss: 16.2728 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7986 +[titan] 2025-07-10 09:41:44,999 - root - INFO - lr: 3.5071e-05 gnorm: 1.02 [20:06:41< 1:54:28] +[titan] 2025-07-10 09:41:48,919 - root - INFO - step: 91340 loss: 16.0412 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5977 +[titan] 2025-07-10 09:41:48,920 - root - INFO - lr: 3.5066e-05 gnorm: 1.02 [20:06:45< 1:54:24] +[titan] 2025-07-10 09:41:52,812 - root - INFO - step: 91345 loss: 15.9793 memory: 44.58GiB(31.99%) tps: 84,192 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.4371 global_avg_mtp_loss: 13.5422 +[titan] 2025-07-10 09:41:52,812 - root - INFO - lr: 3.5060e-05 gnorm: 1.03 [20:06:48< 1:54:20] +[titan] 2025-07-10 09:41:55,930 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:41:56,728 - root - INFO - step: 91350 loss: 15.9455 memory: 44.58GiB(31.99%) tps: 83,672 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4281 global_avg_mtp_loss: 13.5174 +[titan] 2025-07-10 09:41:56,729 - root - INFO - lr: 3.5054e-05 gnorm: 1.04 [20:06:52< 1:54:16] +[titan] 2025-07-10 09:42:00,650 - root - INFO - step: 91355 loss: 15.9476 memory: 44.58GiB(31.99%) tps: 83,560 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4282 global_avg_mtp_loss: 13.5194 +[titan] 2025-07-10 09:42:00,651 - root - INFO - lr: 3.5048e-05 gnorm: 1.04 [20:06:56< 1:54:12] +[titan] 2025-07-10 09:42:04,588 - root - INFO - step: 91360 loss: 16.1865 memory: 44.58GiB(31.99%) tps: 83,227 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4546 global_avg_mtp_loss: 13.7320 +[titan] 2025-07-10 09:42:04,588 - root - INFO - lr: 3.5042e-05 gnorm: 1.07 [20:07:00< 1:54:08] +[titan] 2025-07-10 09:42:08,552 - root - INFO - step: 91365 loss: 16.4450 memory: 44.58GiB(31.99%) tps: 82,665 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.9367 +[titan] 2025-07-10 09:42:08,553 - root - INFO - lr: 3.5037e-05 gnorm: 1.01 [20:07:04< 1:54:04] +[titan] 2025-07-10 09:42:12,491 - root - INFO - step: 91370 loss: 16.4709 memory: 44.58GiB(31.99%) tps: 83,199 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.5261 global_avg_mtp_loss: 13.9448 +[titan] 2025-07-10 09:42:12,492 - root - INFO - lr: 3.5031e-05 gnorm: 1.02 [20:07:08< 1:54:00] +[titan] 2025-07-10 09:42:16,418 - root - INFO - step: 91375 loss: 16.1151 memory: 44.58GiB(31.99%) tps: 83,464 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6502 +[titan] 2025-07-10 09:42:16,418 - root - INFO - lr: 3.5025e-05 gnorm: 1.05 [20:07:12< 1:53:57] +[titan] 2025-07-10 09:42:20,336 - root - INFO - step: 91380 loss: 16.4072 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.5171 global_avg_mtp_loss: 13.8901 +[titan] 2025-07-10 09:42:20,336 - root - INFO - lr: 3.5019e-05 gnorm: 1.03 [20:07:16< 1:53:53] +[titan] 2025-07-10 09:42:24,262 - root - INFO - step: 91385 loss: 15.8857 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4208 global_avg_mtp_loss: 13.4649 +[titan] 2025-07-10 09:42:24,263 - root - INFO - lr: 3.5013e-05 gnorm: 1.03 [20:07:20< 1:53:49] +[titan] 2025-07-10 09:42:28,192 - root - INFO - step: 91390 loss: 16.3109 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8167 +[titan] 2025-07-10 09:42:28,192 - root - INFO - lr: 3.5008e-05 gnorm: 1.02 [20:07:24< 1:53:45] +[titan] 2025-07-10 09:42:32,111 - root - INFO - step: 91395 loss: 16.1886 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4714 global_avg_mtp_loss: 13.7173 +[titan] 2025-07-10 09:42:32,112 - root - INFO - lr: 3.5002e-05 gnorm: 1.01 [20:07:28< 1:53:41] +[titan] 2025-07-10 09:42:35,238 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:42:36,039 - root - INFO - step: 91400 loss: 16.1496 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.12% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6873 +[titan] 2025-07-10 09:42:36,039 - root - INFO - lr: 3.4996e-05 gnorm: 1.00 [20:07:32< 1:53:37] +[titan] 2025-07-10 09:42:39,965 - root - INFO - step: 91405 loss: 16.0261 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4370 global_avg_mtp_loss: 13.5891 +[titan] 2025-07-10 09:42:39,965 - root - INFO - lr: 3.4990e-05 gnorm: 1.03 [20:07:36< 1:53:33] +[titan] 2025-07-10 09:42:43,883 - root - INFO - step: 91410 loss: 16.2256 memory: 44.58GiB(31.99%) tps: 83,633 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7516 +[titan] 2025-07-10 09:42:43,884 - root - INFO - lr: 3.4984e-05 gnorm: 1.02 [20:07:40< 1:53:29] +[titan] 2025-07-10 09:42:47,805 - root - INFO - step: 91415 loss: 16.0255 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.5690 +[titan] 2025-07-10 09:42:47,806 - root - INFO - lr: 3.4979e-05 gnorm: 1.09 [20:07:43< 1:53:25] +[titan] 2025-07-10 09:42:51,728 - root - INFO - step: 91420 loss: 16.1974 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4674 global_avg_mtp_loss: 13.7300 +[titan] 2025-07-10 09:42:51,728 - root - INFO - lr: 3.4973e-05 gnorm: 1.05 [20:07:47< 1:53:21] +[titan] 2025-07-10 09:42:55,688 - root - INFO - step: 91425 loss: 16.1775 memory: 44.58GiB(31.99%) tps: 82,744 tflops: 285.56 mfu: 28.87% global_avg_ntp_loss: 2.5016 global_avg_mtp_loss: 13.6759 +[titan] 2025-07-10 09:42:55,689 - root - INFO - lr: 3.4967e-05 gnorm: 1.04 [20:07:51< 1:53:17] +[titan] 2025-07-10 09:42:59,638 - root - INFO - step: 91430 loss: 16.4162 memory: 44.58GiB(31.99%) tps: 82,968 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.5079 global_avg_mtp_loss: 13.9084 +[titan] 2025-07-10 09:42:59,639 - root - INFO - lr: 3.4961e-05 gnorm: 0.99 [20:07:55< 1:53:13] +[titan] 2025-07-10 09:43:03,554 - root - INFO - step: 91435 loss: 16.2921 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.8059 +[titan] 2025-07-10 09:43:03,555 - root - INFO - lr: 3.4956e-05 gnorm: 1.04 [20:07:59< 1:53:09] +[titan] 2025-07-10 09:43:07,498 - root - INFO - step: 91440 loss: 16.1881 memory: 44.58GiB(31.99%) tps: 83,098 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4769 global_avg_mtp_loss: 13.7112 +[titan] 2025-07-10 09:43:07,498 - root - INFO - lr: 3.4950e-05 gnorm: 1.05 [20:08:03< 1:53:05] +[titan] 2025-07-10 09:43:11,421 - root - INFO - step: 91445 loss: 15.9596 memory: 44.58GiB(31.99%) tps: 83,542 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4284 global_avg_mtp_loss: 13.5312 +[titan] 2025-07-10 09:43:11,421 - root - INFO - lr: 3.4944e-05 gnorm: 1.03 [20:08:07< 1:53:01] +[titan] 2025-07-10 09:43:14,553 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:43:15,346 - root - INFO - step: 91450 loss: 16.2825 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.7901 +[titan] 2025-07-10 09:43:15,346 - root - INFO - lr: 3.4938e-05 gnorm: 0.98 [20:08:11< 1:52:57] +[titan] 2025-07-10 09:43:19,295 - root - INFO - step: 91455 loss: 16.3509 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8482 +[titan] 2025-07-10 09:43:19,295 - root - INFO - lr: 3.4933e-05 gnorm: 1.01 [20:08:15< 1:52:53] +[titan] 2025-07-10 09:43:23,235 - root - INFO - step: 91460 loss: 16.4813 memory: 44.58GiB(31.99%) tps: 83,188 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5476 global_avg_mtp_loss: 13.9337 +[titan] 2025-07-10 09:43:23,235 - root - INFO - lr: 3.4927e-05 gnorm: 1.05 [20:08:19< 1:52:49] +[titan] 2025-07-10 09:43:27,149 - root - INFO - step: 91465 loss: 16.2601 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7842 +[titan] 2025-07-10 09:43:27,149 - root - INFO - lr: 3.4921e-05 gnorm: 1.06 [20:08:23< 1:52:45] +[titan] 2025-07-10 09:43:31,059 - root - INFO - step: 91470 loss: 16.3133 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8202 +[titan] 2025-07-10 09:43:31,060 - root - INFO - lr: 3.4916e-05 gnorm: 1.05 [20:08:27< 1:52:41] +[titan] 2025-07-10 09:43:34,973 - root - INFO - step: 91475 loss: 16.3147 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4858 global_avg_mtp_loss: 13.8289 +[titan] 2025-07-10 09:43:34,974 - root - INFO - lr: 3.4910e-05 gnorm: 1.04 [20:08:31< 1:52:37] +[titan] 2025-07-10 09:43:38,892 - root - INFO - step: 91480 loss: 16.0045 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5649 +[titan] 2025-07-10 09:43:38,892 - root - INFO - lr: 3.4904e-05 gnorm: 1.00 [20:08:35< 1:52:33] +[titan] 2025-07-10 09:43:42,817 - root - INFO - step: 91485 loss: 15.9669 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4304 global_avg_mtp_loss: 13.5365 +[titan] 2025-07-10 09:43:42,817 - root - INFO - lr: 3.4898e-05 gnorm: 0.97 [20:08:38< 1:52:29] +[titan] 2025-07-10 09:43:46,725 - root - INFO - step: 91490 loss: 16.0587 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.6274 +[titan] 2025-07-10 09:43:46,725 - root - INFO - lr: 3.4893e-05 gnorm: 0.98 [20:08:42< 1:52:25] +[titan] 2025-07-10 09:43:50,626 - root - INFO - step: 91495 loss: 15.9874 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4285 global_avg_mtp_loss: 13.5589 +[titan] 2025-07-10 09:43:50,626 - root - INFO - lr: 3.4887e-05 gnorm: 1.00 [20:08:46< 1:52:21] +[titan] 2025-07-10 09:43:53,753 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:43:54,542 - root - INFO - step: 91500 loss: 16.4378 memory: 44.58GiB(31.99%) tps: 83,683 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.5257 global_avg_mtp_loss: 13.9121 +[titan] 2025-07-10 09:43:54,542 - root - INFO - lr: 3.4881e-05 gnorm: 1.02 [20:08:50< 1:52:17] +[titan] 2025-07-10 09:43:58,457 - root - INFO - step: 91505 loss: 16.0733 memory: 44.58GiB(31.99%) tps: 83,699 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4551 global_avg_mtp_loss: 13.6182 +[titan] 2025-07-10 09:43:58,458 - root - INFO - lr: 3.4876e-05 gnorm: 1.09 [20:08:54< 1:52:13] +[titan] 2025-07-10 09:44:02,363 - root - INFO - step: 91510 loss: 15.9821 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4247 global_avg_mtp_loss: 13.5573 +[titan] 2025-07-10 09:44:02,364 - root - INFO - lr: 3.4870e-05 gnorm: 1.04 [20:08:58< 1:52:09] +[titan] 2025-07-10 09:44:06,311 - root - INFO - step: 91515 loss: 16.0260 memory: 44.58GiB(31.99%) tps: 83,005 tflops: 286.46 mfu: 28.96% global_avg_ntp_loss: 2.4298 global_avg_mtp_loss: 13.5962 +[titan] 2025-07-10 09:44:06,312 - root - INFO - lr: 3.4864e-05 gnorm: 1.05 [20:09:02< 1:52:05] +[titan] 2025-07-10 09:44:10,228 - root - INFO - step: 91520 loss: 15.9914 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.5422 +[titan] 2025-07-10 09:44:10,228 - root - INFO - lr: 3.4858e-05 gnorm: 1.03 [20:09:06< 1:52:01] +[titan] 2025-07-10 09:44:14,144 - root - INFO - step: 91525 loss: 16.2173 memory: 44.58GiB(31.99%) tps: 83,698 tflops: 288.86 mfu: 29.21% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7430 +[titan] 2025-07-10 09:44:14,144 - root - INFO - lr: 3.4853e-05 gnorm: 1.03 [20:09:10< 1:51:57] +[titan] 2025-07-10 09:44:18,042 - root - INFO - step: 91530 loss: 16.3617 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8636 +[titan] 2025-07-10 09:44:18,043 - root - INFO - lr: 3.4847e-05 gnorm: 1.03 [20:09:14< 1:51:54] +[titan] 2025-07-10 09:44:21,948 - root - INFO - step: 91535 loss: 15.7837 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4043 global_avg_mtp_loss: 13.3794 +[titan] 2025-07-10 09:44:21,948 - root - INFO - lr: 3.4841e-05 gnorm: 1.02 [20:09:18< 1:51:50] +[titan] 2025-07-10 09:44:25,843 - root - INFO - step: 91540 loss: 16.0201 memory: 44.58GiB(31.99%) tps: 84,134 tflops: 290.36 mfu: 29.36% global_avg_ntp_loss: 2.4288 global_avg_mtp_loss: 13.5913 +[titan] 2025-07-10 09:44:25,843 - root - INFO - lr: 3.4836e-05 gnorm: 1.07 [20:09:21< 1:51:46] +[titan] 2025-07-10 09:44:29,761 - root - INFO - step: 91545 loss: 16.1830 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.7080 +[titan] 2025-07-10 09:44:29,761 - root - INFO - lr: 3.4830e-05 gnorm: 1.04 [20:09:25< 1:51:42] +[titan] 2025-07-10 09:44:32,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:44:33,670 - root - INFO - step: 91550 loss: 16.1159 memory: 44.58GiB(31.99%) tps: 83,832 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6621 +[titan] 2025-07-10 09:44:33,671 - root - INFO - lr: 3.4824e-05 gnorm: 1.01 [20:09:29< 1:51:38] +[titan] 2025-07-10 09:44:37,576 - root - INFO - step: 91555 loss: 16.3764 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.5177 global_avg_mtp_loss: 13.8587 +[titan] 2025-07-10 09:44:37,576 - root - INFO - lr: 3.4819e-05 gnorm: 1.06 [20:09:33< 1:51:34] +[titan] 2025-07-10 09:44:41,515 - root - INFO - step: 91560 loss: 16.3590 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8605 +[titan] 2025-07-10 09:44:41,516 - root - INFO - lr: 3.4813e-05 gnorm: 1.00 [20:09:37< 1:51:30] +[titan] 2025-07-10 09:44:45,454 - root - INFO - step: 91565 loss: 15.9203 memory: 44.58GiB(31.99%) tps: 83,200 tflops: 287.14 mfu: 29.03% global_avg_ntp_loss: 2.4202 global_avg_mtp_loss: 13.5002 +[titan] 2025-07-10 09:44:45,455 - root - INFO - lr: 3.4807e-05 gnorm: 1.10 [20:09:41< 1:51:26] +[titan] 2025-07-10 09:44:49,408 - root - INFO - step: 91570 loss: 16.2047 memory: 44.58GiB(31.99%) tps: 82,895 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7270 +[titan] 2025-07-10 09:44:49,408 - root - INFO - lr: 3.4802e-05 gnorm: 1.06 [20:09:45< 1:51:22] +[titan] 2025-07-10 09:44:53,312 - root - INFO - step: 91575 loss: 16.1804 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.7132 +[titan] 2025-07-10 09:44:53,312 - root - INFO - lr: 3.4796e-05 gnorm: 1.03 [20:09:49< 1:51:18] +[titan] 2025-07-10 09:44:57,228 - root - INFO - step: 91580 loss: 15.9590 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4190 global_avg_mtp_loss: 13.5399 +[titan] 2025-07-10 09:44:57,229 - root - INFO - lr: 3.4790e-05 gnorm: 1.04 [20:09:53< 1:51:14] +[titan] 2025-07-10 09:45:01,133 - root - INFO - step: 91585 loss: 16.2044 memory: 44.58GiB(31.99%) tps: 83,925 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7388 +[titan] 2025-07-10 09:45:01,134 - root - INFO - lr: 3.4785e-05 gnorm: 1.04 [20:09:57< 1:51:10] +[titan] 2025-07-10 09:45:05,066 - root - INFO - step: 91590 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6820 +[titan] 2025-07-10 09:45:05,066 - root - INFO - lr: 3.4779e-05 gnorm: 1.00 [20:10:01< 1:51:06] +[titan] 2025-07-10 09:45:08,984 - root - INFO - step: 91595 loss: 16.0954 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.6533 +[titan] 2025-07-10 09:45:08,984 - root - INFO - lr: 3.4773e-05 gnorm: 1.02 [20:10:05< 1:51:02] +[titan] 2025-07-10 09:45:12,111 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:45:12,907 - root - INFO - step: 91600 loss: 16.0765 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.6332 +[titan] 2025-07-10 09:45:12,907 - root - INFO - lr: 3.4768e-05 gnorm: 1.02 [20:10:09< 1:50:58] +[titan] 2025-07-10 09:45:16,817 - root - INFO - step: 91605 loss: 16.1970 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7260 +[titan] 2025-07-10 09:45:16,818 - root - INFO - lr: 3.4762e-05 gnorm: 1.04 [20:10:12< 1:50:54] +[titan] 2025-07-10 09:45:20,728 - root - INFO - step: 91610 loss: 16.3727 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5146 global_avg_mtp_loss: 13.8581 +[titan] 2025-07-10 09:45:20,729 - root - INFO - lr: 3.4756e-05 gnorm: 0.99 [20:10:16< 1:50:50] +[titan] 2025-07-10 09:45:24,632 - root - INFO - step: 91615 loss: 16.2880 memory: 44.58GiB(31.99%) tps: 83,945 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.7983 +[titan] 2025-07-10 09:45:24,633 - root - INFO - lr: 3.4751e-05 gnorm: 1.00 [20:10:20< 1:50:46] +[titan] 2025-07-10 09:45:28,527 - root - INFO - step: 91620 loss: 16.1800 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.7073 +[titan] 2025-07-10 09:45:28,527 - root - INFO - lr: 3.4745e-05 gnorm: 1.09 [20:10:24< 1:50:42] +[titan] 2025-07-10 09:45:32,432 - root - INFO - step: 91625 loss: 16.1636 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4579 global_avg_mtp_loss: 13.7057 +[titan] 2025-07-10 09:45:32,432 - root - INFO - lr: 3.4740e-05 gnorm: 1.08 [20:10:28< 1:50:38] +[titan] 2025-07-10 09:45:36,316 - root - INFO - step: 91630 loss: 16.2201 memory: 44.58GiB(31.99%) tps: 84,360 tflops: 291.14 mfu: 29.44% global_avg_ntp_loss: 2.4718 global_avg_mtp_loss: 13.7483 +[titan] 2025-07-10 09:45:36,317 - root - INFO - lr: 3.4734e-05 gnorm: 1.01 [20:10:32< 1:50:34] +[titan] 2025-07-10 09:45:40,235 - root - INFO - step: 91635 loss: 16.5126 memory: 44.58GiB(31.99%) tps: 83,639 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.5428 global_avg_mtp_loss: 13.9697 +[titan] 2025-07-10 09:45:40,235 - root - INFO - lr: 3.4728e-05 gnorm: 1.05 [20:10:36< 1:50:30] +[titan] 2025-07-10 09:45:44,161 - root - INFO - step: 91640 loss: 16.1120 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4388 global_avg_mtp_loss: 13.6732 +[titan] 2025-07-10 09:45:44,161 - root - INFO - lr: 3.4723e-05 gnorm: 1.04 [20:10:40< 1:50:26] +[titan] 2025-07-10 09:45:48,145 - root - INFO - step: 91645 loss: 16.2066 memory: 44.58GiB(31.99%) tps: 82,251 tflops: 283.86 mfu: 28.70% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7346 +[titan] 2025-07-10 09:45:48,146 - root - INFO - lr: 3.4717e-05 gnorm: 1.01 [20:10:44< 1:50:22] +[titan] 2025-07-10 09:45:50,651 - root - INFO - Dumping profiler traces at step 91648 +[titan] 2025-07-10 09:45:50,683 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:45:51,475 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:45:52,259 - root - INFO - step: 91650 loss: 16.2254 memory: 44.58GiB(31.99%) tps: 79,655 tflops: 274.90 mfu: 27.80% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7523 +[titan] 2025-07-10 09:45:52,260 - root - INFO - lr: 3.4711e-05 gnorm: 1.01 [20:10:48< 1:50:18] +[titan] 2025-07-10 09:45:56,202 - root - INFO - step: 91655 loss: 16.3292 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8342 +[titan] 2025-07-10 09:45:56,202 - root - INFO - lr: 3.4706e-05 gnorm: 1.03 [20:10:52< 1:50:14] +[titan] 2025-07-10 09:46:00,127 - root - INFO - step: 91660 loss: 16.1233 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6611 +[titan] 2025-07-10 09:46:00,127 - root - INFO - lr: 3.4700e-05 gnorm: 1.02 [20:10:56< 1:50:10] +[titan] 2025-07-10 09:46:04,069 - root - INFO - step: 91665 loss: 16.3941 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8995 +[titan] 2025-07-10 09:46:04,069 - root - INFO - lr: 3.4695e-05 gnorm: 1.07 [20:11:00< 1:50:06] +[titan] 2025-07-10 09:46:07,996 - root - INFO - step: 91670 loss: 16.3619 memory: 44.58GiB(31.99%) tps: 83,466 tflops: 288.05 mfu: 29.13% global_avg_ntp_loss: 2.5142 global_avg_mtp_loss: 13.8477 +[titan] 2025-07-10 09:46:07,996 - root - INFO - lr: 3.4689e-05 gnorm: 1.03 [20:11:04< 1:50:02] +[titan] 2025-07-10 09:46:11,962 - root - INFO - step: 91675 loss: 16.0735 memory: 44.58GiB(31.99%) tps: 82,619 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.4511 global_avg_mtp_loss: 13.6224 +[titan] 2025-07-10 09:46:11,962 - root - INFO - lr: 3.4683e-05 gnorm: 1.01 [20:11:08< 1:49:58] +[titan] 2025-07-10 09:46:15,879 - root - INFO - step: 91680 loss: 16.2594 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.7640 +[titan] 2025-07-10 09:46:15,879 - root - INFO - lr: 3.4678e-05 gnorm: 1.05 [20:11:12< 1:49:55] +[titan] 2025-07-10 09:46:19,785 - root - INFO - step: 91685 loss: 16.1326 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6796 +[titan] 2025-07-10 09:46:19,786 - root - INFO - lr: 3.4672e-05 gnorm: 1.04 [20:11:15< 1:49:51] +[titan] 2025-07-10 09:46:23,721 - root - INFO - step: 91690 loss: 16.2370 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7556 +[titan] 2025-07-10 09:46:23,721 - root - INFO - lr: 3.4667e-05 gnorm: 1.03 [20:11:19< 1:49:47] +[titan] 2025-07-10 09:46:27,690 - root - INFO - step: 91695 loss: 15.8622 memory: 44.58GiB(31.99%) tps: 82,572 tflops: 284.97 mfu: 28.81% global_avg_ntp_loss: 2.4153 global_avg_mtp_loss: 13.4469 +[titan] 2025-07-10 09:46:27,690 - root - INFO - lr: 3.4661e-05 gnorm: 1.05 [20:11:23< 1:49:43] +[titan] 2025-07-10 09:46:30,824 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:46:31,613 - root - INFO - step: 91700 loss: 16.0793 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4363 global_avg_mtp_loss: 13.6430 +[titan] 2025-07-10 09:46:31,613 - root - INFO - lr: 3.4656e-05 gnorm: 1.01 [20:11:27< 1:49:39] +[titan] 2025-07-10 09:46:35,551 - root - INFO - step: 91705 loss: 16.2805 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.7872 +[titan] 2025-07-10 09:46:35,551 - root - INFO - lr: 3.4650e-05 gnorm: 1.03 [20:11:31< 1:49:35] +[titan] 2025-07-10 09:46:39,468 - root - INFO - step: 91710 loss: 16.0253 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4373 global_avg_mtp_loss: 13.5880 +[titan] 2025-07-10 09:46:39,468 - root - INFO - lr: 3.4644e-05 gnorm: 1.00 [20:11:35< 1:49:31] +[titan] 2025-07-10 09:46:43,419 - root - INFO - step: 91715 loss: 16.1216 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.4529 global_avg_mtp_loss: 13.6687 +[titan] 2025-07-10 09:46:43,420 - root - INFO - lr: 3.4639e-05 gnorm: 1.02 [20:11:39< 1:49:27] +[titan] 2025-07-10 09:46:47,379 - root - INFO - step: 91720 loss: 16.0367 memory: 44.58GiB(31.99%) tps: 82,775 tflops: 285.67 mfu: 28.88% global_avg_ntp_loss: 2.4434 global_avg_mtp_loss: 13.5933 +[titan] 2025-07-10 09:46:47,379 - root - INFO - lr: 3.4633e-05 gnorm: 1.07 [20:11:43< 1:49:23] +[titan] 2025-07-10 09:46:51,337 - root - INFO - step: 91725 loss: 16.1982 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7284 +[titan] 2025-07-10 09:46:51,337 - root - INFO - lr: 3.4628e-05 gnorm: 1.05 [20:11:47< 1:49:19] +[titan] 2025-07-10 09:46:55,270 - root - INFO - step: 91730 loss: 16.0349 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.08% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.5902 +[titan] 2025-07-10 09:46:55,270 - root - INFO - lr: 3.4622e-05 gnorm: 1.03 [20:11:51< 1:49:15] +[titan] 2025-07-10 09:46:59,211 - root - INFO - step: 91735 loss: 16.0652 memory: 44.58GiB(31.99%) tps: 83,155 tflops: 286.98 mfu: 29.02% global_avg_ntp_loss: 2.4452 global_avg_mtp_loss: 13.6200 +[titan] 2025-07-10 09:46:59,211 - root - INFO - lr: 3.4617e-05 gnorm: 0.99 [20:11:55< 1:49:11] +[titan] 2025-07-10 09:47:03,148 - root - INFO - step: 91740 loss: 16.0468 memory: 44.58GiB(31.99%) tps: 83,245 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4416 global_avg_mtp_loss: 13.6052 +[titan] 2025-07-10 09:47:03,148 - root - INFO - lr: 3.4611e-05 gnorm: 1.09 [20:11:59< 1:49:07] +[titan] 2025-07-10 09:47:07,091 - root - INFO - step: 91745 loss: 15.8787 memory: 44.58GiB(31.99%) tps: 83,110 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4164 global_avg_mtp_loss: 13.4623 +[titan] 2025-07-10 09:47:07,091 - root - INFO - lr: 3.4605e-05 gnorm: 1.02 [20:12:03< 1:49:03] +[titan] 2025-07-10 09:47:10,224 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:47:11,027 - root - INFO - step: 91750 loss: 16.3574 memory: 44.58GiB(31.99%) tps: 83,253 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8605 +[titan] 2025-07-10 09:47:11,027 - root - INFO - lr: 3.4600e-05 gnorm: 1.03 [20:12:07< 1:48:59] +[titan] 2025-07-10 09:47:14,972 - root - INFO - step: 91755 loss: 16.0137 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4286 global_avg_mtp_loss: 13.5851 +[titan] 2025-07-10 09:47:14,972 - root - INFO - lr: 3.4594e-05 gnorm: 1.07 [20:12:11< 1:48:55] +[titan] 2025-07-10 09:47:18,912 - root - INFO - step: 91760 loss: 16.1491 memory: 44.58GiB(31.99%) tps: 83,163 tflops: 287.01 mfu: 29.02% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6875 +[titan] 2025-07-10 09:47:18,913 - root - INFO - lr: 3.4589e-05 gnorm: 0.97 [20:12:15< 1:48:51] +[titan] 2025-07-10 09:47:22,808 - root - INFO - step: 91765 loss: 16.4052 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4986 global_avg_mtp_loss: 13.9067 +[titan] 2025-07-10 09:47:22,809 - root - INFO - lr: 3.4583e-05 gnorm: 1.04 [20:12:18< 1:48:47] +[titan] 2025-07-10 09:47:26,728 - root - INFO - step: 91770 loss: 16.1444 memory: 44.58GiB(31.99%) tps: 83,602 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6871 +[titan] 2025-07-10 09:47:26,729 - root - INFO - lr: 3.4578e-05 gnorm: 0.99 [20:12:22< 1:48:43] +[titan] 2025-07-10 09:47:30,662 - root - INFO - step: 91775 loss: 16.3557 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4965 global_avg_mtp_loss: 13.8593 +[titan] 2025-07-10 09:47:30,662 - root - INFO - lr: 3.4572e-05 gnorm: 1.00 [20:12:26< 1:48:39] +[titan] 2025-07-10 09:47:34,566 - root - INFO - step: 91780 loss: 16.2209 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4810 global_avg_mtp_loss: 13.7399 +[titan] 2025-07-10 09:47:34,567 - root - INFO - lr: 3.4567e-05 gnorm: 1.02 [20:12:30< 1:48:35] +[titan] 2025-07-10 09:47:38,472 - root - INFO - step: 91785 loss: 16.1368 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.6708 +[titan] 2025-07-10 09:47:38,472 - root - INFO - lr: 3.4561e-05 gnorm: 1.05 [20:12:34< 1:48:31] +[titan] 2025-07-10 09:47:42,398 - root - INFO - step: 91790 loss: 16.0102 memory: 44.58GiB(31.99%) tps: 83,463 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4392 global_avg_mtp_loss: 13.5710 +[titan] 2025-07-10 09:47:42,399 - root - INFO - lr: 3.4556e-05 gnorm: 1.21 [20:12:38< 1:48:27] +[titan] 2025-07-10 09:47:46,304 - root - INFO - step: 91795 loss: 16.1941 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7186 +[titan] 2025-07-10 09:47:46,305 - root - INFO - lr: 3.4550e-05 gnorm: 1.04 [20:12:42< 1:48:23] +[titan] 2025-07-10 09:47:49,423 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:47:50,215 - root - INFO - step: 91800 loss: 16.0422 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.5935 +[titan] 2025-07-10 09:47:50,215 - root - INFO - lr: 3.4545e-05 gnorm: 1.02 [20:12:46< 1:48:19] +[titan] 2025-07-10 09:47:54,148 - root - INFO - step: 91805 loss: 16.6224 memory: 44.58GiB(31.99%) tps: 83,316 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.5562 global_avg_mtp_loss: 14.0662 +[titan] 2025-07-10 09:47:54,148 - root - INFO - lr: 3.4539e-05 gnorm: 1.02 [20:12:50< 1:48:15] +[titan] 2025-07-10 09:47:58,055 - root - INFO - step: 91810 loss: 16.2633 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.7725 +[titan] 2025-07-10 09:47:58,056 - root - INFO - lr: 3.4534e-05 gnorm: 0.95 [20:12:54< 1:48:11] +[titan] 2025-07-10 09:48:01,973 - root - INFO - step: 91815 loss: 15.9814 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4331 global_avg_mtp_loss: 13.5482 +[titan] 2025-07-10 09:48:01,974 - root - INFO - lr: 3.4528e-05 gnorm: 1.02 [20:12:58< 1:48:07] +[titan] 2025-07-10 09:48:05,918 - root - INFO - step: 91820 loss: 15.9246 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.4259 global_avg_mtp_loss: 13.4987 +[titan] 2025-07-10 09:48:05,919 - root - INFO - lr: 3.4523e-05 gnorm: 1.03 [20:13:02< 1:48:03] +[titan] 2025-07-10 09:48:09,833 - root - INFO - step: 91825 loss: 16.1315 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4505 global_avg_mtp_loss: 13.6810 +[titan] 2025-07-10 09:48:09,833 - root - INFO - lr: 3.4517e-05 gnorm: 1.07 [20:13:05< 1:47:59] +[titan] 2025-07-10 09:48:13,785 - root - INFO - step: 91830 loss: 16.4762 memory: 44.58GiB(31.99%) tps: 82,917 tflops: 286.16 mfu: 28.93% global_avg_ntp_loss: 2.5226 global_avg_mtp_loss: 13.9536 +[titan] 2025-07-10 09:48:13,785 - root - INFO - lr: 3.4512e-05 gnorm: 1.00 [20:13:09< 1:47:56] +[titan] 2025-07-10 09:48:17,718 - root - INFO - step: 91835 loss: 16.3861 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.8786 +[titan] 2025-07-10 09:48:17,719 - root - INFO - lr: 3.4506e-05 gnorm: 1.08 [20:13:13< 1:47:52] +[titan] 2025-07-10 09:48:21,646 - root - INFO - step: 91840 loss: 16.1877 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7216 +[titan] 2025-07-10 09:48:21,646 - root - INFO - lr: 3.4501e-05 gnorm: 1.03 [20:13:17< 1:47:48] +[titan] 2025-07-10 09:48:25,559 - root - INFO - step: 91845 loss: 16.1326 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6740 +[titan] 2025-07-10 09:48:25,560 - root - INFO - lr: 3.4495e-05 gnorm: 1.00 [20:13:21< 1:47:44] +[titan] 2025-07-10 09:48:28,699 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:48:29,487 - root - INFO - step: 91850 loss: 16.5714 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5396 global_avg_mtp_loss: 14.0318 +[titan] 2025-07-10 09:48:29,487 - root - INFO - lr: 3.4490e-05 gnorm: 1.07 [20:13:25< 1:47:40] +[titan] 2025-07-10 09:48:33,441 - root - INFO - step: 91855 loss: 16.2872 memory: 44.58GiB(31.99%) tps: 82,880 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.8107 +[titan] 2025-07-10 09:48:33,441 - root - INFO - lr: 3.4484e-05 gnorm: 0.99 [20:13:29< 1:47:36] +[titan] 2025-07-10 09:48:37,353 - root - INFO - step: 91860 loss: 16.1731 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.7031 +[titan] 2025-07-10 09:48:37,353 - root - INFO - lr: 3.4479e-05 gnorm: 1.06 [20:13:33< 1:47:32] +[titan] 2025-07-10 09:48:41,263 - root - INFO - step: 91865 loss: 16.2938 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.8148 +[titan] 2025-07-10 09:48:41,263 - root - INFO - lr: 3.4473e-05 gnorm: 1.03 [20:13:37< 1:47:28] +[titan] 2025-07-10 09:48:45,192 - root - INFO - step: 91870 loss: 16.2200 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7472 +[titan] 2025-07-10 09:48:45,193 - root - INFO - lr: 3.4468e-05 gnorm: 1.01 [20:13:41< 1:47:24] +[titan] 2025-07-10 09:48:49,100 - root - INFO - step: 91875 loss: 16.0632 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4413 global_avg_mtp_loss: 13.6219 +[titan] 2025-07-10 09:48:49,100 - root - INFO - lr: 3.4462e-05 gnorm: 1.02 [20:13:45< 1:47:20] +[titan] 2025-07-10 09:48:53,016 - root - INFO - step: 91880 loss: 16.1017 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6327 +[titan] 2025-07-10 09:48:53,017 - root - INFO - lr: 3.4457e-05 gnorm: 1.05 [20:13:49< 1:47:16] +[titan] 2025-07-10 09:48:56,935 - root - INFO - step: 91885 loss: 16.2772 memory: 44.58GiB(31.99%) tps: 83,641 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7912 +[titan] 2025-07-10 09:48:56,935 - root - INFO - lr: 3.4451e-05 gnorm: 1.02 [20:13:53< 1:47:12] +[titan] 2025-07-10 09:49:00,863 - root - INFO - step: 91890 loss: 16.1300 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.6619 +[titan] 2025-07-10 09:49:00,864 - root - INFO - lr: 3.4446e-05 gnorm: 1.01 [20:13:56< 1:47:08] +[titan] 2025-07-10 09:49:04,787 - root - INFO - step: 91895 loss: 16.1336 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.6855 +[titan] 2025-07-10 09:49:04,787 - root - INFO - lr: 3.4441e-05 gnorm: 1.02 [20:14:00< 1:47:04] +[titan] 2025-07-10 09:49:07,908 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:49:08,692 - root - INFO - step: 91900 loss: 16.1514 memory: 44.58GiB(31.99%) tps: 83,931 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4656 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 09:49:08,692 - root - INFO - lr: 3.4435e-05 gnorm: 1.01 [20:14:04< 1:47:00] +[titan] 2025-07-10 09:49:12,637 - root - INFO - step: 91905 loss: 16.0365 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.5724 +[titan] 2025-07-10 09:49:12,637 - root - INFO - lr: 3.4430e-05 gnorm: 1.10 [20:14:08< 1:46:56] +[titan] 2025-07-10 09:49:16,538 - root - INFO - step: 91910 loss: 15.9904 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4446 global_avg_mtp_loss: 13.5459 +[titan] 2025-07-10 09:49:16,538 - root - INFO - lr: 3.4424e-05 gnorm: 1.10 [20:14:12< 1:46:52] +[titan] 2025-07-10 09:49:20,460 - root - INFO - step: 91915 loss: 16.2753 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7824 +[titan] 2025-07-10 09:49:20,460 - root - INFO - lr: 3.4419e-05 gnorm: 1.03 [20:14:16< 1:46:48] +[titan] 2025-07-10 09:49:24,371 - root - INFO - step: 91920 loss: 16.2769 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7972 +[titan] 2025-07-10 09:49:24,372 - root - INFO - lr: 3.4413e-05 gnorm: 0.98 [20:14:20< 1:46:44] +[titan] 2025-07-10 09:49:28,286 - root - INFO - step: 91925 loss: 16.2176 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7436 +[titan] 2025-07-10 09:49:28,286 - root - INFO - lr: 3.4408e-05 gnorm: 0.99 [20:14:24< 1:46:40] +[titan] 2025-07-10 09:49:32,192 - root - INFO - step: 91930 loss: 16.0761 memory: 44.58GiB(31.99%) tps: 83,886 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4466 global_avg_mtp_loss: 13.6295 +[titan] 2025-07-10 09:49:32,192 - root - INFO - lr: 3.4402e-05 gnorm: 1.02 [20:14:28< 1:46:36] +[titan] 2025-07-10 09:49:36,088 - root - INFO - step: 91935 loss: 16.0819 memory: 44.58GiB(31.99%) tps: 84,111 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6252 +[titan] 2025-07-10 09:49:36,089 - root - INFO - lr: 3.4397e-05 gnorm: 1.03 [20:14:32< 1:46:32] +[titan] 2025-07-10 09:49:40,042 - root - INFO - step: 91940 loss: 16.2876 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.8021 +[titan] 2025-07-10 09:49:40,043 - root - INFO - lr: 3.4392e-05 gnorm: 1.11 [20:14:36< 1:46:28] +[titan] 2025-07-10 09:49:43,967 - root - INFO - step: 91945 loss: 16.0851 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.6430 +[titan] 2025-07-10 09:49:43,968 - root - INFO - lr: 3.4386e-05 gnorm: 1.08 [20:14:40< 1:46:24] +[titan] 2025-07-10 09:49:47,090 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:49:47,882 - root - INFO - step: 91950 loss: 16.2161 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4761 global_avg_mtp_loss: 13.7400 +[titan] 2025-07-10 09:49:47,883 - root - INFO - lr: 3.4381e-05 gnorm: 1.05 [20:14:43< 1:46:20] +[titan] 2025-07-10 09:49:51,802 - root - INFO - step: 91955 loss: 16.3071 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4949 global_avg_mtp_loss: 13.8123 +[titan] 2025-07-10 09:49:51,802 - root - INFO - lr: 3.4375e-05 gnorm: 0.99 [20:14:47< 1:46:16] +[titan] 2025-07-10 09:49:55,714 - root - INFO - step: 91960 loss: 16.0352 memory: 44.58GiB(31.99%) tps: 83,766 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4415 global_avg_mtp_loss: 13.5936 +[titan] 2025-07-10 09:49:55,714 - root - INFO - lr: 3.4370e-05 gnorm: 1.03 [20:14:51< 1:46:12] +[titan] 2025-07-10 09:49:59,647 - root - INFO - step: 91965 loss: 15.8890 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4215 global_avg_mtp_loss: 13.4675 +[titan] 2025-07-10 09:49:59,648 - root - INFO - lr: 3.4365e-05 gnorm: 1.02 [20:14:55< 1:46:08] +[titan] 2025-07-10 09:50:03,566 - root - INFO - step: 91970 loss: 16.1901 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.7235 +[titan] 2025-07-10 09:50:03,567 - root - INFO - lr: 3.4359e-05 gnorm: 1.03 [20:14:59< 1:46:04] +[titan] 2025-07-10 09:50:07,474 - root - INFO - step: 91975 loss: 16.1684 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.7013 +[titan] 2025-07-10 09:50:07,474 - root - INFO - lr: 3.4354e-05 gnorm: 1.00 [20:15:03< 1:46:00] +[titan] 2025-07-10 09:50:11,415 - root - INFO - step: 91980 loss: 16.0344 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.5856 +[titan] 2025-07-10 09:50:11,415 - root - INFO - lr: 3.4348e-05 gnorm: 1.07 [20:15:07< 1:45:57] +[titan] 2025-07-10 09:50:15,335 - root - INFO - step: 91985 loss: 16.0723 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.6303 +[titan] 2025-07-10 09:50:15,335 - root - INFO - lr: 3.4343e-05 gnorm: 1.01 [20:15:11< 1:45:53] +[titan] 2025-07-10 09:50:19,269 - root - INFO - step: 91990 loss: 16.2064 memory: 44.58GiB(31.99%) tps: 83,303 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7347 +[titan] 2025-07-10 09:50:19,269 - root - INFO - lr: 3.4338e-05 gnorm: 0.99 [20:15:15< 1:45:49] +[titan] 2025-07-10 09:50:23,179 - root - INFO - step: 91995 loss: 16.2004 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-10 09:50:23,180 - root - INFO - lr: 3.4332e-05 gnorm: 1.09 [20:15:19< 1:45:45] +[titan] 2025-07-10 09:50:26,325 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:50:27,148 - root - INFO - step: 92000 loss: 16.3533 memory: 44.58GiB(31.99%) tps: 82,580 tflops: 285.00 mfu: 28.82% global_avg_ntp_loss: 2.5077 global_avg_mtp_loss: 13.8455 +[titan] 2025-07-10 09:50:27,148 - root - INFO - lr: 3.4327e-05 gnorm: 1.06 [20:15:23< 1:45:41] +[titan] 2025-07-10 09:50:31,078 - root - INFO - step: 92005 loss: 16.1662 memory: 44.58GiB(31.99%) tps: 83,378 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.6987 +[titan] 2025-07-10 09:50:31,079 - root - INFO - lr: 3.4321e-05 gnorm: 1.04 [20:15:27< 1:45:37] +[titan] 2025-07-10 09:50:35,033 - root - INFO - step: 92010 loss: 16.0526 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.4544 global_avg_mtp_loss: 13.5981 +[titan] 2025-07-10 09:50:35,033 - root - INFO - lr: 3.4316e-05 gnorm: 1.05 [20:15:31< 1:45:33] +[titan] 2025-07-10 09:50:38,928 - root - INFO - step: 92015 loss: 16.2460 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7720 +[titan] 2025-07-10 09:50:38,929 - root - INFO - lr: 3.4311e-05 gnorm: 1.02 [20:15:35< 1:45:29] +[titan] 2025-07-10 09:50:42,820 - root - INFO - step: 92020 loss: 16.3278 memory: 44.58GiB(31.99%) tps: 84,206 tflops: 290.61 mfu: 29.38% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.8350 +[titan] 2025-07-10 09:50:42,820 - root - INFO - lr: 3.4305e-05 gnorm: 1.01 [20:15:38< 1:45:25] +[titan] 2025-07-10 09:50:46,744 - root - INFO - step: 92025 loss: 16.2655 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7826 +[titan] 2025-07-10 09:50:46,745 - root - INFO - lr: 3.4300e-05 gnorm: 1.09 [20:15:42< 1:45:21] +[titan] 2025-07-10 09:50:50,676 - root - INFO - step: 92030 loss: 16.1027 memory: 44.58GiB(31.99%) tps: 83,349 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4481 global_avg_mtp_loss: 13.6546 +[titan] 2025-07-10 09:50:50,677 - root - INFO - lr: 3.4295e-05 gnorm: 1.00 [20:15:46< 1:45:17] +[titan] 2025-07-10 09:50:54,609 - root - INFO - step: 92035 loss: 16.2058 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7393 +[titan] 2025-07-10 09:50:54,609 - root - INFO - lr: 3.4289e-05 gnorm: 1.02 [20:15:50< 1:45:13] +[titan] 2025-07-10 09:50:58,545 - root - INFO - step: 92040 loss: 16.3144 memory: 44.58GiB(31.99%) tps: 83,268 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8229 +[titan] 2025-07-10 09:50:58,545 - root - INFO - lr: 3.4284e-05 gnorm: 1.04 [20:15:54< 1:45:09] +[titan] 2025-07-10 09:51:02,466 - root - INFO - step: 92045 loss: 16.2265 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7465 +[titan] 2025-07-10 09:51:02,466 - root - INFO - lr: 3.4279e-05 gnorm: 0.99 [20:15:58< 1:45:05] +[titan] 2025-07-10 09:51:05,597 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:51:06,386 - root - INFO - step: 92050 loss: 16.1499 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 09:51:06,386 - root - INFO - lr: 3.4273e-05 gnorm: 1.00 [20:16:02< 1:45:01] +[titan] 2025-07-10 09:51:10,306 - root - INFO - step: 92055 loss: 16.2163 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7377 +[titan] 2025-07-10 09:51:10,307 - root - INFO - lr: 3.4268e-05 gnorm: 0.98 [20:16:06< 1:44:57] +[titan] 2025-07-10 09:51:14,236 - root - INFO - step: 92060 loss: 16.2292 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7492 +[titan] 2025-07-10 09:51:14,236 - root - INFO - lr: 3.4263e-05 gnorm: 1.01 [20:16:10< 1:44:53] +[titan] 2025-07-10 09:51:18,170 - root - INFO - step: 92065 loss: 16.2223 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7389 +[titan] 2025-07-10 09:51:18,170 - root - INFO - lr: 3.4257e-05 gnorm: 1.02 [20:16:14< 1:44:49] +[titan] 2025-07-10 09:51:22,117 - root - INFO - step: 92070 loss: 16.3926 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8822 +[titan] 2025-07-10 09:51:22,117 - root - INFO - lr: 3.4252e-05 gnorm: 1.00 [20:16:18< 1:44:45] +[titan] 2025-07-10 09:51:26,057 - root - INFO - step: 92075 loss: 16.3642 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8657 +[titan] 2025-07-10 09:51:26,058 - root - INFO - lr: 3.4247e-05 gnorm: 1.04 [20:16:22< 1:44:41] +[titan] 2025-07-10 09:51:29,982 - root - INFO - step: 92080 loss: 16.2009 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.7370 +[titan] 2025-07-10 09:51:29,982 - root - INFO - lr: 3.4241e-05 gnorm: 1.08 [20:16:26< 1:44:37] +[titan] 2025-07-10 09:51:33,904 - root - INFO - step: 92085 loss: 16.1550 memory: 44.58GiB(31.99%) tps: 83,552 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6919 +[titan] 2025-07-10 09:51:33,904 - root - INFO - lr: 3.4236e-05 gnorm: 1.01 [20:16:29< 1:44:33] +[titan] 2025-07-10 09:51:37,816 - root - INFO - step: 92090 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.6879 +[titan] 2025-07-10 09:51:37,817 - root - INFO - lr: 3.4231e-05 gnorm: 1.04 [20:16:33< 1:44:29] +[titan] 2025-07-10 09:51:41,748 - root - INFO - step: 92095 loss: 16.0845 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.6387 +[titan] 2025-07-10 09:51:41,748 - root - INFO - lr: 3.4225e-05 gnorm: 1.01 [20:16:37< 1:44:25] +[titan] 2025-07-10 09:51:44,897 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:51:45,691 - root - INFO - step: 92100 loss: 16.1437 memory: 44.58GiB(31.99%) tps: 83,111 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4597 global_avg_mtp_loss: 13.6841 +[titan] 2025-07-10 09:51:45,691 - root - INFO - lr: 3.4220e-05 gnorm: 0.99 [20:16:41< 1:44:21] +[titan] 2025-07-10 09:51:49,588 - root - INFO - step: 92105 loss: 16.2793 memory: 44.58GiB(31.99%) tps: 84,077 tflops: 290.16 mfu: 29.34% global_avg_ntp_loss: 2.4844 global_avg_mtp_loss: 13.7949 +[titan] 2025-07-10 09:51:49,589 - root - INFO - lr: 3.4215e-05 gnorm: 1.05 [20:16:45< 1:44:17] +[titan] 2025-07-10 09:51:53,521 - root - INFO - step: 92110 loss: 16.0532 memory: 44.58GiB(31.99%) tps: 83,326 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4449 global_avg_mtp_loss: 13.6084 +[titan] 2025-07-10 09:51:53,522 - root - INFO - lr: 3.4209e-05 gnorm: 1.09 [20:16:49< 1:44:13] +[titan] 2025-07-10 09:51:57,433 - root - INFO - step: 92115 loss: 16.3913 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 09:51:57,434 - root - INFO - lr: 3.4204e-05 gnorm: 0.98 [20:16:53< 1:44:09] +[titan] 2025-07-10 09:52:01,345 - root - INFO - step: 92120 loss: 16.2385 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4786 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 09:52:01,345 - root - INFO - lr: 3.4199e-05 gnorm: 1.02 [20:16:57< 1:44:05] +[titan] 2025-07-10 09:52:05,267 - root - INFO - step: 92125 loss: 16.1377 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4598 global_avg_mtp_loss: 13.6779 +[titan] 2025-07-10 09:52:05,267 - root - INFO - lr: 3.4193e-05 gnorm: 1.05 [20:17:01< 1:44:01] +[titan] 2025-07-10 09:52:09,207 - root - INFO - step: 92130 loss: 16.0310 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.5973 +[titan] 2025-07-10 09:52:09,207 - root - INFO - lr: 3.4188e-05 gnorm: 1.04 [20:17:05< 1:43:58] +[titan] 2025-07-10 09:52:13,152 - root - INFO - step: 92135 loss: 16.1955 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7146 +[titan] 2025-07-10 09:52:13,152 - root - INFO - lr: 3.4183e-05 gnorm: 1.12 [20:17:09< 1:43:54] +[titan] 2025-07-10 09:52:17,094 - root - INFO - step: 92140 loss: 16.3619 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.8687 +[titan] 2025-07-10 09:52:17,094 - root - INFO - lr: 3.4178e-05 gnorm: 1.06 [20:17:13< 1:43:50] +[titan] 2025-07-10 09:52:21,027 - root - INFO - step: 92145 loss: 15.8247 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4040 global_avg_mtp_loss: 13.4206 +[titan] 2025-07-10 09:52:21,028 - root - INFO - lr: 3.4172e-05 gnorm: 1.02 [20:17:17< 1:43:46] +[titan] 2025-07-10 09:52:24,147 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:52:24,937 - root - INFO - step: 92150 loss: 16.4583 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5116 global_avg_mtp_loss: 13.9468 +[titan] 2025-07-10 09:52:24,937 - root - INFO - lr: 3.4167e-05 gnorm: 1.03 [20:17:21< 1:43:42] +[titan] 2025-07-10 09:52:28,876 - root - INFO - step: 92155 loss: 16.2834 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.7841 +[titan] 2025-07-10 09:52:28,876 - root - INFO - lr: 3.4162e-05 gnorm: 1.08 [20:17:24< 1:43:38] +[titan] 2025-07-10 09:52:32,884 - root - INFO - step: 92160 loss: 16.1423 memory: 44.58GiB(31.99%) tps: 81,750 tflops: 282.13 mfu: 28.53% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.6786 +[titan] 2025-07-10 09:52:32,885 - root - INFO - lr: 3.4156e-05 gnorm: 1.01 [20:17:28< 1:43:34] +[titan] 2025-07-10 09:52:33,041 - root - INFO - Dumping profiler traces at step 92160 +[titan] 2025-07-10 09:52:33,075 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:52:37,012 - root - INFO - step: 92165 loss: 16.1700 memory: 44.58GiB(31.99%) tps: 79,393 tflops: 274.00 mfu: 27.70% global_avg_ntp_loss: 2.4763 global_avg_mtp_loss: 13.6938 +[titan] 2025-07-10 09:52:37,012 - root - INFO - lr: 3.4151e-05 gnorm: 1.02 [20:17:33< 1:43:30] +[titan] 2025-07-10 09:52:40,922 - root - INFO - step: 92170 loss: 16.1655 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.6934 +[titan] 2025-07-10 09:52:40,922 - root - INFO - lr: 3.4146e-05 gnorm: 1.01 [20:17:37< 1:43:26] +[titan] 2025-07-10 09:52:44,865 - root - INFO - step: 92175 loss: 16.0157 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5721 +[titan] 2025-07-10 09:52:44,865 - root - INFO - lr: 3.4141e-05 gnorm: 0.98 [20:17:40< 1:43:22] +[titan] 2025-07-10 09:52:48,815 - root - INFO - step: 92180 loss: 16.1768 memory: 44.58GiB(31.99%) tps: 82,959 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.7166 +[titan] 2025-07-10 09:52:48,816 - root - INFO - lr: 3.4135e-05 gnorm: 1.03 [20:17:44< 1:43:18] +[titan] 2025-07-10 09:52:52,724 - root - INFO - step: 92185 loss: 16.2147 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7368 +[titan] 2025-07-10 09:52:52,725 - root - INFO - lr: 3.4130e-05 gnorm: 1.01 [20:17:48< 1:43:14] +[titan] 2025-07-10 09:52:56,684 - root - INFO - step: 92190 loss: 16.2441 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4788 global_avg_mtp_loss: 13.7653 +[titan] 2025-07-10 09:52:56,685 - root - INFO - lr: 3.4125e-05 gnorm: 1.07 [20:17:52< 1:43:10] +[titan] 2025-07-10 09:53:00,605 - root - INFO - step: 92195 loss: 16.3333 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5066 global_avg_mtp_loss: 13.8266 +[titan] 2025-07-10 09:53:00,606 - root - INFO - lr: 3.4120e-05 gnorm: 0.99 [20:17:56< 1:43:06] +[titan] 2025-07-10 09:53:03,732 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:53:04,524 - root - INFO - step: 92200 loss: 16.6230 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.5581 global_avg_mtp_loss: 14.0649 +[titan] 2025-07-10 09:53:04,524 - root - INFO - lr: 3.4114e-05 gnorm: 1.02 [20:18:00< 1:43:02] +[titan] 2025-07-10 09:53:08,454 - root - INFO - step: 92205 loss: 16.1813 memory: 44.58GiB(31.99%) tps: 83,382 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7085 +[titan] 2025-07-10 09:53:08,454 - root - INFO - lr: 3.4109e-05 gnorm: 1.04 [20:18:04< 1:42:58] +[titan] 2025-07-10 09:53:12,363 - root - INFO - step: 92210 loss: 16.1863 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.7187 +[titan] 2025-07-10 09:53:12,363 - root - INFO - lr: 3.4104e-05 gnorm: 1.00 [20:18:08< 1:42:54] +[titan] 2025-07-10 09:53:16,280 - root - INFO - step: 92215 loss: 16.0053 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.5652 +[titan] 2025-07-10 09:53:16,280 - root - INFO - lr: 3.4099e-05 gnorm: 1.05 [20:18:12< 1:42:50] +[titan] 2025-07-10 09:53:20,221 - root - INFO - step: 92220 loss: 16.2489 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7684 +[titan] 2025-07-10 09:53:20,222 - root - INFO - lr: 3.4093e-05 gnorm: 1.00 [20:18:16< 1:42:46] +[titan] 2025-07-10 09:53:24,144 - root - INFO - step: 92225 loss: 16.1449 memory: 44.58GiB(31.99%) tps: 83,539 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6858 +[titan] 2025-07-10 09:53:24,144 - root - INFO - lr: 3.4088e-05 gnorm: 1.04 [20:18:20< 1:42:42] +[titan] 2025-07-10 09:53:28,106 - root - INFO - step: 92230 loss: 16.4322 memory: 44.58GiB(31.99%) tps: 82,728 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.5181 global_avg_mtp_loss: 13.9142 +[titan] 2025-07-10 09:53:28,106 - root - INFO - lr: 3.4083e-05 gnorm: 1.07 [20:18:24< 1:42:38] +[titan] 2025-07-10 09:53:32,001 - root - INFO - step: 92235 loss: 16.0229 memory: 44.58GiB(31.99%) tps: 84,125 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5811 +[titan] 2025-07-10 09:53:32,001 - root - INFO - lr: 3.4078e-05 gnorm: 1.01 [20:18:28< 1:42:34] +[titan] 2025-07-10 09:53:35,910 - root - INFO - step: 92240 loss: 15.8388 memory: 44.58GiB(31.99%) tps: 83,836 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4138 global_avg_mtp_loss: 13.4250 +[titan] 2025-07-10 09:53:35,910 - root - INFO - lr: 3.4072e-05 gnorm: 0.99 [20:18:31< 1:42:30] +[titan] 2025-07-10 09:53:39,842 - root - INFO - step: 92245 loss: 15.9148 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.4819 +[titan] 2025-07-10 09:53:39,842 - root - INFO - lr: 3.4067e-05 gnorm: 1.01 [20:18:35< 1:42:26] +[titan] 2025-07-10 09:53:42,981 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:53:43,784 - root - INFO - step: 92250 loss: 16.1044 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4361 global_avg_mtp_loss: 13.6682 +[titan] 2025-07-10 09:53:43,784 - root - INFO - lr: 3.4062e-05 gnorm: 1.07 [20:18:39< 1:42:22] +[titan] 2025-07-10 09:53:47,698 - root - INFO - step: 92255 loss: 16.0540 memory: 44.58GiB(31.99%) tps: 83,714 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.6093 +[titan] 2025-07-10 09:53:47,699 - root - INFO - lr: 3.4057e-05 gnorm: 1.03 [20:18:43< 1:42:18] +[titan] 2025-07-10 09:53:51,595 - root - INFO - step: 92260 loss: 16.1107 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6580 +[titan] 2025-07-10 09:53:51,595 - root - INFO - lr: 3.4052e-05 gnorm: 1.00 [20:18:47< 1:42:14] +[titan] 2025-07-10 09:53:55,503 - root - INFO - step: 92265 loss: 16.3374 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8381 +[titan] 2025-07-10 09:53:55,503 - root - INFO - lr: 3.4046e-05 gnorm: 1.02 [20:18:51< 1:42:10] +[titan] 2025-07-10 09:53:59,433 - root - INFO - step: 92270 loss: 16.0158 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.5829 +[titan] 2025-07-10 09:53:59,433 - root - INFO - lr: 3.4041e-05 gnorm: 1.08 [20:18:55< 1:42:06] +[titan] 2025-07-10 09:54:03,358 - root - INFO - step: 92275 loss: 16.1542 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.6735 +[titan] 2025-07-10 09:54:03,358 - root - INFO - lr: 3.4036e-05 gnorm: 1.10 [20:18:59< 1:42:03] +[titan] 2025-07-10 09:54:07,286 - root - INFO - step: 92280 loss: 16.4641 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.5109 global_avg_mtp_loss: 13.9533 +[titan] 2025-07-10 09:54:07,287 - root - INFO - lr: 3.4031e-05 gnorm: 1.07 [20:19:03< 1:41:59] +[titan] 2025-07-10 09:54:11,182 - root - INFO - step: 92285 loss: 16.0946 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.6478 +[titan] 2025-07-10 09:54:11,182 - root - INFO - lr: 3.4026e-05 gnorm: 1.08 [20:19:07< 1:41:55] +[titan] 2025-07-10 09:54:15,096 - root - INFO - step: 92290 loss: 16.2590 memory: 44.58GiB(31.99%) tps: 83,728 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7833 +[titan] 2025-07-10 09:54:15,096 - root - INFO - lr: 3.4020e-05 gnorm: 0.97 [20:19:11< 1:41:51] +[titan] 2025-07-10 09:54:19,056 - root - INFO - step: 92295 loss: 16.2534 memory: 44.58GiB(31.99%) tps: 82,771 tflops: 285.65 mfu: 28.88% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 09:54:19,056 - root - INFO - lr: 3.4015e-05 gnorm: 1.03 [20:19:15< 1:41:47] +[titan] 2025-07-10 09:54:22,177 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:54:23,010 - root - INFO - step: 92300 loss: 16.2066 memory: 44.58GiB(31.99%) tps: 82,882 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.7370 +[titan] 2025-07-10 09:54:23,010 - root - INFO - lr: 3.4010e-05 gnorm: 1.04 [20:19:19< 1:41:43] +[titan] 2025-07-10 09:54:26,923 - root - INFO - step: 92305 loss: 15.9820 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4367 global_avg_mtp_loss: 13.5453 +[titan] 2025-07-10 09:54:26,923 - root - INFO - lr: 3.4005e-05 gnorm: 1.06 [20:19:22< 1:41:39] +[titan] 2025-07-10 09:54:30,842 - root - INFO - step: 92310 loss: 15.9125 memory: 44.58GiB(31.99%) tps: 83,608 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4198 global_avg_mtp_loss: 13.4927 +[titan] 2025-07-10 09:54:30,843 - root - INFO - lr: 3.4000e-05 gnorm: 1.01 [20:19:26< 1:41:35] +[titan] 2025-07-10 09:54:34,736 - root - INFO - step: 92315 loss: 15.9846 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5410 +[titan] 2025-07-10 09:54:34,736 - root - INFO - lr: 3.3994e-05 gnorm: 1.06 [20:19:30< 1:41:31] +[titan] 2025-07-10 09:54:38,644 - root - INFO - step: 92320 loss: 16.1735 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7075 +[titan] 2025-07-10 09:54:38,644 - root - INFO - lr: 3.3989e-05 gnorm: 1.03 [20:19:34< 1:41:27] +[titan] 2025-07-10 09:54:42,575 - root - INFO - step: 92325 loss: 16.4618 memory: 44.58GiB(31.99%) tps: 83,361 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5275 global_avg_mtp_loss: 13.9343 +[titan] 2025-07-10 09:54:42,575 - root - INFO - lr: 3.3984e-05 gnorm: 1.01 [20:19:38< 1:41:23] +[titan] 2025-07-10 09:54:46,533 - root - INFO - step: 92330 loss: 16.1856 memory: 44.58GiB(31.99%) tps: 82,807 tflops: 285.78 mfu: 28.90% global_avg_ntp_loss: 2.4621 global_avg_mtp_loss: 13.7235 +[titan] 2025-07-10 09:54:46,533 - root - INFO - lr: 3.3979e-05 gnorm: 0.98 [20:19:42< 1:41:19] +[titan] 2025-07-10 09:54:50,443 - root - INFO - step: 92335 loss: 16.4418 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.5201 global_avg_mtp_loss: 13.9216 +[titan] 2025-07-10 09:54:50,443 - root - INFO - lr: 3.3974e-05 gnorm: 1.01 [20:19:46< 1:41:15] +[titan] 2025-07-10 09:54:54,437 - root - INFO - step: 92340 loss: 16.3529 memory: 44.58GiB(31.99%) tps: 82,049 tflops: 283.16 mfu: 28.63% global_avg_ntp_loss: 2.5070 global_avg_mtp_loss: 13.8459 +[titan] 2025-07-10 09:54:54,438 - root - INFO - lr: 3.3969e-05 gnorm: 1.03 [20:19:50< 1:41:11] +[titan] 2025-07-10 09:54:58,357 - root - INFO - step: 92345 loss: 15.8825 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.4572 +[titan] 2025-07-10 09:54:58,357 - root - INFO - lr: 3.3963e-05 gnorm: 1.19 [20:19:54< 1:41:07] +[titan] 2025-07-10 09:55:01,527 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:55:02,339 - root - INFO - step: 92350 loss: 16.1241 memory: 44.58GiB(31.99%) tps: 82,293 tflops: 284.01 mfu: 28.72% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.6673 +[titan] 2025-07-10 09:55:02,339 - root - INFO - lr: 3.3958e-05 gnorm: 1.02 [20:19:58< 1:41:03] +[titan] 2025-07-10 09:55:06,256 - root - INFO - step: 92355 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 83,660 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.7739 +[titan] 2025-07-10 09:55:06,256 - root - INFO - lr: 3.3953e-05 gnorm: 1.03 [20:20:02< 1:40:59] +[titan] 2025-07-10 09:55:10,150 - root - INFO - step: 92360 loss: 16.1235 memory: 44.58GiB(31.99%) tps: 84,156 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.6576 +[titan] 2025-07-10 09:55:10,151 - root - INFO - lr: 3.3948e-05 gnorm: 1.09 [20:20:06< 1:40:55] +[titan] 2025-07-10 09:55:14,144 - root - INFO - step: 92365 loss: 16.1822 memory: 44.58GiB(31.99%) tps: 82,053 tflops: 283.18 mfu: 28.63% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7117 +[titan] 2025-07-10 09:55:14,145 - root - INFO - lr: 3.3943e-05 gnorm: 1.02 [20:20:10< 1:40:51] +[titan] 2025-07-10 09:55:18,071 - root - INFO - step: 92370 loss: 16.3593 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.8642 +[titan] 2025-07-10 09:55:18,072 - root - INFO - lr: 3.3938e-05 gnorm: 1.02 [20:20:14< 1:40:47] +[titan] 2025-07-10 09:55:21,972 - root - INFO - step: 92375 loss: 16.0615 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4451 global_avg_mtp_loss: 13.6164 +[titan] 2025-07-10 09:55:21,973 - root - INFO - lr: 3.3933e-05 gnorm: 1.06 [20:20:18< 1:40:43] +[titan] 2025-07-10 09:55:25,891 - root - INFO - step: 92380 loss: 16.3804 memory: 44.58GiB(31.99%) tps: 83,623 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 09:55:25,892 - root - INFO - lr: 3.3928e-05 gnorm: 1.06 [20:20:21< 1:40:39] +[titan] 2025-07-10 09:55:29,804 - root - INFO - step: 92385 loss: 16.2591 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7838 +[titan] 2025-07-10 09:55:29,804 - root - INFO - lr: 3.3922e-05 gnorm: 1.01 [20:20:25< 1:40:35] +[titan] 2025-07-10 09:55:33,747 - root - INFO - step: 92390 loss: 16.6017 memory: 44.58GiB(31.99%) tps: 83,107 tflops: 286.82 mfu: 29.00% global_avg_ntp_loss: 2.5457 global_avg_mtp_loss: 14.0560 +[titan] 2025-07-10 09:55:33,747 - root - INFO - lr: 3.3917e-05 gnorm: 1.01 [20:20:29< 1:40:31] +[titan] 2025-07-10 09:55:37,658 - root - INFO - step: 92395 loss: 16.2663 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7882 +[titan] 2025-07-10 09:55:37,658 - root - INFO - lr: 3.3912e-05 gnorm: 1.05 [20:20:33< 1:40:27] +[titan] 2025-07-10 09:55:40,802 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:55:41,588 - root - INFO - step: 92400 loss: 16.1056 memory: 44.58GiB(31.99%) tps: 83,372 tflops: 287.73 mfu: 29.09% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.6599 +[titan] 2025-07-10 09:55:41,589 - root - INFO - lr: 3.3907e-05 gnorm: 1.06 [20:20:37< 1:40:23] +[titan] 2025-07-10 09:55:45,492 - root - INFO - step: 92405 loss: 15.7794 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.3914 global_avg_mtp_loss: 13.3880 +[titan] 2025-07-10 09:55:45,493 - root - INFO - lr: 3.3902e-05 gnorm: 1.04 [20:20:41< 1:40:19] +[titan] 2025-07-10 09:55:49,414 - root - INFO - step: 92410 loss: 16.3428 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-10 09:55:49,414 - root - INFO - lr: 3.3897e-05 gnorm: 1.04 [20:20:45< 1:40:15] +[titan] 2025-07-10 09:55:53,347 - root - INFO - step: 92415 loss: 15.9620 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4266 global_avg_mtp_loss: 13.5354 +[titan] 2025-07-10 09:55:53,347 - root - INFO - lr: 3.3892e-05 gnorm: 1.04 [20:20:49< 1:40:11] +[titan] 2025-07-10 09:55:57,264 - root - INFO - step: 92420 loss: 16.4264 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.9214 +[titan] 2025-07-10 09:55:57,264 - root - INFO - lr: 3.3887e-05 gnorm: 1.05 [20:20:53< 1:40:08] +[titan] 2025-07-10 09:56:01,176 - root - INFO - step: 92425 loss: 16.0509 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4491 global_avg_mtp_loss: 13.6018 +[titan] 2025-07-10 09:56:01,176 - root - INFO - lr: 3.3881e-05 gnorm: 1.06 [20:20:57< 1:40:04] +[titan] 2025-07-10 09:56:05,094 - root - INFO - step: 92430 loss: 15.8949 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4293 global_avg_mtp_loss: 13.4656 +[titan] 2025-07-10 09:56:05,094 - root - INFO - lr: 3.3876e-05 gnorm: 1.09 [20:21:01< 1:40:00] +[titan] 2025-07-10 09:56:09,007 - root - INFO - step: 92435 loss: 16.2665 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7872 +[titan] 2025-07-10 09:56:09,008 - root - INFO - lr: 3.3871e-05 gnorm: 1.06 [20:21:05< 1:39:56] +[titan] 2025-07-10 09:56:12,914 - root - INFO - step: 92440 loss: 16.3580 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4924 global_avg_mtp_loss: 13.8656 +[titan] 2025-07-10 09:56:12,914 - root - INFO - lr: 3.3866e-05 gnorm: 0.99 [20:21:08< 1:39:52] +[titan] 2025-07-10 09:56:16,817 - root - INFO - step: 92445 loss: 16.0745 memory: 44.58GiB(31.99%) tps: 83,954 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4361 global_avg_mtp_loss: 13.6385 +[titan] 2025-07-10 09:56:16,818 - root - INFO - lr: 3.3861e-05 gnorm: 1.01 [20:21:12< 1:39:48] +[titan] 2025-07-10 09:56:19,921 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:56:20,714 - root - INFO - step: 92450 loss: 16.1184 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4648 global_avg_mtp_loss: 13.6536 +[titan] 2025-07-10 09:56:20,714 - root - INFO - lr: 3.3856e-05 gnorm: 1.00 [20:21:16< 1:39:44] +[titan] 2025-07-10 09:56:24,617 - root - INFO - step: 92455 loss: 16.2628 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7796 +[titan] 2025-07-10 09:56:24,617 - root - INFO - lr: 3.3851e-05 gnorm: 1.03 [20:21:20< 1:39:40] +[titan] 2025-07-10 09:56:28,562 - root - INFO - step: 92460 loss: 16.2995 memory: 44.58GiB(31.99%) tps: 83,077 tflops: 286.71 mfu: 28.99% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8054 +[titan] 2025-07-10 09:56:28,562 - root - INFO - lr: 3.3846e-05 gnorm: 1.02 [20:21:24< 1:39:36] +[titan] 2025-07-10 09:56:32,476 - root - INFO - step: 92465 loss: 16.4887 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.5147 global_avg_mtp_loss: 13.9740 +[titan] 2025-07-10 09:56:32,477 - root - INFO - lr: 3.3841e-05 gnorm: 1.02 [20:21:28< 1:39:32] +[titan] 2025-07-10 09:56:36,374 - root - INFO - step: 92470 loss: 15.9947 memory: 44.58GiB(31.99%) tps: 84,091 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.4295 global_avg_mtp_loss: 13.5652 +[titan] 2025-07-10 09:56:36,374 - root - INFO - lr: 3.3836e-05 gnorm: 1.04 [20:21:32< 1:39:28] +[titan] 2025-07-10 09:56:40,286 - root - INFO - step: 92475 loss: 16.2231 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7492 +[titan] 2025-07-10 09:56:40,286 - root - INFO - lr: 3.3831e-05 gnorm: 1.04 [20:21:36< 1:39:24] +[titan] 2025-07-10 09:56:44,191 - root - INFO - step: 92480 loss: 16.2577 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7782 +[titan] 2025-07-10 09:56:44,192 - root - INFO - lr: 3.3826e-05 gnorm: 0.99 [20:21:40< 1:39:20] +[titan] 2025-07-10 09:56:48,125 - root - INFO - step: 92485 loss: 16.0082 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5664 +[titan] 2025-07-10 09:56:48,125 - root - INFO - lr: 3.3821e-05 gnorm: 1.00 [20:21:44< 1:39:16] +[titan] 2025-07-10 09:56:52,042 - root - INFO - step: 92490 loss: 16.4994 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.5212 global_avg_mtp_loss: 13.9782 +[titan] 2025-07-10 09:56:52,043 - root - INFO - lr: 3.3815e-05 gnorm: 1.01 [20:21:48< 1:39:12] +[titan] 2025-07-10 09:56:55,959 - root - INFO - step: 92495 loss: 16.3173 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.8325 +[titan] 2025-07-10 09:56:55,960 - root - INFO - lr: 3.3810e-05 gnorm: 1.02 [20:21:52< 1:39:08] +[titan] 2025-07-10 09:56:59,107 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:56:59,895 - root - INFO - step: 92500 loss: 16.1931 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7198 +[titan] 2025-07-10 09:56:59,896 - root - INFO - lr: 3.3805e-05 gnorm: 1.07 [20:21:55< 1:39:04] +[titan] 2025-07-10 09:57:03,801 - root - INFO - step: 92505 loss: 16.0881 memory: 44.58GiB(31.99%) tps: 83,912 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4551 global_avg_mtp_loss: 13.6331 +[titan] 2025-07-10 09:57:03,801 - root - INFO - lr: 3.3800e-05 gnorm: 1.10 [20:21:59< 1:39:00] +[titan] 2025-07-10 09:57:07,709 - root - INFO - step: 92510 loss: 16.1211 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4568 global_avg_mtp_loss: 13.6643 +[titan] 2025-07-10 09:57:07,710 - root - INFO - lr: 3.3795e-05 gnorm: 1.03 [20:22:03< 1:38:56] +[titan] 2025-07-10 09:57:11,623 - root - INFO - step: 92515 loss: 15.9879 memory: 44.58GiB(31.99%) tps: 83,744 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4470 global_avg_mtp_loss: 13.5409 +[titan] 2025-07-10 09:57:11,623 - root - INFO - lr: 3.3790e-05 gnorm: 1.04 [20:22:07< 1:38:52] +[titan] 2025-07-10 09:57:15,541 - root - INFO - step: 92520 loss: 16.3217 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.8281 +[titan] 2025-07-10 09:57:15,541 - root - INFO - lr: 3.3785e-05 gnorm: 1.03 [20:22:11< 1:38:48] +[titan] 2025-07-10 09:57:19,466 - root - INFO - step: 92525 loss: 16.0824 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4580 global_avg_mtp_loss: 13.6244 +[titan] 2025-07-10 09:57:19,466 - root - INFO - lr: 3.3780e-05 gnorm: 0.98 [20:22:15< 1:38:44] +[titan] 2025-07-10 09:57:23,370 - root - INFO - step: 92530 loss: 15.9050 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4262 global_avg_mtp_loss: 13.4788 +[titan] 2025-07-10 09:57:23,370 - root - INFO - lr: 3.3775e-05 gnorm: 1.04 [20:22:19< 1:38:40] +[titan] 2025-07-10 09:57:27,286 - root - INFO - step: 92535 loss: 16.3273 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8274 +[titan] 2025-07-10 09:57:27,287 - root - INFO - lr: 3.3770e-05 gnorm: 1.02 [20:22:23< 1:38:36] +[titan] 2025-07-10 09:57:31,240 - root - INFO - step: 92540 loss: 16.2002 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.7230 +[titan] 2025-07-10 09:57:31,240 - root - INFO - lr: 3.3765e-05 gnorm: 1.07 [20:22:27< 1:38:32] +[titan] 2025-07-10 09:57:35,148 - root - INFO - step: 92545 loss: 16.3212 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8227 +[titan] 2025-07-10 09:57:35,148 - root - INFO - lr: 3.3760e-05 gnorm: 1.09 [20:22:31< 1:38:28] +[titan] 2025-07-10 09:57:38,266 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:57:39,058 - root - INFO - step: 92550 loss: 16.3419 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4942 global_avg_mtp_loss: 13.8477 +[titan] 2025-07-10 09:57:39,058 - root - INFO - lr: 3.3755e-05 gnorm: 1.02 [20:22:35< 1:38:24] +[titan] 2025-07-10 09:57:42,966 - root - INFO - step: 92555 loss: 16.2782 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.8022 +[titan] 2025-07-10 09:57:42,967 - root - INFO - lr: 3.3750e-05 gnorm: 1.02 [20:22:39< 1:38:20] +[titan] 2025-07-10 09:57:46,877 - root - INFO - step: 92560 loss: 16.4134 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.5132 global_avg_mtp_loss: 13.9002 +[titan] 2025-07-10 09:57:46,877 - root - INFO - lr: 3.3745e-05 gnorm: 1.00 [20:22:42< 1:38:16] +[titan] 2025-07-10 09:57:50,801 - root - INFO - step: 92565 loss: 16.2864 memory: 44.58GiB(31.99%) tps: 83,515 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4825 global_avg_mtp_loss: 13.8039 +[titan] 2025-07-10 09:57:50,801 - root - INFO - lr: 3.3740e-05 gnorm: 1.05 [20:22:46< 1:38:12] +[titan] 2025-07-10 09:57:54,718 - root - INFO - step: 92570 loss: 16.1290 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.6518 +[titan] 2025-07-10 09:57:54,719 - root - INFO - lr: 3.3735e-05 gnorm: 1.05 [20:22:50< 1:38:08] +[titan] 2025-07-10 09:57:58,648 - root - INFO - step: 92575 loss: 16.2544 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7824 +[titan] 2025-07-10 09:57:58,649 - root - INFO - lr: 3.3730e-05 gnorm: 1.00 [20:22:54< 1:38:05] +[titan] 2025-07-10 09:58:02,594 - root - INFO - step: 92580 loss: 16.2697 memory: 44.58GiB(31.99%) tps: 83,054 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7830 +[titan] 2025-07-10 09:58:02,594 - root - INFO - lr: 3.3725e-05 gnorm: 1.03 [20:22:58< 1:38:01] +[titan] 2025-07-10 09:58:06,508 - root - INFO - step: 92585 loss: 16.4083 memory: 44.58GiB(31.99%) tps: 83,739 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.8987 +[titan] 2025-07-10 09:58:06,508 - root - INFO - lr: 3.3720e-05 gnorm: 1.10 [20:23:02< 1:37:57] +[titan] 2025-07-10 09:58:10,419 - root - INFO - step: 92590 loss: 16.2091 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7354 +[titan] 2025-07-10 09:58:10,419 - root - INFO - lr: 3.3715e-05 gnorm: 1.07 [20:23:06< 1:37:53] +[titan] 2025-07-10 09:58:14,366 - root - INFO - step: 92595 loss: 15.9602 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.4327 global_avg_mtp_loss: 13.5275 +[titan] 2025-07-10 09:58:14,366 - root - INFO - lr: 3.3710e-05 gnorm: 1.05 [20:23:10< 1:37:49] +[titan] 2025-07-10 09:58:17,476 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:58:18,263 - root - INFO - step: 92600 loss: 15.8797 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.4300 global_avg_mtp_loss: 13.4496 +[titan] 2025-07-10 09:58:18,264 - root - INFO - lr: 3.3705e-05 gnorm: 1.09 [20:23:14< 1:37:45] +[titan] 2025-07-10 09:58:22,186 - root - INFO - step: 92605 loss: 16.4058 memory: 44.58GiB(31.99%) tps: 83,551 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8993 +[titan] 2025-07-10 09:58:22,186 - root - INFO - lr: 3.3700e-05 gnorm: 1.01 [20:23:18< 1:37:41] +[titan] 2025-07-10 09:58:26,090 - root - INFO - step: 92610 loss: 16.2160 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7420 +[titan] 2025-07-10 09:58:26,090 - root - INFO - lr: 3.3695e-05 gnorm: 1.06 [20:23:22< 1:37:37] +[titan] 2025-07-10 09:58:29,989 - root - INFO - step: 92615 loss: 16.0402 memory: 44.58GiB(31.99%) tps: 84,044 tflops: 290.05 mfu: 29.33% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.6006 +[titan] 2025-07-10 09:58:29,990 - root - INFO - lr: 3.3690e-05 gnorm: 1.05 [20:23:26< 1:37:33] +[titan] 2025-07-10 09:58:33,909 - root - INFO - step: 92620 loss: 16.2633 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7833 +[titan] 2025-07-10 09:58:33,910 - root - INFO - lr: 3.3685e-05 gnorm: 1.05 [20:23:29< 1:37:29] +[titan] 2025-07-10 09:58:37,860 - root - INFO - step: 92625 loss: 16.0952 memory: 44.58GiB(31.99%) tps: 82,957 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4584 global_avg_mtp_loss: 13.6367 +[titan] 2025-07-10 09:58:37,860 - root - INFO - lr: 3.3680e-05 gnorm: 0.99 [20:23:33< 1:37:25] +[titan] 2025-07-10 09:58:41,776 - root - INFO - step: 92630 loss: 16.2667 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4814 global_avg_mtp_loss: 13.7854 +[titan] 2025-07-10 09:58:41,776 - root - INFO - lr: 3.3675e-05 gnorm: 1.01 [20:23:37< 1:37:21] +[titan] 2025-07-10 09:58:45,722 - root - INFO - step: 92635 loss: 15.9798 memory: 44.58GiB(31.99%) tps: 83,048 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4417 global_avg_mtp_loss: 13.5381 +[titan] 2025-07-10 09:58:45,722 - root - INFO - lr: 3.3670e-05 gnorm: 1.04 [20:23:41< 1:37:17] +[titan] 2025-07-10 09:58:49,659 - root - INFO - step: 92640 loss: 16.4104 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.8941 +[titan] 2025-07-10 09:58:49,660 - root - INFO - lr: 3.3665e-05 gnorm: 1.02 [20:23:45< 1:37:13] +[titan] 2025-07-10 09:58:53,575 - root - INFO - step: 92645 loss: 16.2743 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 09:58:53,576 - root - INFO - lr: 3.3660e-05 gnorm: 1.05 [20:23:49< 1:37:09] +[titan] 2025-07-10 09:58:56,700 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:58:57,509 - root - INFO - step: 92650 loss: 16.1688 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.6979 +[titan] 2025-07-10 09:58:57,509 - root - INFO - lr: 3.3655e-05 gnorm: 1.09 [20:23:53< 1:37:05] +[titan] 2025-07-10 09:59:01,421 - root - INFO - step: 92655 loss: 16.2067 memory: 44.58GiB(31.99%) tps: 83,767 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7251 +[titan] 2025-07-10 09:59:01,421 - root - INFO - lr: 3.3650e-05 gnorm: 1.05 [20:23:57< 1:37:01] +[titan] 2025-07-10 09:59:05,345 - root - INFO - step: 92660 loss: 16.0305 memory: 44.58GiB(31.99%) tps: 83,518 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5887 +[titan] 2025-07-10 09:59:05,345 - root - INFO - lr: 3.3645e-05 gnorm: 1.04 [20:24:01< 1:36:57] +[titan] 2025-07-10 09:59:09,285 - root - INFO - step: 92665 loss: 16.1851 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4668 global_avg_mtp_loss: 13.7183 +[titan] 2025-07-10 09:59:09,285 - root - INFO - lr: 3.3641e-05 gnorm: 1.01 [20:24:05< 1:36:53] +[titan] 2025-07-10 09:59:13,299 - root - INFO - step: 92670 loss: 15.7036 memory: 44.58GiB(31.99%) tps: 81,651 tflops: 281.79 mfu: 28.49% global_avg_ntp_loss: 2.3966 global_avg_mtp_loss: 13.3071 +[titan] 2025-07-10 09:59:13,299 - root - INFO - lr: 3.3636e-05 gnorm: 1.05 [20:24:09< 1:36:49] +[titan] 2025-07-10 09:59:15,022 - root - INFO - Dumping profiler traces at step 92672 +[titan] 2025-07-10 09:59:15,054 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 09:59:17,458 - root - INFO - step: 92675 loss: 16.4344 memory: 44.58GiB(31.99%) tps: 78,798 tflops: 271.94 mfu: 27.50% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.9296 +[titan] 2025-07-10 09:59:17,458 - root - INFO - lr: 3.3631e-05 gnorm: 1.02 [20:24:13< 1:36:45] +[titan] 2025-07-10 09:59:21,388 - root - INFO - step: 92680 loss: 16.2199 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.7402 +[titan] 2025-07-10 09:59:21,388 - root - INFO - lr: 3.3626e-05 gnorm: 1.02 [20:24:17< 1:36:41] +[titan] 2025-07-10 09:59:25,288 - root - INFO - step: 92685 loss: 16.2751 memory: 44.58GiB(31.99%) tps: 84,028 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.7805 +[titan] 2025-07-10 09:59:25,288 - root - INFO - lr: 3.3621e-05 gnorm: 0.98 [20:24:21< 1:36:37] +[titan] 2025-07-10 09:59:29,191 - root - INFO - step: 92690 loss: 15.9160 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4254 global_avg_mtp_loss: 13.4906 +[titan] 2025-07-10 09:59:29,191 - root - INFO - lr: 3.3616e-05 gnorm: 1.04 [20:24:25< 1:36:33] +[titan] 2025-07-10 09:59:33,114 - root - INFO - step: 92695 loss: 15.8624 memory: 44.58GiB(31.99%) tps: 83,540 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4043 global_avg_mtp_loss: 13.4580 +[titan] 2025-07-10 09:59:33,114 - root - INFO - lr: 3.3611e-05 gnorm: 1.04 [20:24:29< 1:36:29] +[titan] 2025-07-10 09:59:36,240 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 09:59:37,035 - root - INFO - step: 92700 loss: 16.2588 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7835 +[titan] 2025-07-10 09:59:37,036 - root - INFO - lr: 3.3606e-05 gnorm: 1.05 [20:24:33< 1:36:25] +[titan] 2025-07-10 09:59:40,932 - root - INFO - step: 92705 loss: 16.1366 memory: 44.58GiB(31.99%) tps: 84,108 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4557 global_avg_mtp_loss: 13.6809 +[titan] 2025-07-10 09:59:40,932 - root - INFO - lr: 3.3601e-05 gnorm: 1.04 [20:24:36< 1:36:21] +[titan] 2025-07-10 09:59:44,857 - root - INFO - step: 92710 loss: 16.1518 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4524 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 09:59:44,858 - root - INFO - lr: 3.3596e-05 gnorm: 1.03 [20:24:40< 1:36:17] +[titan] 2025-07-10 09:59:48,745 - root - INFO - step: 92715 loss: 16.2229 memory: 44.58GiB(31.99%) tps: 84,289 tflops: 290.90 mfu: 29.41% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7365 +[titan] 2025-07-10 09:59:48,746 - root - INFO - lr: 3.3591e-05 gnorm: 1.08 [20:24:44< 1:36:14] +[titan] 2025-07-10 09:59:52,645 - root - INFO - step: 92720 loss: 16.1935 memory: 44.58GiB(31.99%) tps: 84,040 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7242 +[titan] 2025-07-10 09:59:52,645 - root - INFO - lr: 3.3586e-05 gnorm: 1.06 [20:24:48< 1:36:10] +[titan] 2025-07-10 09:59:56,557 - root - INFO - step: 92725 loss: 16.0350 memory: 44.58GiB(31.99%) tps: 83,779 tflops: 289.13 mfu: 29.24% global_avg_ntp_loss: 2.4175 global_avg_mtp_loss: 13.6175 +[titan] 2025-07-10 09:59:56,557 - root - INFO - lr: 3.3581e-05 gnorm: 1.08 [20:24:52< 1:36:06] +[titan] 2025-07-10 10:00:00,480 - root - INFO - step: 92730 loss: 16.3840 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8841 +[titan] 2025-07-10 10:00:00,481 - root - INFO - lr: 3.3577e-05 gnorm: 1.01 [20:24:56< 1:36:02] +[titan] 2025-07-10 10:00:04,398 - root - INFO - step: 92735 loss: 16.2378 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.7505 +[titan] 2025-07-10 10:00:04,398 - root - INFO - lr: 3.3572e-05 gnorm: 1.05 [20:25:00< 1:35:58] +[titan] 2025-07-10 10:00:08,349 - root - INFO - step: 92740 loss: 16.2304 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4928 global_avg_mtp_loss: 13.7377 +[titan] 2025-07-10 10:00:08,349 - root - INFO - lr: 3.3567e-05 gnorm: 1.00 [20:25:04< 1:35:54] +[titan] 2025-07-10 10:00:12,281 - root - INFO - step: 92745 loss: 16.0912 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.6423 +[titan] 2025-07-10 10:00:12,282 - root - INFO - lr: 3.3562e-05 gnorm: 1.06 [20:25:08< 1:35:50] +[titan] 2025-07-10 10:00:15,385 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:00:16,175 - root - INFO - step: 92750 loss: 16.3066 memory: 44.58GiB(31.99%) tps: 84,161 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.8194 +[titan] 2025-07-10 10:00:16,176 - root - INFO - lr: 3.3557e-05 gnorm: 1.01 [20:25:12< 1:35:46] +[titan] 2025-07-10 10:00:20,089 - root - INFO - step: 92755 loss: 16.0588 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4429 global_avg_mtp_loss: 13.6159 +[titan] 2025-07-10 10:00:20,089 - root - INFO - lr: 3.3552e-05 gnorm: 1.05 [20:25:16< 1:35:42] +[titan] 2025-07-10 10:00:23,987 - root - INFO - step: 92760 loss: 16.3378 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8418 +[titan] 2025-07-10 10:00:23,987 - root - INFO - lr: 3.3547e-05 gnorm: 1.06 [20:25:20< 1:35:38] +[titan] 2025-07-10 10:00:27,914 - root - INFO - step: 92765 loss: 16.1796 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.7079 +[titan] 2025-07-10 10:00:27,915 - root - INFO - lr: 3.3542e-05 gnorm: 1.05 [20:25:23< 1:35:34] +[titan] 2025-07-10 10:00:31,811 - root - INFO - step: 92770 loss: 16.0385 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4368 global_avg_mtp_loss: 13.6017 +[titan] 2025-07-10 10:00:31,811 - root - INFO - lr: 3.3537e-05 gnorm: 1.02 [20:25:27< 1:35:30] +[titan] 2025-07-10 10:00:35,724 - root - INFO - step: 92775 loss: 16.1057 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.4470 global_avg_mtp_loss: 13.6587 +[titan] 2025-07-10 10:00:35,725 - root - INFO - lr: 3.3533e-05 gnorm: 1.09 [20:25:31< 1:35:26] +[titan] 2025-07-10 10:00:39,623 - root - INFO - step: 92780 loss: 16.3216 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8249 +[titan] 2025-07-10 10:00:39,623 - root - INFO - lr: 3.3528e-05 gnorm: 1.05 [20:25:35< 1:35:22] +[titan] 2025-07-10 10:00:43,535 - root - INFO - step: 92785 loss: 16.2179 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 10:00:43,535 - root - INFO - lr: 3.3523e-05 gnorm: 1.04 [20:25:39< 1:35:18] +[titan] 2025-07-10 10:00:47,433 - root - INFO - step: 92790 loss: 16.3746 memory: 44.58GiB(31.99%) tps: 84,069 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.5008 global_avg_mtp_loss: 13.8737 +[titan] 2025-07-10 10:00:47,433 - root - INFO - lr: 3.3518e-05 gnorm: 1.10 [20:25:43< 1:35:14] +[titan] 2025-07-10 10:00:51,355 - root - INFO - step: 92795 loss: 16.2778 memory: 44.58GiB(31.99%) tps: 83,562 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4952 global_avg_mtp_loss: 13.7827 +[titan] 2025-07-10 10:00:51,355 - root - INFO - lr: 3.3513e-05 gnorm: 1.07 [20:25:47< 1:35:10] +[titan] 2025-07-10 10:00:54,467 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:00:55,258 - root - INFO - step: 92800 loss: 16.1403 memory: 44.58GiB(31.99%) tps: 83,957 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6761 +[titan] 2025-07-10 10:00:55,259 - root - INFO - lr: 3.3508e-05 gnorm: 1.04 [20:25:51< 1:35:06] +[titan] 2025-07-10 10:00:59,178 - root - INFO - step: 92805 loss: 15.9845 memory: 44.58GiB(31.99%) tps: 83,612 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4296 global_avg_mtp_loss: 13.5548 +[titan] 2025-07-10 10:00:59,178 - root - INFO - lr: 3.3503e-05 gnorm: 1.05 [20:25:55< 1:35:02] +[titan] 2025-07-10 10:01:03,084 - root - INFO - step: 92810 loss: 15.6641 memory: 44.58GiB(31.99%) tps: 83,897 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.3717 global_avg_mtp_loss: 13.2924 +[titan] 2025-07-10 10:01:03,084 - root - INFO - lr: 3.3499e-05 gnorm: 1.04 [20:25:59< 1:34:58] +[titan] 2025-07-10 10:01:07,009 - root - INFO - step: 92815 loss: 16.1412 memory: 44.58GiB(31.99%) tps: 83,489 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.6879 +[titan] 2025-07-10 10:01:07,009 - root - INFO - lr: 3.3494e-05 gnorm: 1.01 [20:26:03< 1:34:54] +[titan] 2025-07-10 10:01:10,939 - root - INFO - step: 92820 loss: 16.1873 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.7270 +[titan] 2025-07-10 10:01:10,939 - root - INFO - lr: 3.3489e-05 gnorm: 1.08 [20:26:06< 1:34:50] +[titan] 2025-07-10 10:01:14,864 - root - INFO - step: 92825 loss: 16.0476 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4407 global_avg_mtp_loss: 13.6069 +[titan] 2025-07-10 10:01:14,864 - root - INFO - lr: 3.3484e-05 gnorm: 1.02 [20:26:10< 1:34:46] +[titan] 2025-07-10 10:01:18,803 - root - INFO - step: 92830 loss: 16.0695 memory: 44.58GiB(31.99%) tps: 83,193 tflops: 287.11 mfu: 29.03% global_avg_ntp_loss: 2.4437 global_avg_mtp_loss: 13.6258 +[titan] 2025-07-10 10:01:18,803 - root - INFO - lr: 3.3479e-05 gnorm: 1.03 [20:26:14< 1:34:42] +[titan] 2025-07-10 10:01:22,704 - root - INFO - step: 92835 loss: 16.2076 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7352 +[titan] 2025-07-10 10:01:22,705 - root - INFO - lr: 3.3474e-05 gnorm: 1.06 [20:26:18< 1:34:38] +[titan] 2025-07-10 10:01:26,606 - root - INFO - step: 92840 loss: 16.1491 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.6745 +[titan] 2025-07-10 10:01:26,606 - root - INFO - lr: 3.3470e-05 gnorm: 1.04 [20:26:22< 1:34:34] +[titan] 2025-07-10 10:01:30,541 - root - INFO - step: 92845 loss: 16.1868 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.7278 +[titan] 2025-07-10 10:01:30,541 - root - INFO - lr: 3.3465e-05 gnorm: 1.05 [20:26:26< 1:34:30] +[titan] 2025-07-10 10:01:33,661 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:01:34,463 - root - INFO - step: 92850 loss: 16.4315 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.5273 global_avg_mtp_loss: 13.9042 +[titan] 2025-07-10 10:01:34,463 - root - INFO - lr: 3.3460e-05 gnorm: 1.04 [20:26:30< 1:34:26] +[titan] 2025-07-10 10:01:38,388 - root - INFO - step: 92855 loss: 16.0580 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6076 +[titan] 2025-07-10 10:01:38,388 - root - INFO - lr: 3.3455e-05 gnorm: 1.04 [20:26:34< 1:34:22] +[titan] 2025-07-10 10:01:42,338 - root - INFO - step: 92860 loss: 16.1816 memory: 44.58GiB(31.99%) tps: 82,974 tflops: 286.36 mfu: 28.95% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.7165 +[titan] 2025-07-10 10:01:42,338 - root - INFO - lr: 3.3450e-05 gnorm: 1.04 [20:26:38< 1:34:18] +[titan] 2025-07-10 10:01:46,257 - root - INFO - step: 92865 loss: 16.3382 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8371 +[titan] 2025-07-10 10:01:46,257 - root - INFO - lr: 3.3446e-05 gnorm: 1.09 [20:26:42< 1:34:15] +[titan] 2025-07-10 10:01:50,179 - root - INFO - step: 92870 loss: 16.4574 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.5197 global_avg_mtp_loss: 13.9377 +[titan] 2025-07-10 10:01:50,179 - root - INFO - lr: 3.3441e-05 gnorm: 1.04 [20:26:46< 1:34:11] +[titan] 2025-07-10 10:01:54,105 - root - INFO - step: 92875 loss: 16.3307 memory: 44.58GiB(31.99%) tps: 83,484 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8379 +[titan] 2025-07-10 10:01:54,105 - root - INFO - lr: 3.3436e-05 gnorm: 1.03 [20:26:50< 1:34:07] +[titan] 2025-07-10 10:01:58,021 - root - INFO - step: 92880 loss: 16.1367 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.6642 +[titan] 2025-07-10 10:01:58,021 - root - INFO - lr: 3.3431e-05 gnorm: 1.18 [20:26:54< 1:34:03] +[titan] 2025-07-10 10:02:01,935 - root - INFO - step: 92885 loss: 16.1034 memory: 44.58GiB(31.99%) tps: 83,723 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6441 +[titan] 2025-07-10 10:02:01,935 - root - INFO - lr: 3.3426e-05 gnorm: 1.01 [20:26:57< 1:33:59] +[titan] 2025-07-10 10:02:05,859 - root - INFO - step: 92890 loss: 15.8281 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4003 global_avg_mtp_loss: 13.4277 +[titan] 2025-07-10 10:02:05,859 - root - INFO - lr: 3.3422e-05 gnorm: 1.01 [20:27:01< 1:33:55] +[titan] 2025-07-10 10:02:09,786 - root - INFO - step: 92895 loss: 15.9777 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4345 global_avg_mtp_loss: 13.5432 +[titan] 2025-07-10 10:02:09,786 - root - INFO - lr: 3.3417e-05 gnorm: 1.05 [20:27:05< 1:33:51] +[titan] 2025-07-10 10:02:12,915 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:02:13,718 - root - INFO - step: 92900 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7197 +[titan] 2025-07-10 10:02:13,718 - root - INFO - lr: 3.3412e-05 gnorm: 0.99 [20:27:09< 1:33:47] +[titan] 2025-07-10 10:02:17,657 - root - INFO - step: 92905 loss: 16.0649 memory: 44.58GiB(31.99%) tps: 83,207 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6055 +[titan] 2025-07-10 10:02:17,657 - root - INFO - lr: 3.3407e-05 gnorm: 1.07 [20:27:13< 1:33:43] +[titan] 2025-07-10 10:02:21,577 - root - INFO - step: 92910 loss: 16.1858 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.7211 +[titan] 2025-07-10 10:02:21,577 - root - INFO - lr: 3.3402e-05 gnorm: 1.06 [20:27:17< 1:33:39] +[titan] 2025-07-10 10:02:25,474 - root - INFO - step: 92915 loss: 16.0685 memory: 44.58GiB(31.99%) tps: 84,079 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.6196 +[titan] 2025-07-10 10:02:25,475 - root - INFO - lr: 3.3398e-05 gnorm: 1.06 [20:27:21< 1:33:35] +[titan] 2025-07-10 10:02:29,377 - root - INFO - step: 92920 loss: 16.1002 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.6511 +[titan] 2025-07-10 10:02:29,377 - root - INFO - lr: 3.3393e-05 gnorm: 1.05 [20:27:25< 1:33:31] +[titan] 2025-07-10 10:02:33,288 - root - INFO - step: 92925 loss: 16.2303 memory: 44.58GiB(31.99%) tps: 83,800 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7570 +[titan] 2025-07-10 10:02:33,288 - root - INFO - lr: 3.3388e-05 gnorm: 0.99 [20:27:29< 1:33:27] +[titan] 2025-07-10 10:02:37,198 - root - INFO - step: 92930 loss: 16.3091 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.8228 +[titan] 2025-07-10 10:02:37,198 - root - INFO - lr: 3.3383e-05 gnorm: 1.01 [20:27:33< 1:33:23] +[titan] 2025-07-10 10:02:41,119 - root - INFO - step: 92935 loss: 16.1797 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7013 +[titan] 2025-07-10 10:02:41,119 - root - INFO - lr: 3.3379e-05 gnorm: 1.03 [20:27:37< 1:33:19] +[titan] 2025-07-10 10:02:45,054 - root - INFO - step: 92940 loss: 16.0614 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4432 global_avg_mtp_loss: 13.6182 +[titan] 2025-07-10 10:02:45,055 - root - INFO - lr: 3.3374e-05 gnorm: 1.08 [20:27:41< 1:33:15] +[titan] 2025-07-10 10:02:48,957 - root - INFO - step: 92945 loss: 15.9271 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4331 global_avg_mtp_loss: 13.4940 +[titan] 2025-07-10 10:02:48,957 - root - INFO - lr: 3.3369e-05 gnorm: 1.03 [20:27:44< 1:33:11] +[titan] 2025-07-10 10:02:52,088 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:02:52,875 - root - INFO - step: 92950 loss: 16.3538 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.5040 global_avg_mtp_loss: 13.8498 +[titan] 2025-07-10 10:02:52,875 - root - INFO - lr: 3.3364e-05 gnorm: 1.09 [20:27:48< 1:33:07] +[titan] 2025-07-10 10:02:56,797 - root - INFO - step: 92955 loss: 16.3221 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 10:02:56,797 - root - INFO - lr: 3.3360e-05 gnorm: 1.05 [20:27:52< 1:33:03] +[titan] 2025-07-10 10:03:00,713 - root - INFO - step: 92960 loss: 16.1615 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4703 global_avg_mtp_loss: 13.6912 +[titan] 2025-07-10 10:03:00,713 - root - INFO - lr: 3.3355e-05 gnorm: 1.06 [20:27:56< 1:32:59] +[titan] 2025-07-10 10:03:04,615 - root - INFO - step: 92965 loss: 15.9694 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4291 global_avg_mtp_loss: 13.5402 +[titan] 2025-07-10 10:03:04,615 - root - INFO - lr: 3.3350e-05 gnorm: 1.16 [20:28:00< 1:32:55] +[titan] 2025-07-10 10:03:08,538 - root - INFO - step: 92970 loss: 16.1153 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6583 +[titan] 2025-07-10 10:03:08,539 - root - INFO - lr: 3.3345e-05 gnorm: 1.03 [20:28:04< 1:32:51] +[titan] 2025-07-10 10:03:12,452 - root - INFO - step: 92975 loss: 15.9485 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4266 global_avg_mtp_loss: 13.5219 +[titan] 2025-07-10 10:03:12,452 - root - INFO - lr: 3.3341e-05 gnorm: 1.07 [20:28:08< 1:32:47] +[titan] 2025-07-10 10:03:16,407 - root - INFO - step: 92980 loss: 15.9887 memory: 44.58GiB(31.99%) tps: 82,867 tflops: 285.99 mfu: 28.92% global_avg_ntp_loss: 2.4334 global_avg_mtp_loss: 13.5554 +[titan] 2025-07-10 10:03:16,407 - root - INFO - lr: 3.3336e-05 gnorm: 1.03 [20:28:12< 1:32:43] +[titan] 2025-07-10 10:03:20,318 - root - INFO - step: 92985 loss: 16.5940 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5445 global_avg_mtp_loss: 14.0495 +[titan] 2025-07-10 10:03:20,318 - root - INFO - lr: 3.3331e-05 gnorm: 1.02 [20:28:16< 1:32:39] +[titan] 2025-07-10 10:03:24,239 - root - INFO - step: 92990 loss: 15.9177 memory: 44.58GiB(31.99%) tps: 83,571 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4022 global_avg_mtp_loss: 13.5155 +[titan] 2025-07-10 10:03:24,239 - root - INFO - lr: 3.3326e-05 gnorm: 1.05 [20:28:20< 1:32:35] +[titan] 2025-07-10 10:03:28,192 - root - INFO - step: 92995 loss: 16.4159 memory: 44.58GiB(31.99%) tps: 82,907 tflops: 286.13 mfu: 28.93% global_avg_ntp_loss: 2.5058 global_avg_mtp_loss: 13.9101 +[titan] 2025-07-10 10:03:28,192 - root - INFO - lr: 3.3322e-05 gnorm: 1.06 [20:28:24< 1:32:31] +[titan] 2025-07-10 10:03:31,315 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:03:32,121 - root - INFO - step: 93000 loss: 15.8217 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4039 global_avg_mtp_loss: 13.4178 +[titan] 2025-07-10 10:03:32,121 - root - INFO - lr: 3.3317e-05 gnorm: 1.03 [20:28:28< 1:32:27] +[titan] 2025-07-10 10:03:36,073 - root - INFO - step: 93005 loss: 16.2934 memory: 44.58GiB(31.99%) tps: 82,926 tflops: 286.19 mfu: 28.94% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.8026 +[titan] 2025-07-10 10:03:36,073 - root - INFO - lr: 3.3312e-05 gnorm: 1.02 [20:28:32< 1:32:23] +[titan] 2025-07-10 10:03:39,969 - root - INFO - step: 93010 loss: 16.3175 memory: 44.58GiB(31.99%) tps: 84,110 tflops: 290.28 mfu: 29.35% global_avg_ntp_loss: 2.4908 global_avg_mtp_loss: 13.8267 +[titan] 2025-07-10 10:03:39,969 - root - INFO - lr: 3.3307e-05 gnorm: 1.03 [20:28:35< 1:32:19] +[titan] 2025-07-10 10:03:43,906 - root - INFO - step: 93015 loss: 16.3111 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.8249 +[titan] 2025-07-10 10:03:43,906 - root - INFO - lr: 3.3303e-05 gnorm: 1.01 [20:28:39< 1:32:16] +[titan] 2025-07-10 10:03:47,846 - root - INFO - step: 93020 loss: 16.3032 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4874 global_avg_mtp_loss: 13.8158 +[titan] 2025-07-10 10:03:47,847 - root - INFO - lr: 3.3298e-05 gnorm: 1.03 [20:28:43< 1:32:12] +[titan] 2025-07-10 10:03:51,795 - root - INFO - step: 93025 loss: 16.3563 memory: 44.58GiB(31.99%) tps: 83,000 tflops: 286.45 mfu: 28.96% global_avg_ntp_loss: 2.5186 global_avg_mtp_loss: 13.8376 +[titan] 2025-07-10 10:03:51,795 - root - INFO - lr: 3.3293e-05 gnorm: 1.00 [20:28:47< 1:32:08] +[titan] 2025-07-10 10:03:55,711 - root - INFO - step: 93030 loss: 16.1253 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6662 +[titan] 2025-07-10 10:03:55,712 - root - INFO - lr: 3.3289e-05 gnorm: 1.05 [20:28:51< 1:32:04] +[titan] 2025-07-10 10:03:59,659 - root - INFO - step: 93035 loss: 16.2315 memory: 44.58GiB(31.99%) tps: 83,021 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7553 +[titan] 2025-07-10 10:03:59,659 - root - INFO - lr: 3.3284e-05 gnorm: 1.05 [20:28:55< 1:32:00] +[titan] 2025-07-10 10:04:03,580 - root - INFO - step: 93040 loss: 16.2939 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4925 global_avg_mtp_loss: 13.8014 +[titan] 2025-07-10 10:04:03,580 - root - INFO - lr: 3.3279e-05 gnorm: 1.07 [20:28:59< 1:31:56] +[titan] 2025-07-10 10:04:07,490 - root - INFO - step: 93045 loss: 16.1997 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.7395 +[titan] 2025-07-10 10:04:07,491 - root - INFO - lr: 3.3275e-05 gnorm: 1.02 [20:29:03< 1:31:52] +[titan] 2025-07-10 10:04:10,615 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:04:11,409 - root - INFO - step: 93050 loss: 16.2048 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.7306 +[titan] 2025-07-10 10:04:11,409 - root - INFO - lr: 3.3270e-05 gnorm: 1.01 [20:29:07< 1:31:48] +[titan] 2025-07-10 10:04:15,378 - root - INFO - step: 93055 loss: 16.0075 memory: 44.58GiB(31.99%) tps: 82,559 tflops: 284.92 mfu: 28.81% global_avg_ntp_loss: 2.4500 global_avg_mtp_loss: 13.5576 +[titan] 2025-07-10 10:04:15,379 - root - INFO - lr: 3.3265e-05 gnorm: 1.04 [20:29:11< 1:31:44] +[titan] 2025-07-10 10:04:19,317 - root - INFO - step: 93060 loss: 16.4744 memory: 44.58GiB(31.99%) tps: 83,205 tflops: 287.16 mfu: 29.03% global_avg_ntp_loss: 2.5167 global_avg_mtp_loss: 13.9577 +[titan] 2025-07-10 10:04:19,317 - root - INFO - lr: 3.3261e-05 gnorm: 1.01 [20:29:15< 1:31:40] +[titan] 2025-07-10 10:04:23,264 - root - INFO - step: 93065 loss: 15.9031 memory: 44.58GiB(31.99%) tps: 83,025 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.4275 global_avg_mtp_loss: 13.4756 +[titan] 2025-07-10 10:04:23,264 - root - INFO - lr: 3.3256e-05 gnorm: 1.07 [20:29:19< 1:31:36] +[titan] 2025-07-10 10:04:27,180 - root - INFO - step: 93070 loss: 16.0790 memory: 44.58GiB(31.99%) tps: 83,681 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4414 global_avg_mtp_loss: 13.6376 +[titan] 2025-07-10 10:04:27,181 - root - INFO - lr: 3.3251e-05 gnorm: 1.05 [20:29:23< 1:31:32] +[titan] 2025-07-10 10:04:31,081 - root - INFO - step: 93075 loss: 15.9487 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4394 global_avg_mtp_loss: 13.5093 +[titan] 2025-07-10 10:04:31,081 - root - INFO - lr: 3.3247e-05 gnorm: 1.03 [20:29:27< 1:31:28] +[titan] 2025-07-10 10:04:35,024 - root - INFO - step: 93080 loss: 16.3177 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.85 mfu: 29.00% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8261 +[titan] 2025-07-10 10:04:35,024 - root - INFO - lr: 3.3242e-05 gnorm: 0.99 [20:29:31< 1:31:24] +[titan] 2025-07-10 10:04:38,941 - root - INFO - step: 93085 loss: 16.2152 memory: 44.58GiB(31.99%) tps: 83,671 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4741 global_avg_mtp_loss: 13.7411 +[titan] 2025-07-10 10:04:38,941 - root - INFO - lr: 3.3237e-05 gnorm: 0.99 [20:29:34< 1:31:20] +[titan] 2025-07-10 10:04:42,866 - root - INFO - step: 93090 loss: 16.1621 memory: 44.58GiB(31.99%) tps: 83,494 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.6949 +[titan] 2025-07-10 10:04:42,866 - root - INFO - lr: 3.3233e-05 gnorm: 1.03 [20:29:38< 1:31:16] +[titan] 2025-07-10 10:04:46,777 - root - INFO - step: 93095 loss: 16.3303 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8371 +[titan] 2025-07-10 10:04:46,777 - root - INFO - lr: 3.3228e-05 gnorm: 1.02 [20:29:42< 1:31:12] +[titan] 2025-07-10 10:04:49,899 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:04:50,685 - root - INFO - step: 93100 loss: 16.3694 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8688 +[titan] 2025-07-10 10:04:50,686 - root - INFO - lr: 3.3223e-05 gnorm: 1.02 [20:29:46< 1:31:08] +[titan] 2025-07-10 10:04:54,576 - root - INFO - step: 93105 loss: 16.1410 memory: 44.58GiB(31.99%) tps: 84,227 tflops: 290.68 mfu: 29.39% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6792 +[titan] 2025-07-10 10:04:54,576 - root - INFO - lr: 3.3219e-05 gnorm: 0.99 [20:29:50< 1:31:04] +[titan] 2025-07-10 10:04:58,538 - root - INFO - step: 93110 loss: 16.2833 memory: 44.58GiB(31.99%) tps: 82,723 tflops: 285.49 mfu: 28.87% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 10:04:58,538 - root - INFO - lr: 3.3214e-05 gnorm: 1.06 [20:29:54< 1:31:00] +[titan] 2025-07-10 10:05:02,443 - root - INFO - step: 93115 loss: 16.4115 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.5088 global_avg_mtp_loss: 13.9027 +[titan] 2025-07-10 10:05:02,443 - root - INFO - lr: 3.3209e-05 gnorm: 1.10 [20:29:58< 1:30:56] +[titan] 2025-07-10 10:05:06,360 - root - INFO - step: 93120 loss: 16.2018 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.7300 +[titan] 2025-07-10 10:05:06,360 - root - INFO - lr: 3.3205e-05 gnorm: 1.03 [20:30:02< 1:30:52] +[titan] 2025-07-10 10:05:10,277 - root - INFO - step: 93125 loss: 16.1639 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.6723 +[titan] 2025-07-10 10:05:10,278 - root - INFO - lr: 3.3200e-05 gnorm: 1.16 [20:30:06< 1:30:48] +[titan] 2025-07-10 10:05:14,184 - root - INFO - step: 93130 loss: 16.1133 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4559 global_avg_mtp_loss: 13.6574 +[titan] 2025-07-10 10:05:14,184 - root - INFO - lr: 3.3195e-05 gnorm: 1.04 [20:30:10< 1:30:44] +[titan] 2025-07-10 10:05:18,099 - root - INFO - step: 93135 loss: 16.2209 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.7406 +[titan] 2025-07-10 10:05:18,100 - root - INFO - lr: 3.3191e-05 gnorm: 1.10 [20:30:14< 1:30:40] +[titan] 2025-07-10 10:05:22,053 - root - INFO - step: 93140 loss: 16.2007 memory: 44.58GiB(31.99%) tps: 82,895 tflops: 286.08 mfu: 28.93% global_avg_ntp_loss: 2.4700 global_avg_mtp_loss: 13.7307 +[titan] 2025-07-10 10:05:22,053 - root - INFO - lr: 3.3186e-05 gnorm: 1.05 [20:30:18< 1:30:36] +[titan] 2025-07-10 10:05:25,969 - root - INFO - step: 93145 loss: 16.2297 memory: 44.58GiB(31.99%) tps: 83,674 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7517 +[titan] 2025-07-10 10:05:25,970 - root - INFO - lr: 3.3181e-05 gnorm: 1.08 [20:30:21< 1:30:32] +[titan] 2025-07-10 10:05:29,106 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:05:29,899 - root - INFO - step: 93150 loss: 15.9493 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4287 global_avg_mtp_loss: 13.5206 +[titan] 2025-07-10 10:05:29,900 - root - INFO - lr: 3.3177e-05 gnorm: 1.05 [20:30:25< 1:30:28] +[titan] 2025-07-10 10:05:33,814 - root - INFO - step: 93155 loss: 16.3800 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5023 global_avg_mtp_loss: 13.8777 +[titan] 2025-07-10 10:05:33,815 - root - INFO - lr: 3.3172e-05 gnorm: 1.04 [20:30:29< 1:30:24] +[titan] 2025-07-10 10:05:37,732 - root - INFO - step: 93160 loss: 15.9125 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4278 global_avg_mtp_loss: 13.4847 +[titan] 2025-07-10 10:05:37,733 - root - INFO - lr: 3.3168e-05 gnorm: 1.06 [20:30:33< 1:30:21] +[titan] 2025-07-10 10:05:41,650 - root - INFO - step: 93165 loss: 16.0037 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4354 global_avg_mtp_loss: 13.5682 +[titan] 2025-07-10 10:05:41,651 - root - INFO - lr: 3.3163e-05 gnorm: 1.05 [20:30:37< 1:30:17] +[titan] 2025-07-10 10:05:45,580 - root - INFO - step: 93170 loss: 16.1085 memory: 44.58GiB(31.99%) tps: 83,395 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4592 global_avg_mtp_loss: 13.6493 +[titan] 2025-07-10 10:05:45,580 - root - INFO - lr: 3.3158e-05 gnorm: 1.02 [20:30:41< 1:30:13] +[titan] 2025-07-10 10:05:49,488 - root - INFO - step: 93175 loss: 16.1110 memory: 44.58GiB(31.99%) tps: 83,861 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4559 global_avg_mtp_loss: 13.6550 +[titan] 2025-07-10 10:05:49,488 - root - INFO - lr: 3.3154e-05 gnorm: 1.07 [20:30:45< 1:30:09] +[titan] 2025-07-10 10:05:53,412 - root - INFO - step: 93180 loss: 15.8812 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4295 global_avg_mtp_loss: 13.4517 +[titan] 2025-07-10 10:05:53,413 - root - INFO - lr: 3.3149e-05 gnorm: 1.14 [20:30:49< 1:30:05] +[titan] 2025-07-10 10:05:56,772 - root - INFO - Dumping profiler traces at step 93184 +[titan] 2025-07-10 10:05:56,805 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:05:57,603 - root - INFO - step: 93185 loss: 15.8610 memory: 44.58GiB(31.99%) tps: 78,208 tflops: 269.91 mfu: 27.29% global_avg_ntp_loss: 2.4257 global_avg_mtp_loss: 13.4352 +[titan] 2025-07-10 10:05:57,603 - root - INFO - lr: 3.3145e-05 gnorm: 1.09 [20:30:53< 1:30:01] +[titan] 2025-07-10 10:06:01,509 - root - INFO - step: 93190 loss: 16.3015 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4904 global_avg_mtp_loss: 13.8111 +[titan] 2025-07-10 10:06:01,509 - root - INFO - lr: 3.3140e-05 gnorm: 1.05 [20:30:57< 1:29:57] +[titan] 2025-07-10 10:06:05,426 - root - INFO - step: 93195 loss: 16.3641 memory: 44.58GiB(31.99%) tps: 83,658 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8642 +[titan] 2025-07-10 10:06:05,427 - root - INFO - lr: 3.3135e-05 gnorm: 1.10 [20:31:01< 1:29:53] +[titan] 2025-07-10 10:06:08,570 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:06:09,360 - root - INFO - step: 93200 loss: 15.9506 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.4280 global_avg_mtp_loss: 13.5227 +[titan] 2025-07-10 10:06:09,360 - root - INFO - lr: 3.3131e-05 gnorm: 1.07 [20:31:05< 1:29:49] +[titan] 2025-07-10 10:06:13,288 - root - INFO - step: 93205 loss: 16.1270 memory: 44.58GiB(31.99%) tps: 83,439 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.6601 +[titan] 2025-07-10 10:06:13,288 - root - INFO - lr: 3.3126e-05 gnorm: 1.05 [20:31:09< 1:29:45] +[titan] 2025-07-10 10:06:17,200 - root - INFO - step: 93210 loss: 16.4613 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5148 global_avg_mtp_loss: 13.9466 +[titan] 2025-07-10 10:06:17,201 - root - INFO - lr: 3.3122e-05 gnorm: 1.03 [20:31:13< 1:29:41] +[titan] 2025-07-10 10:06:21,097 - root - INFO - step: 93215 loss: 16.0921 memory: 44.58GiB(31.99%) tps: 84,103 tflops: 290.25 mfu: 29.35% global_avg_ntp_loss: 2.4496 global_avg_mtp_loss: 13.6425 +[titan] 2025-07-10 10:06:21,097 - root - INFO - lr: 3.3117e-05 gnorm: 1.04 [20:31:17< 1:29:37] +[titan] 2025-07-10 10:06:25,002 - root - INFO - step: 93220 loss: 16.0520 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4342 global_avg_mtp_loss: 13.6178 +[titan] 2025-07-10 10:06:25,002 - root - INFO - lr: 3.3112e-05 gnorm: 1.07 [20:31:20< 1:29:33] +[titan] 2025-07-10 10:06:28,966 - root - INFO - step: 93225 loss: 16.1582 memory: 44.58GiB(31.99%) tps: 82,662 tflops: 285.28 mfu: 28.85% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.6887 +[titan] 2025-07-10 10:06:28,967 - root - INFO - lr: 3.3108e-05 gnorm: 1.08 [20:31:24< 1:29:29] +[titan] 2025-07-10 10:06:32,896 - root - INFO - step: 93230 loss: 15.9974 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4307 global_avg_mtp_loss: 13.5667 +[titan] 2025-07-10 10:06:32,896 - root - INFO - lr: 3.3103e-05 gnorm: 1.06 [20:31:28< 1:29:25] +[titan] 2025-07-10 10:06:36,823 - root - INFO - step: 93235 loss: 16.4132 memory: 44.58GiB(31.99%) tps: 83,441 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.9013 +[titan] 2025-07-10 10:06:36,824 - root - INFO - lr: 3.3099e-05 gnorm: 1.04 [20:31:32< 1:29:21] +[titan] 2025-07-10 10:06:40,747 - root - INFO - step: 93240 loss: 16.1421 memory: 44.58GiB(31.99%) tps: 83,526 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6731 +[titan] 2025-07-10 10:06:40,747 - root - INFO - lr: 3.3094e-05 gnorm: 1.06 [20:31:36< 1:29:17] +[titan] 2025-07-10 10:06:44,663 - root - INFO - step: 93245 loss: 16.3020 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.8025 +[titan] 2025-07-10 10:06:44,663 - root - INFO - lr: 3.3090e-05 gnorm: 1.02 [20:31:40< 1:29:13] +[titan] 2025-07-10 10:06:47,802 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:06:48,592 - root - INFO - step: 93250 loss: 16.0455 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.5994 +[titan] 2025-07-10 10:06:48,592 - root - INFO - lr: 3.3085e-05 gnorm: 1.04 [20:31:44< 1:29:09] +[titan] 2025-07-10 10:06:52,518 - root - INFO - step: 93255 loss: 16.4375 memory: 44.58GiB(31.99%) tps: 83,469 tflops: 288.07 mfu: 29.13% global_avg_ntp_loss: 2.5154 global_avg_mtp_loss: 13.9220 +[titan] 2025-07-10 10:06:52,518 - root - INFO - lr: 3.3081e-05 gnorm: 1.06 [20:31:48< 1:29:05] +[titan] 2025-07-10 10:06:56,451 - root - INFO - step: 93260 loss: 16.3923 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.5111 global_avg_mtp_loss: 13.8812 +[titan] 2025-07-10 10:06:56,452 - root - INFO - lr: 3.3076e-05 gnorm: 1.03 [20:31:52< 1:29:01] +[titan] 2025-07-10 10:07:00,395 - root - INFO - step: 93265 loss: 16.1337 memory: 44.58GiB(31.99%) tps: 83,095 tflops: 286.77 mfu: 29.00% global_avg_ntp_loss: 2.4640 global_avg_mtp_loss: 13.6697 +[titan] 2025-07-10 10:07:00,396 - root - INFO - lr: 3.3071e-05 gnorm: 1.04 [20:31:56< 1:28:57] +[titan] 2025-07-10 10:07:04,305 - root - INFO - step: 93270 loss: 16.2756 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4943 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 10:07:04,306 - root - INFO - lr: 3.3067e-05 gnorm: 1.04 [20:32:00< 1:28:53] +[titan] 2025-07-10 10:07:08,218 - root - INFO - step: 93275 loss: 16.1985 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7204 +[titan] 2025-07-10 10:07:08,219 - root - INFO - lr: 3.3062e-05 gnorm: 1.03 [20:32:04< 1:28:49] +[titan] 2025-07-10 10:07:12,128 - root - INFO - step: 93280 loss: 16.1764 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.7116 +[titan] 2025-07-10 10:07:12,128 - root - INFO - lr: 3.3058e-05 gnorm: 1.05 [20:32:08< 1:28:45] +[titan] 2025-07-10 10:07:16,028 - root - INFO - step: 93285 loss: 16.0397 memory: 44.58GiB(31.99%) tps: 84,023 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.5996 +[titan] 2025-07-10 10:07:16,028 - root - INFO - lr: 3.3053e-05 gnorm: 1.02 [20:32:11< 1:28:41] +[titan] 2025-07-10 10:07:19,974 - root - INFO - step: 93290 loss: 16.0842 memory: 44.58GiB(31.99%) tps: 83,061 tflops: 286.66 mfu: 28.98% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.6308 +[titan] 2025-07-10 10:07:19,974 - root - INFO - lr: 3.3049e-05 gnorm: 1.12 [20:32:15< 1:28:37] +[titan] 2025-07-10 10:07:23,885 - root - INFO - step: 93295 loss: 16.0856 memory: 44.58GiB(31.99%) tps: 83,783 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4428 global_avg_mtp_loss: 13.6428 +[titan] 2025-07-10 10:07:23,885 - root - INFO - lr: 3.3044e-05 gnorm: 1.00 [20:32:19< 1:28:33] +[titan] 2025-07-10 10:07:27,041 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:07:27,841 - root - INFO - step: 93300 loss: 16.1370 memory: 44.58GiB(31.99%) tps: 82,837 tflops: 285.88 mfu: 28.91% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6785 +[titan] 2025-07-10 10:07:27,842 - root - INFO - lr: 3.3040e-05 gnorm: 1.05 [20:32:23< 1:28:30] +[titan] 2025-07-10 10:07:31,763 - root - INFO - step: 93305 loss: 16.3352 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8391 +[titan] 2025-07-10 10:07:31,763 - root - INFO - lr: 3.3035e-05 gnorm: 0.99 [20:32:27< 1:28:26] +[titan] 2025-07-10 10:07:35,664 - root - INFO - step: 93310 loss: 16.1434 memory: 44.58GiB(31.99%) tps: 84,006 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6857 +[titan] 2025-07-10 10:07:35,664 - root - INFO - lr: 3.3031e-05 gnorm: 1.04 [20:32:31< 1:28:22] +[titan] 2025-07-10 10:07:39,579 - root - INFO - step: 93315 loss: 16.0414 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4403 global_avg_mtp_loss: 13.6011 +[titan] 2025-07-10 10:07:39,579 - root - INFO - lr: 3.3026e-05 gnorm: 1.06 [20:32:35< 1:28:18] +[titan] 2025-07-10 10:07:43,526 - root - INFO - step: 93320 loss: 16.1179 memory: 44.58GiB(31.99%) tps: 83,022 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.6687 +[titan] 2025-07-10 10:07:43,526 - root - INFO - lr: 3.3022e-05 gnorm: 1.08 [20:32:39< 1:28:14] +[titan] 2025-07-10 10:07:47,437 - root - INFO - step: 93325 loss: 16.3059 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8081 +[titan] 2025-07-10 10:07:47,437 - root - INFO - lr: 3.3017e-05 gnorm: 0.99 [20:32:43< 1:28:10] +[titan] 2025-07-10 10:07:51,334 - root - INFO - step: 93330 loss: 16.0366 memory: 44.58GiB(31.99%) tps: 84,095 tflops: 290.23 mfu: 29.35% global_avg_ntp_loss: 2.4431 global_avg_mtp_loss: 13.5935 +[titan] 2025-07-10 10:07:51,334 - root - INFO - lr: 3.3013e-05 gnorm: 1.05 [20:32:47< 1:28:06] +[titan] 2025-07-10 10:07:55,228 - root - INFO - step: 93335 loss: 16.2169 memory: 44.58GiB(31.99%) tps: 84,165 tflops: 290.47 mfu: 29.37% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7382 +[titan] 2025-07-10 10:07:55,228 - root - INFO - lr: 3.3008e-05 gnorm: 1.05 [20:32:51< 1:28:02] +[titan] 2025-07-10 10:07:59,174 - root - INFO - step: 93340 loss: 16.3219 memory: 44.58GiB(31.99%) tps: 83,045 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.8399 +[titan] 2025-07-10 10:07:59,174 - root - INFO - lr: 3.3004e-05 gnorm: 1.02 [20:32:55< 1:27:58] +[titan] 2025-07-10 10:08:03,098 - root - INFO - step: 93345 loss: 16.2748 memory: 44.58GiB(31.99%) tps: 83,512 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 10:08:03,099 - root - INFO - lr: 3.2999e-05 gnorm: 1.07 [20:32:59< 1:27:54] +[titan] 2025-07-10 10:08:06,236 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:08:07,026 - root - INFO - step: 93350 loss: 16.1721 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.7044 +[titan] 2025-07-10 10:08:07,026 - root - INFO - lr: 3.2995e-05 gnorm: 1.04 [20:33:02< 1:27:50] +[titan] 2025-07-10 10:08:10,940 - root - INFO - step: 93355 loss: 16.4650 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5085 global_avg_mtp_loss: 13.9565 +[titan] 2025-07-10 10:08:10,940 - root - INFO - lr: 3.2990e-05 gnorm: 1.11 [20:33:06< 1:27:46] +[titan] 2025-07-10 10:08:14,862 - root - INFO - step: 93360 loss: 16.0293 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.5912 +[titan] 2025-07-10 10:08:14,862 - root - INFO - lr: 3.2986e-05 gnorm: 1.11 [20:33:10< 1:27:42] +[titan] 2025-07-10 10:08:18,798 - root - INFO - step: 93365 loss: 15.8447 memory: 44.58GiB(31.99%) tps: 83,258 tflops: 287.34 mfu: 29.05% global_avg_ntp_loss: 2.4103 global_avg_mtp_loss: 13.4344 +[titan] 2025-07-10 10:08:18,798 - root - INFO - lr: 3.2981e-05 gnorm: 0.98 [20:33:14< 1:27:38] +[titan] 2025-07-10 10:08:22,714 - root - INFO - step: 93370 loss: 15.9578 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4411 global_avg_mtp_loss: 13.5167 +[titan] 2025-07-10 10:08:22,714 - root - INFO - lr: 3.2977e-05 gnorm: 1.04 [20:33:18< 1:27:34] +[titan] 2025-07-10 10:08:26,663 - root - INFO - step: 93375 loss: 16.0134 memory: 44.58GiB(31.99%) tps: 82,984 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4361 global_avg_mtp_loss: 13.5773 +[titan] 2025-07-10 10:08:26,663 - root - INFO - lr: 3.2972e-05 gnorm: 1.00 [20:33:22< 1:27:30] +[titan] 2025-07-10 10:08:30,604 - root - INFO - step: 93380 loss: 15.9136 memory: 44.58GiB(31.99%) tps: 83,145 tflops: 286.95 mfu: 29.01% global_avg_ntp_loss: 2.4232 global_avg_mtp_loss: 13.4905 +[titan] 2025-07-10 10:08:30,605 - root - INFO - lr: 3.2968e-05 gnorm: 1.08 [20:33:26< 1:27:26] +[titan] 2025-07-10 10:08:34,570 - root - INFO - step: 93385 loss: 16.1303 memory: 44.58GiB(31.99%) tps: 82,643 tflops: 285.21 mfu: 28.84% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.6667 +[titan] 2025-07-10 10:08:34,570 - root - INFO - lr: 3.2963e-05 gnorm: 1.02 [20:33:30< 1:27:22] +[titan] 2025-07-10 10:08:38,481 - root - INFO - step: 93390 loss: 16.2446 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7716 +[titan] 2025-07-10 10:08:38,481 - root - INFO - lr: 3.2959e-05 gnorm: 1.00 [20:33:34< 1:27:18] +[titan] 2025-07-10 10:08:42,401 - root - INFO - step: 93395 loss: 15.9434 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4306 global_avg_mtp_loss: 13.5128 +[titan] 2025-07-10 10:08:42,401 - root - INFO - lr: 3.2954e-05 gnorm: 1.05 [20:33:38< 1:27:14] +[titan] 2025-07-10 10:08:45,538 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:08:46,343 - root - INFO - step: 93400 loss: 16.3917 memory: 44.58GiB(31.99%) tps: 83,123 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8861 +[titan] 2025-07-10 10:08:46,344 - root - INFO - lr: 3.2950e-05 gnorm: 1.09 [20:33:42< 1:27:10] +[titan] 2025-07-10 10:08:50,280 - root - INFO - step: 93405 loss: 15.9807 memory: 44.58GiB(31.99%) tps: 83,252 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4428 global_avg_mtp_loss: 13.5380 +[titan] 2025-07-10 10:08:50,280 - root - INFO - lr: 3.2946e-05 gnorm: 1.04 [20:33:46< 1:27:06] +[titan] 2025-07-10 10:08:54,217 - root - INFO - step: 93410 loss: 16.3250 memory: 44.58GiB(31.99%) tps: 83,234 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.8379 +[titan] 2025-07-10 10:08:54,217 - root - INFO - lr: 3.2941e-05 gnorm: 1.06 [20:33:50< 1:27:02] +[titan] 2025-07-10 10:08:58,132 - root - INFO - step: 93415 loss: 16.0230 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5804 +[titan] 2025-07-10 10:08:58,132 - root - INFO - lr: 3.2937e-05 gnorm: 1.07 [20:33:54< 1:26:58] +[titan] 2025-07-10 10:09:02,067 - root - INFO - step: 93420 loss: 16.2551 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7830 +[titan] 2025-07-10 10:09:02,068 - root - INFO - lr: 3.2932e-05 gnorm: 1.02 [20:33:58< 1:26:54] +[titan] 2025-07-10 10:09:05,997 - root - INFO - step: 93425 loss: 16.7048 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.5633 global_avg_mtp_loss: 14.1415 +[titan] 2025-07-10 10:09:05,998 - root - INFO - lr: 3.2928e-05 gnorm: 1.06 [20:34:01< 1:26:50] +[titan] 2025-07-10 10:09:09,931 - root - INFO - step: 93430 loss: 16.2341 memory: 44.58GiB(31.99%) tps: 83,314 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4772 global_avg_mtp_loss: 13.7569 +[titan] 2025-07-10 10:09:09,931 - root - INFO - lr: 3.2923e-05 gnorm: 1.05 [20:34:05< 1:26:46] +[titan] 2025-07-10 10:09:13,871 - root - INFO - step: 93435 loss: 16.1031 memory: 44.58GiB(31.99%) tps: 83,170 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.6420 +[titan] 2025-07-10 10:09:13,871 - root - INFO - lr: 3.2919e-05 gnorm: 1.06 [20:34:09< 1:26:42] +[titan] 2025-07-10 10:09:17,773 - root - INFO - step: 93440 loss: 16.3687 memory: 44.58GiB(31.99%) tps: 83,977 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.8643 +[titan] 2025-07-10 10:09:17,774 - root - INFO - lr: 3.2914e-05 gnorm: 1.04 [20:34:13< 1:26:38] +[titan] 2025-07-10 10:09:21,701 - root - INFO - step: 93445 loss: 16.4467 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.5318 global_avg_mtp_loss: 13.9149 +[titan] 2025-07-10 10:09:21,701 - root - INFO - lr: 3.2910e-05 gnorm: 1.02 [20:34:17< 1:26:35] +[titan] 2025-07-10 10:09:24,824 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:09:25,611 - root - INFO - step: 93450 loss: 16.0813 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.6059 +[titan] 2025-07-10 10:09:25,611 - root - INFO - lr: 3.2906e-05 gnorm: 1.05 [20:34:21< 1:26:31] +[titan] 2025-07-10 10:09:29,529 - root - INFO - step: 93455 loss: 16.2154 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 10:09:29,529 - root - INFO - lr: 3.2901e-05 gnorm: 1.01 [20:34:25< 1:26:27] +[titan] 2025-07-10 10:09:33,448 - root - INFO - step: 93460 loss: 16.2887 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.5024 global_avg_mtp_loss: 13.7862 +[titan] 2025-07-10 10:09:33,448 - root - INFO - lr: 3.2897e-05 gnorm: 1.02 [20:34:29< 1:26:23] +[titan] 2025-07-10 10:09:37,386 - root - INFO - step: 93465 loss: 16.4305 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.20 mfu: 29.04% global_avg_ntp_loss: 2.5299 global_avg_mtp_loss: 13.9006 +[titan] 2025-07-10 10:09:37,386 - root - INFO - lr: 3.2892e-05 gnorm: 1.11 [20:34:33< 1:26:19] +[titan] 2025-07-10 10:09:41,287 - root - INFO - step: 93470 loss: 16.2216 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7529 +[titan] 2025-07-10 10:09:41,287 - root - INFO - lr: 3.2888e-05 gnorm: 1.02 [20:34:37< 1:26:15] +[titan] 2025-07-10 10:09:45,196 - root - INFO - step: 93475 loss: 16.2590 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 10:09:45,196 - root - INFO - lr: 3.2884e-05 gnorm: 1.07 [20:34:41< 1:26:11] +[titan] 2025-07-10 10:09:49,123 - root - INFO - step: 93480 loss: 16.1053 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4437 global_avg_mtp_loss: 13.6616 +[titan] 2025-07-10 10:09:49,123 - root - INFO - lr: 3.2879e-05 gnorm: 1.03 [20:34:45< 1:26:07] +[titan] 2025-07-10 10:09:53,033 - root - INFO - step: 93485 loss: 15.9969 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4371 global_avg_mtp_loss: 13.5598 +[titan] 2025-07-10 10:09:53,034 - root - INFO - lr: 3.2875e-05 gnorm: 1.11 [20:34:48< 1:26:03] +[titan] 2025-07-10 10:09:56,951 - root - INFO - step: 93490 loss: 16.1717 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.7067 +[titan] 2025-07-10 10:09:56,951 - root - INFO - lr: 3.2870e-05 gnorm: 1.04 [20:34:52< 1:25:59] +[titan] 2025-07-10 10:10:00,862 - root - INFO - step: 93495 loss: 16.0569 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.5937 +[titan] 2025-07-10 10:10:00,862 - root - INFO - lr: 3.2866e-05 gnorm: 1.07 [20:34:56< 1:25:55] +[titan] 2025-07-10 10:10:03,974 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:10:04,768 - root - INFO - step: 93500 loss: 16.5008 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.5213 global_avg_mtp_loss: 13.9795 +[titan] 2025-07-10 10:10:04,768 - root - INFO - lr: 3.2862e-05 gnorm: 1.03 [20:35:00< 1:25:51] +[titan] 2025-07-10 10:10:08,682 - root - INFO - step: 93505 loss: 16.0936 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6286 +[titan] 2025-07-10 10:10:08,682 - root - INFO - lr: 3.2857e-05 gnorm: 1.11 [20:35:04< 1:25:47] +[titan] 2025-07-10 10:10:12,598 - root - INFO - step: 93510 loss: 16.1168 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6612 +[titan] 2025-07-10 10:10:12,598 - root - INFO - lr: 3.2853e-05 gnorm: 1.06 [20:35:08< 1:25:43] +[titan] 2025-07-10 10:10:16,524 - root - INFO - step: 93515 loss: 16.2493 memory: 44.58GiB(31.99%) tps: 83,472 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4851 global_avg_mtp_loss: 13.7643 +[titan] 2025-07-10 10:10:16,525 - root - INFO - lr: 3.2848e-05 gnorm: 1.04 [20:35:12< 1:25:39] +[titan] 2025-07-10 10:10:20,466 - root - INFO - step: 93520 loss: 16.1769 memory: 44.58GiB(31.99%) tps: 83,136 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7076 +[titan] 2025-07-10 10:10:20,466 - root - INFO - lr: 3.2844e-05 gnorm: 1.13 [20:35:16< 1:25:35] +[titan] 2025-07-10 10:10:24,404 - root - INFO - step: 93525 loss: 16.0789 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.6076 +[titan] 2025-07-10 10:10:24,404 - root - INFO - lr: 3.2840e-05 gnorm: 1.03 [20:35:20< 1:25:31] +[titan] 2025-07-10 10:10:28,322 - root - INFO - step: 93530 loss: 16.0762 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6192 +[titan] 2025-07-10 10:10:28,323 - root - INFO - lr: 3.2835e-05 gnorm: 1.13 [20:35:24< 1:25:27] +[titan] 2025-07-10 10:10:32,231 - root - INFO - step: 93535 loss: 15.9619 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4319 global_avg_mtp_loss: 13.5301 +[titan] 2025-07-10 10:10:32,231 - root - INFO - lr: 3.2831e-05 gnorm: 1.03 [20:35:28< 1:25:23] +[titan] 2025-07-10 10:10:36,126 - root - INFO - step: 93540 loss: 15.9873 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4351 global_avg_mtp_loss: 13.5522 +[titan] 2025-07-10 10:10:36,127 - root - INFO - lr: 3.2827e-05 gnorm: 1.02 [20:35:32< 1:25:19] +[titan] 2025-07-10 10:10:40,041 - root - INFO - step: 93545 loss: 16.3572 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.5106 global_avg_mtp_loss: 13.8466 +[titan] 2025-07-10 10:10:40,041 - root - INFO - lr: 3.2822e-05 gnorm: 1.06 [20:35:35< 1:25:15] +[titan] 2025-07-10 10:10:43,164 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:10:43,957 - root - INFO - step: 93550 loss: 16.3145 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.8231 +[titan] 2025-07-10 10:10:43,957 - root - INFO - lr: 3.2818e-05 gnorm: 1.05 [20:35:39< 1:25:11] +[titan] 2025-07-10 10:10:47,872 - root - INFO - step: 93555 loss: 16.4273 memory: 44.58GiB(31.99%) tps: 83,701 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5099 global_avg_mtp_loss: 13.9173 +[titan] 2025-07-10 10:10:47,873 - root - INFO - lr: 3.2814e-05 gnorm: 1.02 [20:35:43< 1:25:07] +[titan] 2025-07-10 10:10:51,775 - root - INFO - step: 93560 loss: 16.1690 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.6897 +[titan] 2025-07-10 10:10:51,775 - root - INFO - lr: 3.2809e-05 gnorm: 1.10 [20:35:47< 1:25:03] +[titan] 2025-07-10 10:10:55,735 - root - INFO - step: 93565 loss: 16.0355 memory: 44.58GiB(31.99%) tps: 82,757 tflops: 285.61 mfu: 28.88% global_avg_ntp_loss: 2.4400 global_avg_mtp_loss: 13.5955 +[titan] 2025-07-10 10:10:55,735 - root - INFO - lr: 3.2805e-05 gnorm: 1.04 [20:35:51< 1:24:59] +[titan] 2025-07-10 10:10:59,640 - root - INFO - step: 93570 loss: 16.3053 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8060 +[titan] 2025-07-10 10:10:59,641 - root - INFO - lr: 3.2801e-05 gnorm: 1.06 [20:35:55< 1:24:55] +[titan] 2025-07-10 10:11:03,594 - root - INFO - step: 93575 loss: 16.1819 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4580 global_avg_mtp_loss: 13.7240 +[titan] 2025-07-10 10:11:03,595 - root - INFO - lr: 3.2796e-05 gnorm: 1.11 [20:35:59< 1:24:51] +[titan] 2025-07-10 10:11:07,491 - root - INFO - step: 93580 loss: 16.1001 memory: 44.58GiB(31.99%) tps: 84,093 tflops: 290.22 mfu: 29.34% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6416 +[titan] 2025-07-10 10:11:07,492 - root - INFO - lr: 3.2792e-05 gnorm: 1.09 [20:36:03< 1:24:47] +[titan] 2025-07-10 10:11:11,406 - root - INFO - step: 93585 loss: 16.2671 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7917 +[titan] 2025-07-10 10:11:11,406 - root - INFO - lr: 3.2788e-05 gnorm: 1.05 [20:36:07< 1:24:43] +[titan] 2025-07-10 10:11:15,325 - root - INFO - step: 93590 loss: 16.1959 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7184 +[titan] 2025-07-10 10:11:15,325 - root - INFO - lr: 3.2783e-05 gnorm: 1.06 [20:36:11< 1:24:40] +[titan] 2025-07-10 10:11:19,230 - root - INFO - step: 93595 loss: 16.3229 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4947 global_avg_mtp_loss: 13.8283 +[titan] 2025-07-10 10:11:19,230 - root - INFO - lr: 3.2779e-05 gnorm: 1.02 [20:36:15< 1:24:36] +[titan] 2025-07-10 10:11:22,352 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:11:23,143 - root - INFO - step: 93600 loss: 16.1753 memory: 44.58GiB(31.99%) tps: 83,753 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.7123 +[titan] 2025-07-10 10:11:23,143 - root - INFO - lr: 3.2775e-05 gnorm: 1.02 [20:36:19< 1:24:32] +[titan] 2025-07-10 10:11:27,105 - root - INFO - step: 93605 loss: 16.2888 memory: 44.58GiB(31.99%) tps: 82,713 tflops: 285.46 mfu: 28.86% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.7979 +[titan] 2025-07-10 10:11:27,106 - root - INFO - lr: 3.2770e-05 gnorm: 1.05 [20:36:23< 1:24:28] +[titan] 2025-07-10 10:11:31,057 - root - INFO - step: 93610 loss: 16.4110 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.5211 global_avg_mtp_loss: 13.8899 +[titan] 2025-07-10 10:11:31,057 - root - INFO - lr: 3.2766e-05 gnorm: 1.05 [20:36:26< 1:24:24] +[titan] 2025-07-10 10:11:34,966 - root - INFO - step: 93615 loss: 16.3944 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4989 global_avg_mtp_loss: 13.8955 +[titan] 2025-07-10 10:11:34,967 - root - INFO - lr: 3.2762e-05 gnorm: 1.08 [20:36:30< 1:24:20] +[titan] 2025-07-10 10:11:38,892 - root - INFO - step: 93620 loss: 16.2703 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7897 +[titan] 2025-07-10 10:11:38,892 - root - INFO - lr: 3.2757e-05 gnorm: 1.02 [20:36:34< 1:24:16] +[titan] 2025-07-10 10:11:42,850 - root - INFO - step: 93625 loss: 16.1741 memory: 44.58GiB(31.99%) tps: 82,794 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7020 +[titan] 2025-07-10 10:11:42,850 - root - INFO - lr: 3.2753e-05 gnorm: 0.99 [20:36:38< 1:24:12] +[titan] 2025-07-10 10:11:46,761 - root - INFO - step: 93630 loss: 15.9887 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4375 global_avg_mtp_loss: 13.5512 +[titan] 2025-07-10 10:11:46,762 - root - INFO - lr: 3.2749e-05 gnorm: 1.05 [20:36:42< 1:24:08] +[titan] 2025-07-10 10:11:50,678 - root - INFO - step: 93635 loss: 16.0451 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.5898 +[titan] 2025-07-10 10:11:50,679 - root - INFO - lr: 3.2744e-05 gnorm: 1.03 [20:36:46< 1:24:04] +[titan] 2025-07-10 10:11:54,589 - root - INFO - step: 93640 loss: 16.1378 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.6865 +[titan] 2025-07-10 10:11:54,589 - root - INFO - lr: 3.2740e-05 gnorm: 1.04 [20:36:50< 1:24:00] +[titan] 2025-07-10 10:11:58,504 - root - INFO - step: 93645 loss: 16.2821 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4892 global_avg_mtp_loss: 13.7929 +[titan] 2025-07-10 10:11:58,505 - root - INFO - lr: 3.2736e-05 gnorm: 1.05 [20:36:54< 1:23:56] +[titan] 2025-07-10 10:12:01,650 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:12:02,438 - root - INFO - step: 93650 loss: 16.4004 memory: 44.58GiB(31.99%) tps: 83,309 tflops: 287.51 mfu: 29.07% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8948 +[titan] 2025-07-10 10:12:02,438 - root - INFO - lr: 3.2732e-05 gnorm: 1.03 [20:36:58< 1:23:52] +[titan] 2025-07-10 10:12:06,369 - root - INFO - step: 93655 loss: 16.1705 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.6943 +[titan] 2025-07-10 10:12:06,369 - root - INFO - lr: 3.2727e-05 gnorm: 1.11 [20:37:02< 1:23:48] +[titan] 2025-07-10 10:12:10,277 - root - INFO - step: 93660 loss: 16.0215 memory: 44.58GiB(31.99%) tps: 83,864 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4491 global_avg_mtp_loss: 13.5724 +[titan] 2025-07-10 10:12:10,277 - root - INFO - lr: 3.2723e-05 gnorm: 1.04 [20:37:06< 1:23:44] +[titan] 2025-07-10 10:12:14,234 - root - INFO - step: 93665 loss: 16.1801 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4802 global_avg_mtp_loss: 13.7000 +[titan] 2025-07-10 10:12:14,234 - root - INFO - lr: 3.2719e-05 gnorm: 1.06 [20:37:10< 1:23:40] +[titan] 2025-07-10 10:12:18,142 - root - INFO - step: 93670 loss: 16.2800 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.7913 +[titan] 2025-07-10 10:12:18,143 - root - INFO - lr: 3.2714e-05 gnorm: 1.00 [20:37:14< 1:23:36] +[titan] 2025-07-10 10:12:22,057 - root - INFO - step: 93675 loss: 16.1836 memory: 44.58GiB(31.99%) tps: 83,722 tflops: 288.94 mfu: 29.22% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7113 +[titan] 2025-07-10 10:12:22,057 - root - INFO - lr: 3.2710e-05 gnorm: 1.07 [20:37:17< 1:23:32] +[titan] 2025-07-10 10:12:25,962 - root - INFO - step: 93680 loss: 15.9485 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4215 global_avg_mtp_loss: 13.5270 +[titan] 2025-07-10 10:12:25,962 - root - INFO - lr: 3.2706e-05 gnorm: 0.99 [20:37:21< 1:23:28] +[titan] 2025-07-10 10:12:29,881 - root - INFO - step: 93685 loss: 15.9821 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.5568 +[titan] 2025-07-10 10:12:29,881 - root - INFO - lr: 3.2702e-05 gnorm: 1.14 [20:37:25< 1:23:24] +[titan] 2025-07-10 10:12:33,784 - root - INFO - step: 93690 loss: 16.4113 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.9052 +[titan] 2025-07-10 10:12:33,784 - root - INFO - lr: 3.2697e-05 gnorm: 1.11 [20:37:29< 1:23:20] +[titan] 2025-07-10 10:12:37,756 - root - INFO - step: 93695 loss: 15.7924 memory: 44.58GiB(31.99%) tps: 82,496 tflops: 284.71 mfu: 28.79% global_avg_ntp_loss: 2.4227 global_avg_mtp_loss: 13.3698 +[titan] 2025-07-10 10:12:37,757 - root - INFO - lr: 3.2693e-05 gnorm: 1.09 [20:37:33< 1:23:16] +[titan] 2025-07-10 10:12:38,696 - root - INFO - Dumping profiler traces at step 93696 +[titan] 2025-07-10 10:12:38,728 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:12:41,114 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:12:41,930 - root - INFO - step: 93700 loss: 16.1499 memory: 44.58GiB(31.99%) tps: 78,526 tflops: 271.01 mfu: 27.40% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.6832 +[titan] 2025-07-10 10:12:41,930 - root - INFO - lr: 3.2689e-05 gnorm: 1.03 [20:37:37< 1:23:12] +[titan] 2025-07-10 10:12:45,859 - root - INFO - step: 93705 loss: 16.2891 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.8016 +[titan] 2025-07-10 10:12:45,859 - root - INFO - lr: 3.2685e-05 gnorm: 1.03 [20:37:41< 1:23:08] +[titan] 2025-07-10 10:12:49,774 - root - INFO - step: 93710 loss: 16.0893 memory: 44.58GiB(31.99%) tps: 83,719 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4458 global_avg_mtp_loss: 13.6435 +[titan] 2025-07-10 10:12:49,774 - root - INFO - lr: 3.2680e-05 gnorm: 1.03 [20:37:45< 1:23:04] +[titan] 2025-07-10 10:12:53,682 - root - INFO - step: 93715 loss: 16.2762 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.7892 +[titan] 2025-07-10 10:12:53,682 - root - INFO - lr: 3.2676e-05 gnorm: 1.06 [20:37:49< 1:23:00] +[titan] 2025-07-10 10:12:57,600 - root - INFO - step: 93720 loss: 16.1163 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.6629 +[titan] 2025-07-10 10:12:57,600 - root - INFO - lr: 3.2672e-05 gnorm: 1.00 [20:37:53< 1:22:56] +[titan] 2025-07-10 10:13:01,556 - root - INFO - step: 93725 loss: 16.0577 memory: 44.58GiB(31.99%) tps: 82,833 tflops: 285.87 mfu: 28.91% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.6002 +[titan] 2025-07-10 10:13:01,557 - root - INFO - lr: 3.2668e-05 gnorm: 1.01 [20:37:57< 1:22:52] +[titan] 2025-07-10 10:13:05,454 - root - INFO - step: 93730 loss: 16.1357 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4560 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 10:13:05,454 - root - INFO - lr: 3.2663e-05 gnorm: 1.04 [20:38:01< 1:22:48] +[titan] 2025-07-10 10:13:09,379 - root - INFO - step: 93735 loss: 16.1785 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4722 global_avg_mtp_loss: 13.7063 +[titan] 2025-07-10 10:13:09,379 - root - INFO - lr: 3.2659e-05 gnorm: 1.04 [20:38:05< 1:22:45] +[titan] 2025-07-10 10:13:13,289 - root - INFO - step: 93740 loss: 16.2421 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7638 +[titan] 2025-07-10 10:13:13,289 - root - INFO - lr: 3.2655e-05 gnorm: 1.08 [20:38:09< 1:22:41] +[titan] 2025-07-10 10:13:17,205 - root - INFO - step: 93745 loss: 15.9707 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4392 global_avg_mtp_loss: 13.5315 +[titan] 2025-07-10 10:13:17,205 - root - INFO - lr: 3.2651e-05 gnorm: 1.06 [20:38:13< 1:22:37] +[titan] 2025-07-10 10:13:20,318 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:13:21,106 - root - INFO - step: 93750 loss: 16.2219 memory: 44.58GiB(31.99%) tps: 83,999 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7471 +[titan] 2025-07-10 10:13:21,107 - root - INFO - lr: 3.2646e-05 gnorm: 1.00 [20:38:17< 1:22:33] +[titan] 2025-07-10 10:13:25,032 - root - INFO - step: 93755 loss: 15.9654 memory: 44.58GiB(31.99%) tps: 83,483 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4244 global_avg_mtp_loss: 13.5410 +[titan] 2025-07-10 10:13:25,032 - root - INFO - lr: 3.2642e-05 gnorm: 1.06 [20:38:20< 1:22:29] +[titan] 2025-07-10 10:13:28,934 - root - INFO - step: 93760 loss: 16.2340 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 10:13:28,935 - root - INFO - lr: 3.2638e-05 gnorm: 1.03 [20:38:24< 1:22:25] +[titan] 2025-07-10 10:13:32,843 - root - INFO - step: 93765 loss: 15.9828 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4245 global_avg_mtp_loss: 13.5583 +[titan] 2025-07-10 10:13:32,843 - root - INFO - lr: 3.2634e-05 gnorm: 1.02 [20:38:28< 1:22:21] +[titan] 2025-07-10 10:13:36,753 - root - INFO - step: 93770 loss: 16.2045 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7316 +[titan] 2025-07-10 10:13:36,753 - root - INFO - lr: 3.2630e-05 gnorm: 1.04 [20:38:32< 1:22:17] +[titan] 2025-07-10 10:13:40,682 - root - INFO - step: 93775 loss: 16.1195 memory: 44.58GiB(31.99%) tps: 83,413 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6677 +[titan] 2025-07-10 10:13:40,682 - root - INFO - lr: 3.2625e-05 gnorm: 1.04 [20:38:36< 1:22:13] +[titan] 2025-07-10 10:13:44,630 - root - INFO - step: 93780 loss: 16.1726 memory: 44.58GiB(31.99%) tps: 82,997 tflops: 286.44 mfu: 28.96% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.6957 +[titan] 2025-07-10 10:13:44,630 - root - INFO - lr: 3.2621e-05 gnorm: 1.01 [20:38:40< 1:22:09] +[titan] 2025-07-10 10:13:48,567 - root - INFO - step: 93785 loss: 16.2565 memory: 44.58GiB(31.99%) tps: 83,249 tflops: 287.30 mfu: 29.05% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7726 +[titan] 2025-07-10 10:13:48,567 - root - INFO - lr: 3.2617e-05 gnorm: 1.07 [20:38:44< 1:22:05] +[titan] 2025-07-10 10:13:52,490 - root - INFO - step: 93790 loss: 16.0809 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6190 +[titan] 2025-07-10 10:13:52,490 - root - INFO - lr: 3.2613e-05 gnorm: 1.07 [20:38:48< 1:22:01] +[titan] 2025-07-10 10:13:56,443 - root - INFO - step: 93795 loss: 15.9340 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.4285 global_avg_mtp_loss: 13.5054 +[titan] 2025-07-10 10:13:56,444 - root - INFO - lr: 3.2609e-05 gnorm: 1.04 [20:38:52< 1:21:57] +[titan] 2025-07-10 10:13:59,620 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:14:00,415 - root - INFO - step: 93800 loss: 16.0985 memory: 44.58GiB(31.99%) tps: 82,507 tflops: 284.75 mfu: 28.79% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6421 +[titan] 2025-07-10 10:14:00,416 - root - INFO - lr: 3.2604e-05 gnorm: 1.05 [20:38:56< 1:21:53] +[titan] 2025-07-10 10:14:04,354 - root - INFO - step: 93805 loss: 16.1826 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7171 +[titan] 2025-07-10 10:14:04,354 - root - INFO - lr: 3.2600e-05 gnorm: 1.04 [20:39:00< 1:21:49] +[titan] 2025-07-10 10:14:08,304 - root - INFO - step: 93810 loss: 16.1890 memory: 44.58GiB(31.99%) tps: 82,970 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.4739 global_avg_mtp_loss: 13.7151 +[titan] 2025-07-10 10:14:08,304 - root - INFO - lr: 3.2596e-05 gnorm: 1.04 [20:39:04< 1:21:45] +[titan] 2025-07-10 10:14:12,221 - root - INFO - step: 93815 loss: 15.7866 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.3930 global_avg_mtp_loss: 13.3936 +[titan] 2025-07-10 10:14:12,222 - root - INFO - lr: 3.2592e-05 gnorm: 1.08 [20:39:08< 1:21:41] +[titan] 2025-07-10 10:14:16,120 - root - INFO - step: 93820 loss: 16.0712 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6207 +[titan] 2025-07-10 10:14:16,121 - root - INFO - lr: 3.2588e-05 gnorm: 1.03 [20:39:12< 1:21:37] +[titan] 2025-07-10 10:14:20,056 - root - INFO - step: 93825 loss: 16.3341 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8381 +[titan] 2025-07-10 10:14:20,056 - root - INFO - lr: 3.2583e-05 gnorm: 1.05 [20:39:15< 1:21:33] +[titan] 2025-07-10 10:14:23,966 - root - INFO - step: 93830 loss: 16.0532 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4448 global_avg_mtp_loss: 13.6085 +[titan] 2025-07-10 10:14:23,966 - root - INFO - lr: 3.2579e-05 gnorm: 1.04 [20:39:19< 1:21:29] +[titan] 2025-07-10 10:14:27,881 - root - INFO - step: 93835 loss: 16.2920 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.8078 +[titan] 2025-07-10 10:14:27,882 - root - INFO - lr: 3.2575e-05 gnorm: 1.07 [20:39:23< 1:21:25] +[titan] 2025-07-10 10:14:31,816 - root - INFO - step: 93840 loss: 16.1899 memory: 44.58GiB(31.99%) tps: 83,287 tflops: 287.44 mfu: 29.06% global_avg_ntp_loss: 2.4857 global_avg_mtp_loss: 13.7041 +[titan] 2025-07-10 10:14:31,816 - root - INFO - lr: 3.2571e-05 gnorm: 1.11 [20:39:27< 1:21:21] +[titan] 2025-07-10 10:14:35,715 - root - INFO - step: 93845 loss: 16.0671 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4482 global_avg_mtp_loss: 13.6189 +[titan] 2025-07-10 10:14:35,715 - root - INFO - lr: 3.2567e-05 gnorm: 1.08 [20:39:31< 1:21:17] +[titan] 2025-07-10 10:14:38,827 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:14:39,627 - root - INFO - step: 93850 loss: 16.0980 memory: 44.58GiB(31.99%) tps: 83,762 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.6479 +[titan] 2025-07-10 10:14:39,627 - root - INFO - lr: 3.2563e-05 gnorm: 1.04 [20:39:35< 1:21:13] +[titan] 2025-07-10 10:14:43,545 - root - INFO - step: 93855 loss: 16.2189 memory: 44.58GiB(31.99%) tps: 83,642 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7422 +[titan] 2025-07-10 10:14:43,546 - root - INFO - lr: 3.2559e-05 gnorm: 1.03 [20:39:39< 1:21:09] +[titan] 2025-07-10 10:14:47,467 - root - INFO - step: 93860 loss: 16.3156 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8211 +[titan] 2025-07-10 10:14:47,468 - root - INFO - lr: 3.2554e-05 gnorm: 1.04 [20:39:43< 1:21:05] +[titan] 2025-07-10 10:14:51,368 - root - INFO - step: 93865 loss: 15.9526 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4337 global_avg_mtp_loss: 13.5189 +[titan] 2025-07-10 10:14:51,368 - root - INFO - lr: 3.2550e-05 gnorm: 1.04 [20:39:47< 1:21:01] +[titan] 2025-07-10 10:14:55,281 - root - INFO - step: 93870 loss: 16.2248 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.7564 +[titan] 2025-07-10 10:14:55,282 - root - INFO - lr: 3.2546e-05 gnorm: 1.02 [20:39:51< 1:20:57] +[titan] 2025-07-10 10:14:59,226 - root - INFO - step: 93875 loss: 16.5485 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 14.0030 +[titan] 2025-07-10 10:14:59,226 - root - INFO - lr: 3.2542e-05 gnorm: 1.02 [20:39:55< 1:20:54] +[titan] 2025-07-10 10:15:03,144 - root - INFO - step: 93880 loss: 16.0534 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4393 global_avg_mtp_loss: 13.6141 +[titan] 2025-07-10 10:15:03,145 - root - INFO - lr: 3.2538e-05 gnorm: 1.08 [20:39:59< 1:20:50] +[titan] 2025-07-10 10:15:07,075 - root - INFO - step: 93885 loss: 16.4037 memory: 44.58GiB(31.99%) tps: 83,373 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.9016 +[titan] 2025-07-10 10:15:07,075 - root - INFO - lr: 3.2534e-05 gnorm: 1.04 [20:40:02< 1:20:46] +[titan] 2025-07-10 10:15:10,983 - root - INFO - step: 93890 loss: 16.3898 memory: 44.58GiB(31.99%) tps: 83,852 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.5238 global_avg_mtp_loss: 13.8660 +[titan] 2025-07-10 10:15:10,984 - root - INFO - lr: 3.2530e-05 gnorm: 1.03 [20:40:06< 1:20:42] +[titan] 2025-07-10 10:15:14,906 - root - INFO - step: 93895 loss: 16.0194 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.5771 +[titan] 2025-07-10 10:15:14,906 - root - INFO - lr: 3.2525e-05 gnorm: 1.03 [20:40:10< 1:20:38] +[titan] 2025-07-10 10:15:18,039 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:15:18,839 - root - INFO - step: 93900 loss: 16.3969 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5205 global_avg_mtp_loss: 13.8764 +[titan] 2025-07-10 10:15:18,839 - root - INFO - lr: 3.2521e-05 gnorm: 1.07 [20:40:14< 1:20:34] +[titan] 2025-07-10 10:15:22,818 - root - INFO - step: 93905 loss: 16.0345 memory: 44.58GiB(31.99%) tps: 82,360 tflops: 284.24 mfu: 28.74% global_avg_ntp_loss: 2.4354 global_avg_mtp_loss: 13.5991 +[titan] 2025-07-10 10:15:22,818 - root - INFO - lr: 3.2517e-05 gnorm: 1.03 [20:40:18< 1:20:30] +[titan] 2025-07-10 10:15:26,748 - root - INFO - step: 93910 loss: 16.3886 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5028 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 10:15:26,748 - root - INFO - lr: 3.2513e-05 gnorm: 1.00 [20:40:22< 1:20:26] +[titan] 2025-07-10 10:15:30,667 - root - INFO - step: 93915 loss: 16.0128 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4325 global_avg_mtp_loss: 13.5803 +[titan] 2025-07-10 10:15:30,667 - root - INFO - lr: 3.2509e-05 gnorm: 1.03 [20:40:26< 1:20:22] +[titan] 2025-07-10 10:15:34,574 - root - INFO - step: 93920 loss: 16.1261 memory: 44.58GiB(31.99%) tps: 83,885 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4671 global_avg_mtp_loss: 13.6590 +[titan] 2025-07-10 10:15:34,574 - root - INFO - lr: 3.2505e-05 gnorm: 1.09 [20:40:30< 1:20:18] +[titan] 2025-07-10 10:15:38,474 - root - INFO - step: 93925 loss: 16.0582 memory: 44.58GiB(31.99%) tps: 84,030 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6104 +[titan] 2025-07-10 10:15:38,474 - root - INFO - lr: 3.2501e-05 gnorm: 1.02 [20:40:34< 1:20:14] +[titan] 2025-07-10 10:15:42,409 - root - INFO - step: 93930 loss: 16.0349 memory: 44.58GiB(31.99%) tps: 83,266 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4464 global_avg_mtp_loss: 13.5885 +[titan] 2025-07-10 10:15:42,410 - root - INFO - lr: 3.2497e-05 gnorm: 1.03 [20:40:38< 1:20:10] +[titan] 2025-07-10 10:15:46,318 - root - INFO - step: 93935 loss: 16.0627 memory: 44.58GiB(31.99%) tps: 83,855 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4502 global_avg_mtp_loss: 13.6125 +[titan] 2025-07-10 10:15:46,318 - root - INFO - lr: 3.2493e-05 gnorm: 1.03 [20:40:42< 1:20:06] +[titan] 2025-07-10 10:15:50,218 - root - INFO - step: 93940 loss: 15.8473 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4105 global_avg_mtp_loss: 13.4368 +[titan] 2025-07-10 10:15:50,218 - root - INFO - lr: 3.2488e-05 gnorm: 1.05 [20:40:46< 1:20:02] +[titan] 2025-07-10 10:15:54,125 - root - INFO - step: 93945 loss: 16.0597 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4502 global_avg_mtp_loss: 13.6095 +[titan] 2025-07-10 10:15:54,125 - root - INFO - lr: 3.2484e-05 gnorm: 1.03 [20:40:50< 1:19:58] +[titan] 2025-07-10 10:15:57,244 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:15:58,029 - root - INFO - step: 93950 loss: 15.9522 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4086 global_avg_mtp_loss: 13.5435 +[titan] 2025-07-10 10:15:58,029 - root - INFO - lr: 3.2480e-05 gnorm: 1.05 [20:40:53< 1:19:54] +[titan] 2025-07-10 10:16:01,928 - root - INFO - step: 93955 loss: 16.2398 memory: 44.58GiB(31.99%) tps: 84,046 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4822 global_avg_mtp_loss: 13.7576 +[titan] 2025-07-10 10:16:01,929 - root - INFO - lr: 3.2476e-05 gnorm: 1.01 [20:40:57< 1:19:50] +[titan] 2025-07-10 10:16:05,827 - root - INFO - step: 93960 loss: 16.1058 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.6443 +[titan] 2025-07-10 10:16:05,828 - root - INFO - lr: 3.2472e-05 gnorm: 1.08 [20:41:01< 1:19:46] +[titan] 2025-07-10 10:16:09,761 - root - INFO - step: 93965 loss: 16.3879 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.5020 global_avg_mtp_loss: 13.8859 +[titan] 2025-07-10 10:16:09,762 - root - INFO - lr: 3.2468e-05 gnorm: 1.02 [20:41:05< 1:19:42] +[titan] 2025-07-10 10:16:13,676 - root - INFO - step: 93970 loss: 16.0811 memory: 44.58GiB(31.99%) tps: 83,713 tflops: 288.91 mfu: 29.21% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.6192 +[titan] 2025-07-10 10:16:13,676 - root - INFO - lr: 3.2464e-05 gnorm: 1.03 [20:41:09< 1:19:38] +[titan] 2025-07-10 10:16:17,583 - root - INFO - step: 93975 loss: 16.2767 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.7883 +[titan] 2025-07-10 10:16:17,583 - root - INFO - lr: 3.2460e-05 gnorm: 1.01 [20:41:13< 1:19:34] +[titan] 2025-07-10 10:16:21,498 - root - INFO - step: 93980 loss: 16.2135 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7328 +[titan] 2025-07-10 10:16:21,498 - root - INFO - lr: 3.2456e-05 gnorm: 1.01 [20:41:17< 1:19:30] +[titan] 2025-07-10 10:16:25,395 - root - INFO - step: 93985 loss: 15.9268 memory: 44.58GiB(31.99%) tps: 84,088 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4363 global_avg_mtp_loss: 13.4906 +[titan] 2025-07-10 10:16:25,395 - root - INFO - lr: 3.2452e-05 gnorm: 1.06 [20:41:21< 1:19:26] +[titan] 2025-07-10 10:16:29,311 - root - INFO - step: 93990 loss: 16.2952 memory: 44.58GiB(31.99%) tps: 83,687 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.8115 +[titan] 2025-07-10 10:16:29,311 - root - INFO - lr: 3.2448e-05 gnorm: 1.03 [20:41:25< 1:19:22] +[titan] 2025-07-10 10:16:33,213 - root - INFO - step: 93995 loss: 15.9183 memory: 44.58GiB(31.99%) tps: 83,971 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4183 global_avg_mtp_loss: 13.5000 +[titan] 2025-07-10 10:16:33,214 - root - INFO - lr: 3.2444e-05 gnorm: 1.02 [20:41:29< 1:19:18] +[titan] 2025-07-10 10:16:36,332 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:16:37,124 - root - INFO - step: 94000 loss: 16.1616 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.6918 +[titan] 2025-07-10 10:16:37,124 - root - INFO - lr: 3.2440e-05 gnorm: 1.05 [20:41:33< 1:19:14] +[titan] 2025-07-10 10:16:41,048 - root - INFO - step: 94005 loss: 16.0788 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6180 +[titan] 2025-07-10 10:16:41,049 - root - INFO - lr: 3.2436e-05 gnorm: 1.01 [20:41:36< 1:19:10] +[titan] 2025-07-10 10:16:44,952 - root - INFO - step: 94010 loss: 16.1595 memory: 44.58GiB(31.99%) tps: 83,949 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.6851 +[titan] 2025-07-10 10:16:44,952 - root - INFO - lr: 3.2431e-05 gnorm: 1.08 [20:41:40< 1:19:06] +[titan] 2025-07-10 10:16:48,850 - root - INFO - step: 94015 loss: 16.0926 memory: 44.58GiB(31.99%) tps: 84,083 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.6410 +[titan] 2025-07-10 10:16:48,850 - root - INFO - lr: 3.2427e-05 gnorm: 1.00 [20:41:44< 1:19:02] +[titan] 2025-07-10 10:16:52,769 - root - INFO - step: 94020 loss: 16.0966 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4491 global_avg_mtp_loss: 13.6475 +[titan] 2025-07-10 10:16:52,770 - root - INFO - lr: 3.2423e-05 gnorm: 1.02 [20:41:48< 1:18:59] +[titan] 2025-07-10 10:16:56,690 - root - INFO - step: 94025 loss: 16.1583 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6893 +[titan] 2025-07-10 10:16:56,691 - root - INFO - lr: 3.2419e-05 gnorm: 1.03 [20:41:52< 1:18:55] +[titan] 2025-07-10 10:17:00,598 - root - INFO - step: 94030 loss: 16.1515 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6834 +[titan] 2025-07-10 10:17:00,598 - root - INFO - lr: 3.2415e-05 gnorm: 1.13 [20:41:56< 1:18:51] +[titan] 2025-07-10 10:17:04,525 - root - INFO - step: 94035 loss: 16.1482 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4534 global_avg_mtp_loss: 13.6948 +[titan] 2025-07-10 10:17:04,525 - root - INFO - lr: 3.2411e-05 gnorm: 1.12 [20:42:00< 1:18:47] +[titan] 2025-07-10 10:17:08,437 - root - INFO - step: 94040 loss: 16.4404 memory: 44.58GiB(31.99%) tps: 83,760 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.5129 global_avg_mtp_loss: 13.9276 +[titan] 2025-07-10 10:17:08,438 - root - INFO - lr: 3.2407e-05 gnorm: 0.99 [20:42:04< 1:18:43] +[titan] 2025-07-10 10:17:12,462 - root - INFO - step: 94045 loss: 16.1912 memory: 44.58GiB(31.99%) tps: 81,419 tflops: 280.99 mfu: 28.41% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.7275 +[titan] 2025-07-10 10:17:12,463 - root - INFO - lr: 3.2403e-05 gnorm: 1.03 [20:42:08< 1:18:39] +[titan] 2025-07-10 10:17:15,583 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:17:16,375 - root - INFO - step: 94050 loss: 16.1039 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.6312 +[titan] 2025-07-10 10:17:16,376 - root - INFO - lr: 3.2399e-05 gnorm: 1.06 [20:42:12< 1:18:35] +[titan] 2025-07-10 10:17:20,278 - root - INFO - step: 94055 loss: 16.1362 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6844 +[titan] 2025-07-10 10:17:20,279 - root - INFO - lr: 3.2395e-05 gnorm: 1.04 [20:42:16< 1:18:31] +[titan] 2025-07-10 10:17:24,205 - root - INFO - step: 94060 loss: 16.3787 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.5083 global_avg_mtp_loss: 13.8704 +[titan] 2025-07-10 10:17:24,205 - root - INFO - lr: 3.2391e-05 gnorm: 1.06 [20:42:20< 1:18:27] +[titan] 2025-07-10 10:17:28,122 - root - INFO - step: 94065 loss: 16.3047 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.5075 global_avg_mtp_loss: 13.7972 +[titan] 2025-07-10 10:17:28,122 - root - INFO - lr: 3.2387e-05 gnorm: 1.14 [20:42:24< 1:18:23] +[titan] 2025-07-10 10:17:32,108 - root - INFO - step: 94070 loss: 16.2308 memory: 44.58GiB(31.99%) tps: 82,216 tflops: 283.74 mfu: 28.69% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7589 +[titan] 2025-07-10 10:17:32,108 - root - INFO - lr: 3.2383e-05 gnorm: 1.07 [20:42:28< 1:18:19] +[titan] 2025-07-10 10:17:36,026 - root - INFO - step: 94075 loss: 16.0909 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4479 global_avg_mtp_loss: 13.6430 +[titan] 2025-07-10 10:17:36,026 - root - INFO - lr: 3.2379e-05 gnorm: 1.02 [20:42:31< 1:18:15] +[titan] 2025-07-10 10:17:39,976 - root - INFO - step: 94080 loss: 16.1255 memory: 44.58GiB(31.99%) tps: 82,962 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6678 +[titan] 2025-07-10 10:17:39,977 - root - INFO - lr: 3.2375e-05 gnorm: 1.04 [20:42:35< 1:18:11] +[titan] 2025-07-10 10:17:43,899 - root - INFO - step: 94085 loss: 16.1581 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6963 +[titan] 2025-07-10 10:17:43,899 - root - INFO - lr: 3.2371e-05 gnorm: 1.02 [20:42:39< 1:18:07] +[titan] 2025-07-10 10:17:47,834 - root - INFO - step: 94090 loss: 16.2061 memory: 44.58GiB(31.99%) tps: 83,285 tflops: 287.43 mfu: 29.06% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7267 +[titan] 2025-07-10 10:17:47,834 - root - INFO - lr: 3.2367e-05 gnorm: 1.05 [20:42:43< 1:18:03] +[titan] 2025-07-10 10:17:51,755 - root - INFO - step: 94095 loss: 16.2648 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7909 +[titan] 2025-07-10 10:17:51,756 - root - INFO - lr: 3.2363e-05 gnorm: 1.05 [20:42:47< 1:17:59] +[titan] 2025-07-10 10:17:54,868 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:17:55,656 - root - INFO - step: 94100 loss: 16.3008 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4963 global_avg_mtp_loss: 13.8045 +[titan] 2025-07-10 10:17:55,657 - root - INFO - lr: 3.2359e-05 gnorm: 1.09 [20:42:51< 1:17:55] +[titan] 2025-07-10 10:17:59,582 - root - INFO - step: 94105 loss: 16.3895 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.8863 +[titan] 2025-07-10 10:17:59,583 - root - INFO - lr: 3.2355e-05 gnorm: 1.04 [20:42:55< 1:17:51] +[titan] 2025-07-10 10:18:03,508 - root - INFO - step: 94110 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 83,476 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 10:18:03,508 - root - INFO - lr: 3.2351e-05 gnorm: 1.08 [20:42:59< 1:17:47] +[titan] 2025-07-10 10:18:07,421 - root - INFO - step: 94115 loss: 16.1783 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.7014 +[titan] 2025-07-10 10:18:07,421 - root - INFO - lr: 3.2347e-05 gnorm: 1.06 [20:43:03< 1:17:43] +[titan] 2025-07-10 10:18:11,319 - root - INFO - step: 94120 loss: 16.1861 memory: 44.58GiB(31.99%) tps: 84,070 tflops: 290.14 mfu: 29.34% global_avg_ntp_loss: 2.4624 global_avg_mtp_loss: 13.7237 +[titan] 2025-07-10 10:18:11,319 - root - INFO - lr: 3.2343e-05 gnorm: 1.03 [20:43:07< 1:17:39] +[titan] 2025-07-10 10:18:15,235 - root - INFO - step: 94125 loss: 16.1248 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.6667 +[titan] 2025-07-10 10:18:15,236 - root - INFO - lr: 3.2339e-05 gnorm: 1.09 [20:43:11< 1:17:35] +[titan] 2025-07-10 10:18:19,165 - root - INFO - step: 94130 loss: 16.1395 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6777 +[titan] 2025-07-10 10:18:19,165 - root - INFO - lr: 3.2335e-05 gnorm: 1.03 [20:43:15< 1:17:31] +[titan] 2025-07-10 10:18:23,085 - root - INFO - step: 94135 loss: 16.2570 memory: 44.58GiB(31.99%) tps: 83,598 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7780 +[titan] 2025-07-10 10:18:23,085 - root - INFO - lr: 3.2331e-05 gnorm: 1.02 [20:43:18< 1:17:27] +[titan] 2025-07-10 10:18:27,009 - root - INFO - step: 94140 loss: 16.0890 memory: 44.58GiB(31.99%) tps: 83,509 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4455 global_avg_mtp_loss: 13.6435 +[titan] 2025-07-10 10:18:27,009 - root - INFO - lr: 3.2327e-05 gnorm: 1.01 [20:43:22< 1:17:23] +[titan] 2025-07-10 10:18:30,926 - root - INFO - step: 94145 loss: 16.2599 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.7757 +[titan] 2025-07-10 10:18:30,926 - root - INFO - lr: 3.2323e-05 gnorm: 1.01 [20:43:26< 1:17:19] +[titan] 2025-07-10 10:18:34,056 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:18:34,849 - root - INFO - step: 94150 loss: 15.9168 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4143 global_avg_mtp_loss: 13.5024 +[titan] 2025-07-10 10:18:34,849 - root - INFO - lr: 3.2319e-05 gnorm: 1.10 [20:43:30< 1:17:15] +[titan] 2025-07-10 10:18:38,794 - root - INFO - step: 94155 loss: 16.2405 memory: 44.58GiB(31.99%) tps: 83,060 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4748 global_avg_mtp_loss: 13.7657 +[titan] 2025-07-10 10:18:38,795 - root - INFO - lr: 3.2316e-05 gnorm: 1.03 [20:43:34< 1:17:11] +[titan] 2025-07-10 10:18:42,696 - root - INFO - step: 94160 loss: 15.7312 memory: 44.58GiB(31.99%) tps: 83,996 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.3891 global_avg_mtp_loss: 13.3421 +[titan] 2025-07-10 10:18:42,696 - root - INFO - lr: 3.2312e-05 gnorm: 1.03 [20:43:38< 1:17:08] +[titan] 2025-07-10 10:18:46,619 - root - INFO - step: 94165 loss: 16.3388 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4900 global_avg_mtp_loss: 13.8488 +[titan] 2025-07-10 10:18:46,619 - root - INFO - lr: 3.2308e-05 gnorm: 1.03 [20:43:42< 1:17:04] +[titan] 2025-07-10 10:18:50,526 - root - INFO - step: 94170 loss: 16.1029 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6351 +[titan] 2025-07-10 10:18:50,527 - root - INFO - lr: 3.2304e-05 gnorm: 1.08 [20:43:46< 1:17:00] +[titan] 2025-07-10 10:18:54,481 - root - INFO - step: 94175 loss: 15.9567 memory: 44.58GiB(31.99%) tps: 82,866 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.4309 global_avg_mtp_loss: 13.5258 +[titan] 2025-07-10 10:18:54,481 - root - INFO - lr: 3.2300e-05 gnorm: 1.01 [20:43:50< 1:16:56] +[titan] 2025-07-10 10:18:58,413 - root - INFO - step: 94180 loss: 16.0315 memory: 44.58GiB(31.99%) tps: 83,347 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.5995 +[titan] 2025-07-10 10:18:58,413 - root - INFO - lr: 3.2296e-05 gnorm: 1.00 [20:43:54< 1:16:52] +[titan] 2025-07-10 10:19:02,337 - root - INFO - step: 94185 loss: 15.9262 memory: 44.58GiB(31.99%) tps: 83,521 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4325 global_avg_mtp_loss: 13.4937 +[titan] 2025-07-10 10:19:02,337 - root - INFO - lr: 3.2292e-05 gnorm: 1.03 [20:43:58< 1:16:48] +[titan] 2025-07-10 10:19:06,237 - root - INFO - step: 94190 loss: 16.0445 memory: 44.58GiB(31.99%) tps: 84,031 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4460 global_avg_mtp_loss: 13.5985 +[titan] 2025-07-10 10:19:06,237 - root - INFO - lr: 3.2288e-05 gnorm: 1.03 [20:44:02< 1:16:44] +[titan] 2025-07-10 10:19:10,139 - root - INFO - step: 94195 loss: 16.2187 memory: 44.58GiB(31.99%) tps: 83,972 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4804 global_avg_mtp_loss: 13.7382 +[titan] 2025-07-10 10:19:10,140 - root - INFO - lr: 3.2284e-05 gnorm: 0.99 [20:44:06< 1:16:40] +[titan] 2025-07-10 10:19:13,277 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:19:14,063 - root - INFO - step: 94200 loss: 16.1698 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.7085 +[titan] 2025-07-10 10:19:14,063 - root - INFO - lr: 3.2280e-05 gnorm: 1.05 [20:44:09< 1:16:36] +[titan] 2025-07-10 10:19:18,043 - root - INFO - step: 94205 loss: 16.0871 memory: 44.58GiB(31.99%) tps: 82,343 tflops: 284.18 mfu: 28.73% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.6391 +[titan] 2025-07-10 10:19:18,043 - root - INFO - lr: 3.2276e-05 gnorm: 1.06 [20:44:13< 1:16:32] +[titan] 2025-07-10 10:19:20,540 - root - INFO - Dumping profiler traces at step 94208 +[titan] 2025-07-10 10:19:20,573 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:19:22,148 - root - INFO - step: 94210 loss: 16.2769 memory: 44.58GiB(31.99%) tps: 79,832 tflops: 275.51 mfu: 27.86% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7900 +[titan] 2025-07-10 10:19:22,148 - root - INFO - lr: 3.2272e-05 gnorm: 1.00 [20:44:18< 1:16:28] +[titan] 2025-07-10 10:19:26,078 - root - INFO - step: 94215 loss: 16.1140 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.6529 +[titan] 2025-07-10 10:19:26,078 - root - INFO - lr: 3.2268e-05 gnorm: 1.04 [20:44:21< 1:16:24] +[titan] 2025-07-10 10:19:30,055 - root - INFO - step: 94220 loss: 16.2085 memory: 44.58GiB(31.99%) tps: 82,413 tflops: 284.42 mfu: 28.76% global_avg_ntp_loss: 2.4691 global_avg_mtp_loss: 13.7394 +[titan] 2025-07-10 10:19:30,055 - root - INFO - lr: 3.2264e-05 gnorm: 1.03 [20:44:25< 1:16:20] +[titan] 2025-07-10 10:19:33,949 - root - INFO - step: 94225 loss: 16.1914 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.7191 +[titan] 2025-07-10 10:19:33,950 - root - INFO - lr: 3.2261e-05 gnorm: 1.09 [20:44:29< 1:16:16] +[titan] 2025-07-10 10:19:37,879 - root - INFO - step: 94230 loss: 16.1220 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.6672 +[titan] 2025-07-10 10:19:37,879 - root - INFO - lr: 3.2257e-05 gnorm: 1.07 [20:44:33< 1:16:12] +[titan] 2025-07-10 10:19:41,790 - root - INFO - step: 94235 loss: 16.0222 memory: 44.58GiB(31.99%) tps: 83,792 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.5533 +[titan] 2025-07-10 10:19:41,790 - root - INFO - lr: 3.2253e-05 gnorm: 1.13 [20:44:37< 1:16:08] +[titan] 2025-07-10 10:19:45,718 - root - INFO - step: 94240 loss: 15.9486 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4231 global_avg_mtp_loss: 13.5255 +[titan] 2025-07-10 10:19:45,718 - root - INFO - lr: 3.2249e-05 gnorm: 1.02 [20:44:41< 1:16:04] +[titan] 2025-07-10 10:19:49,647 - root - INFO - step: 94245 loss: 16.3926 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8950 +[titan] 2025-07-10 10:19:49,648 - root - INFO - lr: 3.2245e-05 gnorm: 1.05 [20:44:45< 1:16:00] +[titan] 2025-07-10 10:19:52,798 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:19:53,590 - root - INFO - step: 94250 loss: 16.3310 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4988 global_avg_mtp_loss: 13.8322 +[titan] 2025-07-10 10:19:53,590 - root - INFO - lr: 3.2241e-05 gnorm: 0.98 [20:44:49< 1:15:56] +[titan] 2025-07-10 10:19:57,503 - root - INFO - step: 94255 loss: 16.0656 memory: 44.58GiB(31.99%) tps: 83,747 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6026 +[titan] 2025-07-10 10:19:57,503 - root - INFO - lr: 3.2237e-05 gnorm: 1.03 [20:44:53< 1:15:52] +[titan] 2025-07-10 10:20:01,429 - root - INFO - step: 94260 loss: 16.3977 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.9048 +[titan] 2025-07-10 10:20:01,429 - root - INFO - lr: 3.2233e-05 gnorm: 1.09 [20:44:57< 1:15:48] +[titan] 2025-07-10 10:20:05,335 - root - INFO - step: 94265 loss: 16.3666 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.5105 global_avg_mtp_loss: 13.8561 +[titan] 2025-07-10 10:20:05,336 - root - INFO - lr: 3.2229e-05 gnorm: 1.11 [20:45:01< 1:15:44] +[titan] 2025-07-10 10:20:09,252 - root - INFO - step: 94270 loss: 16.3900 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.5054 global_avg_mtp_loss: 13.8845 +[titan] 2025-07-10 10:20:09,252 - root - INFO - lr: 3.2226e-05 gnorm: 1.02 [20:45:05< 1:15:40] +[titan] 2025-07-10 10:20:13,168 - root - INFO - step: 94275 loss: 16.0214 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4367 global_avg_mtp_loss: 13.5847 +[titan] 2025-07-10 10:20:13,168 - root - INFO - lr: 3.2222e-05 gnorm: 1.04 [20:45:09< 1:15:36] +[titan] 2025-07-10 10:20:17,122 - root - INFO - step: 94280 loss: 16.1879 memory: 44.58GiB(31.99%) tps: 82,881 tflops: 286.03 mfu: 28.92% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7078 +[titan] 2025-07-10 10:20:17,122 - root - INFO - lr: 3.2218e-05 gnorm: 1.05 [20:45:12< 1:15:32] +[titan] 2025-07-10 10:20:21,050 - root - INFO - step: 94285 loss: 15.9628 memory: 44.58GiB(31.99%) tps: 83,444 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4373 global_avg_mtp_loss: 13.5255 +[titan] 2025-07-10 10:20:21,050 - root - INFO - lr: 3.2214e-05 gnorm: 1.17 [20:45:16< 1:15:28] +[titan] 2025-07-10 10:20:25,005 - root - INFO - step: 94290 loss: 16.0888 memory: 44.58GiB(31.99%) tps: 82,852 tflops: 285.94 mfu: 28.91% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6273 +[titan] 2025-07-10 10:20:25,005 - root - INFO - lr: 3.2210e-05 gnorm: 1.10 [20:45:20< 1:15:24] +[titan] 2025-07-10 10:20:28,911 - root - INFO - step: 94295 loss: 16.1786 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4646 global_avg_mtp_loss: 13.7140 +[titan] 2025-07-10 10:20:28,912 - root - INFO - lr: 3.2206e-05 gnorm: 1.07 [20:45:24< 1:15:20] +[titan] 2025-07-10 10:20:32,013 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:20:32,805 - root - INFO - step: 94300 loss: 16.2085 memory: 44.58GiB(31.99%) tps: 84,159 tflops: 290.45 mfu: 29.37% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 10:20:32,806 - root - INFO - lr: 3.2202e-05 gnorm: 1.07 [20:45:28< 1:15:17] +[titan] 2025-07-10 10:20:36,715 - root - INFO - step: 94305 loss: 16.0602 memory: 44.58GiB(31.99%) tps: 83,820 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4481 global_avg_mtp_loss: 13.6121 +[titan] 2025-07-10 10:20:36,715 - root - INFO - lr: 3.2199e-05 gnorm: 1.10 [20:45:32< 1:15:13] +[titan] 2025-07-10 10:20:40,612 - root - INFO - step: 94310 loss: 16.2851 memory: 44.58GiB(31.99%) tps: 84,098 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 10:20:40,612 - root - INFO - lr: 3.2195e-05 gnorm: 1.11 [20:45:36< 1:15:09] +[titan] 2025-07-10 10:20:44,543 - root - INFO - step: 94315 loss: 16.2501 memory: 44.58GiB(31.99%) tps: 83,356 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.7829 +[titan] 2025-07-10 10:20:44,544 - root - INFO - lr: 3.2191e-05 gnorm: 1.08 [20:45:40< 1:15:05] +[titan] 2025-07-10 10:20:48,462 - root - INFO - step: 94320 loss: 15.9579 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4284 global_avg_mtp_loss: 13.5295 +[titan] 2025-07-10 10:20:48,463 - root - INFO - lr: 3.2187e-05 gnorm: 1.06 [20:45:44< 1:15:01] +[titan] 2025-07-10 10:20:52,364 - root - INFO - step: 94325 loss: 15.7788 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.3843 global_avg_mtp_loss: 13.3944 +[titan] 2025-07-10 10:20:52,364 - root - INFO - lr: 3.2183e-05 gnorm: 1.03 [20:45:48< 1:14:57] +[titan] 2025-07-10 10:20:56,305 - root - INFO - step: 94330 loss: 16.0776 memory: 44.58GiB(31.99%) tps: 83,162 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4342 global_avg_mtp_loss: 13.6434 +[titan] 2025-07-10 10:20:56,305 - root - INFO - lr: 3.2179e-05 gnorm: 1.10 [20:45:52< 1:14:53] +[titan] 2025-07-10 10:21:00,226 - root - INFO - step: 94335 loss: 16.5354 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5248 global_avg_mtp_loss: 14.0106 +[titan] 2025-07-10 10:21:00,226 - root - INFO - lr: 3.2175e-05 gnorm: 1.06 [20:45:56< 1:14:49] +[titan] 2025-07-10 10:21:04,142 - root - INFO - step: 94340 loss: 16.0658 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4540 global_avg_mtp_loss: 13.6118 +[titan] 2025-07-10 10:21:04,143 - root - INFO - lr: 3.2172e-05 gnorm: 1.03 [20:46:00< 1:14:45] +[titan] 2025-07-10 10:21:08,055 - root - INFO - step: 94345 loss: 16.1470 memory: 44.58GiB(31.99%) tps: 83,749 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.6804 +[titan] 2025-07-10 10:21:08,056 - root - INFO - lr: 3.2168e-05 gnorm: 1.03 [20:46:03< 1:14:41] +[titan] 2025-07-10 10:21:11,175 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:21:11,969 - root - INFO - step: 94350 loss: 15.8122 memory: 44.58GiB(31.99%) tps: 83,742 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4148 global_avg_mtp_loss: 13.3974 +[titan] 2025-07-10 10:21:11,969 - root - INFO - lr: 3.2164e-05 gnorm: 1.07 [20:46:07< 1:14:37] +[titan] 2025-07-10 10:21:15,907 - root - INFO - step: 94355 loss: 16.3982 memory: 44.58GiB(31.99%) tps: 83,209 tflops: 287.17 mfu: 29.04% global_avg_ntp_loss: 2.5157 global_avg_mtp_loss: 13.8826 +[titan] 2025-07-10 10:21:15,908 - root - INFO - lr: 3.2160e-05 gnorm: 1.06 [20:46:11< 1:14:33] +[titan] 2025-07-10 10:21:19,819 - root - INFO - step: 94360 loss: 16.2935 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4984 global_avg_mtp_loss: 13.7952 +[titan] 2025-07-10 10:21:19,819 - root - INFO - lr: 3.2156e-05 gnorm: 1.02 [20:46:15< 1:14:29] +[titan] 2025-07-10 10:21:23,734 - root - INFO - step: 94365 loss: 16.1432 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4579 global_avg_mtp_loss: 13.6853 +[titan] 2025-07-10 10:21:23,735 - root - INFO - lr: 3.2153e-05 gnorm: 1.06 [20:46:19< 1:14:25] +[titan] 2025-07-10 10:21:27,651 - root - INFO - step: 94370 loss: 16.2488 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7711 +[titan] 2025-07-10 10:21:27,652 - root - INFO - lr: 3.2149e-05 gnorm: 1.01 [20:46:23< 1:14:21] +[titan] 2025-07-10 10:21:31,575 - root - INFO - step: 94375 loss: 16.1518 memory: 44.58GiB(31.99%) tps: 83,527 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.6800 +[titan] 2025-07-10 10:21:31,575 - root - INFO - lr: 3.2145e-05 gnorm: 1.04 [20:46:27< 1:14:17] +[titan] 2025-07-10 10:21:35,509 - root - INFO - step: 94380 loss: 16.0604 memory: 44.58GiB(31.99%) tps: 83,290 tflops: 287.45 mfu: 29.06% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.6142 +[titan] 2025-07-10 10:21:35,510 - root - INFO - lr: 3.2141e-05 gnorm: 1.04 [20:46:31< 1:14:13] +[titan] 2025-07-10 10:21:39,442 - root - INFO - step: 94385 loss: 16.0550 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.6071 +[titan] 2025-07-10 10:21:39,442 - root - INFO - lr: 3.2137e-05 gnorm: 1.04 [20:46:35< 1:14:09] +[titan] 2025-07-10 10:21:43,371 - root - INFO - step: 94390 loss: 16.1334 memory: 44.58GiB(31.99%) tps: 83,407 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4578 global_avg_mtp_loss: 13.6757 +[titan] 2025-07-10 10:21:43,371 - root - INFO - lr: 3.2134e-05 gnorm: 1.02 [20:46:39< 1:14:05] +[titan] 2025-07-10 10:21:47,329 - root - INFO - step: 94395 loss: 16.3364 memory: 44.58GiB(31.99%) tps: 82,792 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.8495 +[titan] 2025-07-10 10:21:47,330 - root - INFO - lr: 3.2130e-05 gnorm: 1.09 [20:46:43< 1:14:01] +[titan] 2025-07-10 10:21:50,437 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:21:51,225 - root - INFO - step: 94400 loss: 16.0443 memory: 44.58GiB(31.99%) tps: 84,118 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4394 global_avg_mtp_loss: 13.6049 +[titan] 2025-07-10 10:21:51,226 - root - INFO - lr: 3.2126e-05 gnorm: 1.05 [20:46:47< 1:13:57] +[titan] 2025-07-10 10:21:55,131 - root - INFO - step: 94405 loss: 16.3003 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.8209 +[titan] 2025-07-10 10:21:55,131 - root - INFO - lr: 3.2122e-05 gnorm: 1.01 [20:46:50< 1:13:53] +[titan] 2025-07-10 10:21:59,047 - root - INFO - step: 94410 loss: 15.9502 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.5081 +[titan] 2025-07-10 10:21:59,047 - root - INFO - lr: 3.2118e-05 gnorm: 1.10 [20:46:54< 1:13:49] +[titan] 2025-07-10 10:22:02,965 - root - INFO - step: 94415 loss: 16.2365 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.7574 +[titan] 2025-07-10 10:22:02,965 - root - INFO - lr: 3.2115e-05 gnorm: 1.13 [20:46:58< 1:13:45] +[titan] 2025-07-10 10:22:06,871 - root - INFO - step: 94420 loss: 16.1360 memory: 44.58GiB(31.99%) tps: 83,891 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.6606 +[titan] 2025-07-10 10:22:06,871 - root - INFO - lr: 3.2111e-05 gnorm: 1.05 [20:47:02< 1:13:41] +[titan] 2025-07-10 10:22:10,778 - root - INFO - step: 94425 loss: 16.0098 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.5676 +[titan] 2025-07-10 10:22:10,778 - root - INFO - lr: 3.2107e-05 gnorm: 0.98 [20:47:06< 1:13:37] +[titan] 2025-07-10 10:22:14,691 - root - INFO - step: 94430 loss: 16.3520 memory: 44.58GiB(31.99%) tps: 83,740 tflops: 289.00 mfu: 29.22% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8470 +[titan] 2025-07-10 10:22:14,692 - root - INFO - lr: 3.2103e-05 gnorm: 1.08 [20:47:10< 1:13:33] +[titan] 2025-07-10 10:22:18,598 - root - INFO - step: 94435 loss: 16.1491 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.6876 +[titan] 2025-07-10 10:22:18,598 - root - INFO - lr: 3.2100e-05 gnorm: 1.00 [20:47:14< 1:13:29] +[titan] 2025-07-10 10:22:22,503 - root - INFO - step: 94440 loss: 16.1843 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.7227 +[titan] 2025-07-10 10:22:22,503 - root - INFO - lr: 3.2096e-05 gnorm: 1.03 [20:47:18< 1:13:25] +[titan] 2025-07-10 10:22:26,403 - root - INFO - step: 94445 loss: 16.2933 memory: 44.58GiB(31.99%) tps: 84,035 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4973 global_avg_mtp_loss: 13.7960 +[titan] 2025-07-10 10:22:26,403 - root - INFO - lr: 3.2092e-05 gnorm: 1.06 [20:47:22< 1:13:22] +[titan] 2025-07-10 10:22:29,522 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:22:30,307 - root - INFO - step: 94450 loss: 16.3677 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4972 global_avg_mtp_loss: 13.8705 +[titan] 2025-07-10 10:22:30,307 - root - INFO - lr: 3.2088e-05 gnorm: 1.04 [20:47:26< 1:13:18] +[titan] 2025-07-10 10:22:34,227 - root - INFO - step: 94455 loss: 16.0803 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6316 +[titan] 2025-07-10 10:22:34,227 - root - INFO - lr: 3.2085e-05 gnorm: 1.06 [20:47:30< 1:13:14] +[titan] 2025-07-10 10:22:38,155 - root - INFO - step: 94460 loss: 16.2651 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7787 +[titan] 2025-07-10 10:22:38,155 - root - INFO - lr: 3.2081e-05 gnorm: 1.06 [20:47:34< 1:13:10] +[titan] 2025-07-10 10:22:42,064 - root - INFO - step: 94465 loss: 16.3432 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.8556 +[titan] 2025-07-10 10:22:42,065 - root - INFO - lr: 3.2077e-05 gnorm: 1.03 [20:47:37< 1:13:06] +[titan] 2025-07-10 10:22:45,982 - root - INFO - step: 94470 loss: 16.2570 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7910 +[titan] 2025-07-10 10:22:45,983 - root - INFO - lr: 3.2073e-05 gnorm: 1.04 [20:47:41< 1:13:02] +[titan] 2025-07-10 10:22:49,888 - root - INFO - step: 94475 loss: 16.2975 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8048 +[titan] 2025-07-10 10:22:49,889 - root - INFO - lr: 3.2070e-05 gnorm: 1.01 [20:47:45< 1:12:58] +[titan] 2025-07-10 10:22:53,800 - root - INFO - step: 94480 loss: 16.2170 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4819 global_avg_mtp_loss: 13.7351 +[titan] 2025-07-10 10:22:53,800 - root - INFO - lr: 3.2066e-05 gnorm: 1.03 [20:47:49< 1:12:54] +[titan] 2025-07-10 10:22:57,696 - root - INFO - step: 94485 loss: 16.3681 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.8664 +[titan] 2025-07-10 10:22:57,696 - root - INFO - lr: 3.2062e-05 gnorm: 1.02 [20:47:53< 1:12:50] +[titan] 2025-07-10 10:23:01,622 - root - INFO - step: 94490 loss: 15.9897 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4255 global_avg_mtp_loss: 13.5642 +[titan] 2025-07-10 10:23:01,622 - root - INFO - lr: 3.2058e-05 gnorm: 1.04 [20:47:57< 1:12:46] +[titan] 2025-07-10 10:23:05,582 - root - INFO - step: 94495 loss: 16.3137 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8132 +[titan] 2025-07-10 10:23:05,583 - root - INFO - lr: 3.2055e-05 gnorm: 1.03 [20:48:01< 1:12:42] +[titan] 2025-07-10 10:23:08,715 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:23:09,502 - root - INFO - step: 94500 loss: 16.0741 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.6273 +[titan] 2025-07-10 10:23:09,502 - root - INFO - lr: 3.2051e-05 gnorm: 1.07 [20:48:05< 1:12:38] +[titan] 2025-07-10 10:23:13,426 - root - INFO - step: 94505 loss: 16.0449 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.5981 +[titan] 2025-07-10 10:23:13,426 - root - INFO - lr: 3.2047e-05 gnorm: 1.04 [20:48:09< 1:12:34] +[titan] 2025-07-10 10:23:17,381 - root - INFO - step: 94510 loss: 16.0092 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.4386 global_avg_mtp_loss: 13.5705 +[titan] 2025-07-10 10:23:17,381 - root - INFO - lr: 3.2043e-05 gnorm: 1.01 [20:48:13< 1:12:30] +[titan] 2025-07-10 10:23:21,308 - root - INFO - step: 94515 loss: 15.9710 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4241 global_avg_mtp_loss: 13.5469 +[titan] 2025-07-10 10:23:21,308 - root - INFO - lr: 3.2040e-05 gnorm: 1.06 [20:48:17< 1:12:26] +[titan] 2025-07-10 10:23:25,226 - root - INFO - step: 94520 loss: 16.1358 memory: 44.58GiB(31.99%) tps: 83,629 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6744 +[titan] 2025-07-10 10:23:25,227 - root - INFO - lr: 3.2036e-05 gnorm: 1.04 [20:48:21< 1:12:22] +[titan] 2025-07-10 10:23:29,155 - root - INFO - step: 94525 loss: 16.2847 memory: 44.58GiB(31.99%) tps: 83,427 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.8054 +[titan] 2025-07-10 10:23:29,155 - root - INFO - lr: 3.2032e-05 gnorm: 1.08 [20:48:25< 1:12:18] +[titan] 2025-07-10 10:23:33,130 - root - INFO - step: 94530 loss: 16.4013 memory: 44.58GiB(31.99%) tps: 82,424 tflops: 284.46 mfu: 28.76% global_avg_ntp_loss: 2.5060 global_avg_mtp_loss: 13.8953 +[titan] 2025-07-10 10:23:33,131 - root - INFO - lr: 3.2029e-05 gnorm: 1.03 [20:48:28< 1:12:14] +[titan] 2025-07-10 10:23:37,109 - root - INFO - step: 94535 loss: 16.3178 memory: 44.58GiB(31.99%) tps: 82,381 tflops: 284.31 mfu: 28.75% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8207 +[titan] 2025-07-10 10:23:37,109 - root - INFO - lr: 3.2025e-05 gnorm: 1.01 [20:48:32< 1:12:10] +[titan] 2025-07-10 10:23:41,075 - root - INFO - step: 94540 loss: 16.3804 memory: 44.58GiB(31.99%) tps: 82,615 tflops: 285.12 mfu: 28.83% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8847 +[titan] 2025-07-10 10:23:41,076 - root - INFO - lr: 3.2021e-05 gnorm: 1.03 [20:48:36< 1:12:06] +[titan] 2025-07-10 10:23:45,028 - root - INFO - step: 94545 loss: 16.2400 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7547 +[titan] 2025-07-10 10:23:45,029 - root - INFO - lr: 3.2018e-05 gnorm: 1.02 [20:48:40< 1:12:02] +[titan] 2025-07-10 10:23:48,197 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:23:48,996 - root - INFO - step: 94550 loss: 16.3097 memory: 44.58GiB(31.99%) tps: 82,587 tflops: 285.02 mfu: 28.82% global_avg_ntp_loss: 2.5175 global_avg_mtp_loss: 13.7922 +[titan] 2025-07-10 10:23:48,997 - root - INFO - lr: 3.2014e-05 gnorm: 1.04 [20:48:44< 1:11:58] +[titan] 2025-07-10 10:23:52,897 - root - INFO - step: 94555 loss: 16.1450 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6820 +[titan] 2025-07-10 10:23:52,897 - root - INFO - lr: 3.2010e-05 gnorm: 1.03 [20:48:48< 1:11:54] +[titan] 2025-07-10 10:23:56,800 - root - INFO - step: 94560 loss: 15.9757 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.5335 +[titan] 2025-07-10 10:23:56,800 - root - INFO - lr: 3.2007e-05 gnorm: 1.01 [20:48:52< 1:11:50] +[titan] 2025-07-10 10:24:00,721 - root - INFO - step: 94565 loss: 15.7320 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.3916 global_avg_mtp_loss: 13.3403 +[titan] 2025-07-10 10:24:00,721 - root - INFO - lr: 3.2003e-05 gnorm: 1.03 [20:48:56< 1:11:46] +[titan] 2025-07-10 10:24:04,640 - root - INFO - step: 94570 loss: 15.8787 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4157 global_avg_mtp_loss: 13.4630 +[titan] 2025-07-10 10:24:04,641 - root - INFO - lr: 3.1999e-05 gnorm: 1.02 [20:49:00< 1:11:42] +[titan] 2025-07-10 10:24:08,542 - root - INFO - step: 94575 loss: 15.8936 memory: 44.58GiB(31.99%) tps: 83,983 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4204 global_avg_mtp_loss: 13.4731 +[titan] 2025-07-10 10:24:08,543 - root - INFO - lr: 3.1995e-05 gnorm: 1.00 [20:49:04< 1:11:38] +[titan] 2025-07-10 10:24:12,482 - root - INFO - step: 94580 loss: 16.2346 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4766 global_avg_mtp_loss: 13.7580 +[titan] 2025-07-10 10:24:12,482 - root - INFO - lr: 3.1992e-05 gnorm: 1.07 [20:49:08< 1:11:34] +[titan] 2025-07-10 10:24:16,399 - root - INFO - step: 94585 loss: 16.2532 memory: 44.58GiB(31.99%) tps: 83,647 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7647 +[titan] 2025-07-10 10:24:16,400 - root - INFO - lr: 3.1988e-05 gnorm: 1.08 [20:49:12< 1:11:31] +[titan] 2025-07-10 10:24:20,387 - root - INFO - step: 94590 loss: 16.0796 memory: 44.58GiB(31.99%) tps: 82,179 tflops: 283.61 mfu: 28.68% global_avg_ntp_loss: 2.4460 global_avg_mtp_loss: 13.6336 +[titan] 2025-07-10 10:24:20,387 - root - INFO - lr: 3.1985e-05 gnorm: 1.06 [20:49:16< 1:11:27] +[titan] 2025-07-10 10:24:24,359 - root - INFO - step: 94595 loss: 16.2229 memory: 44.58GiB(31.99%) tps: 82,506 tflops: 284.74 mfu: 28.79% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7391 +[titan] 2025-07-10 10:24:24,359 - root - INFO - lr: 3.1981e-05 gnorm: 1.04 [20:49:20< 1:11:23] +[titan] 2025-07-10 10:24:27,497 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:24:28,285 - root - INFO - step: 94600 loss: 16.1846 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.7299 +[titan] 2025-07-10 10:24:28,285 - root - INFO - lr: 3.1977e-05 gnorm: 1.02 [20:49:24< 1:11:19] +[titan] 2025-07-10 10:24:32,192 - root - INFO - step: 94605 loss: 16.2420 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7610 +[titan] 2025-07-10 10:24:32,192 - root - INFO - lr: 3.1974e-05 gnorm: 1.39 [20:49:28< 1:11:15] +[titan] 2025-07-10 10:24:36,086 - root - INFO - step: 94610 loss: 16.3127 memory: 44.58GiB(31.99%) tps: 84,139 tflops: 290.38 mfu: 29.36% global_avg_ntp_loss: 2.4843 global_avg_mtp_loss: 13.8285 +[titan] 2025-07-10 10:24:36,087 - root - INFO - lr: 3.1970e-05 gnorm: 1.02 [20:49:31< 1:11:11] +[titan] 2025-07-10 10:24:40,008 - root - INFO - step: 94615 loss: 16.1135 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4594 global_avg_mtp_loss: 13.6541 +[titan] 2025-07-10 10:24:40,008 - root - INFO - lr: 3.1966e-05 gnorm: 1.13 [20:49:35< 1:11:07] +[titan] 2025-07-10 10:24:43,926 - root - INFO - step: 94620 loss: 15.9146 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.4826 +[titan] 2025-07-10 10:24:43,926 - root - INFO - lr: 3.1963e-05 gnorm: 1.00 [20:49:39< 1:11:03] +[titan] 2025-07-10 10:24:47,819 - root - INFO - step: 94625 loss: 16.0165 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.4407 global_avg_mtp_loss: 13.5758 +[titan] 2025-07-10 10:24:47,820 - root - INFO - lr: 3.1959e-05 gnorm: 1.08 [20:49:43< 1:10:59] +[titan] 2025-07-10 10:24:51,722 - root - INFO - step: 94630 loss: 16.4748 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.5200 global_avg_mtp_loss: 13.9548 +[titan] 2025-07-10 10:24:51,723 - root - INFO - lr: 3.1955e-05 gnorm: 1.01 [20:49:47< 1:10:55] +[titan] 2025-07-10 10:24:55,618 - root - INFO - step: 94635 loss: 16.1547 memory: 44.58GiB(31.99%) tps: 84,123 tflops: 290.32 mfu: 29.36% global_avg_ntp_loss: 2.4580 global_avg_mtp_loss: 13.6967 +[titan] 2025-07-10 10:24:55,618 - root - INFO - lr: 3.1952e-05 gnorm: 1.08 [20:49:51< 1:10:51] +[titan] 2025-07-10 10:24:59,555 - root - INFO - step: 94640 loss: 16.2254 memory: 44.58GiB(31.99%) tps: 83,230 tflops: 287.24 mfu: 29.04% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7425 +[titan] 2025-07-10 10:24:59,556 - root - INFO - lr: 3.1948e-05 gnorm: 1.10 [20:49:55< 1:10:47] +[titan] 2025-07-10 10:25:03,471 - root - INFO - step: 94645 loss: 16.2130 memory: 44.58GiB(31.99%) tps: 83,697 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.7380 +[titan] 2025-07-10 10:25:03,471 - root - INFO - lr: 3.1944e-05 gnorm: 0.99 [20:49:59< 1:10:43] +[titan] 2025-07-10 10:25:06,587 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:25:07,379 - root - INFO - step: 94650 loss: 16.0086 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4451 global_avg_mtp_loss: 13.5635 +[titan] 2025-07-10 10:25:07,380 - root - INFO - lr: 3.1941e-05 gnorm: 1.06 [20:50:03< 1:10:39] +[titan] 2025-07-10 10:25:11,296 - root - INFO - step: 94655 loss: 16.2011 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7254 +[titan] 2025-07-10 10:25:11,296 - root - INFO - lr: 3.1937e-05 gnorm: 1.02 [20:50:07< 1:10:35] +[titan] 2025-07-10 10:25:15,227 - root - INFO - step: 94660 loss: 16.2970 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.8020 +[titan] 2025-07-10 10:25:15,227 - root - INFO - lr: 3.1934e-05 gnorm: 1.01 [20:50:11< 1:10:31] +[titan] 2025-07-10 10:25:19,149 - root - INFO - step: 94665 loss: 16.3100 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.8086 +[titan] 2025-07-10 10:25:19,150 - root - INFO - lr: 3.1930e-05 gnorm: 1.08 [20:50:14< 1:10:27] +[titan] 2025-07-10 10:25:23,075 - root - INFO - step: 94670 loss: 16.2014 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.7354 +[titan] 2025-07-10 10:25:23,076 - root - INFO - lr: 3.1926e-05 gnorm: 1.04 [20:50:18< 1:10:23] +[titan] 2025-07-10 10:25:27,009 - root - INFO - step: 94675 loss: 16.0399 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.5810 +[titan] 2025-07-10 10:25:27,009 - root - INFO - lr: 3.1923e-05 gnorm: 1.06 [20:50:22< 1:10:19] +[titan] 2025-07-10 10:25:30,950 - root - INFO - step: 94680 loss: 16.2287 memory: 44.58GiB(31.99%) tps: 83,149 tflops: 286.96 mfu: 29.02% global_avg_ntp_loss: 2.4794 global_avg_mtp_loss: 13.7493 +[titan] 2025-07-10 10:25:30,950 - root - INFO - lr: 3.1919e-05 gnorm: 1.06 [20:50:26< 1:10:15] +[titan] 2025-07-10 10:25:34,886 - root - INFO - step: 94685 loss: 16.3410 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.8501 +[titan] 2025-07-10 10:25:34,886 - root - INFO - lr: 3.1916e-05 gnorm: 1.03 [20:50:30< 1:10:11] +[titan] 2025-07-10 10:25:38,825 - root - INFO - step: 94690 loss: 16.3546 memory: 44.58GiB(31.99%) tps: 83,181 tflops: 287.07 mfu: 29.03% global_avg_ntp_loss: 2.4880 global_avg_mtp_loss: 13.8666 +[titan] 2025-07-10 10:25:38,826 - root - INFO - lr: 3.1912e-05 gnorm: 1.01 [20:50:34< 1:10:07] +[titan] 2025-07-10 10:25:42,758 - root - INFO - step: 94695 loss: 16.0081 memory: 44.58GiB(31.99%) tps: 83,329 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4294 global_avg_mtp_loss: 13.5787 +[titan] 2025-07-10 10:25:42,758 - root - INFO - lr: 3.1908e-05 gnorm: 1.05 [20:50:38< 1:10:03] +[titan] 2025-07-10 10:25:45,867 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:25:46,657 - root - INFO - step: 94700 loss: 16.3753 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.5048 global_avg_mtp_loss: 13.8705 +[titan] 2025-07-10 10:25:46,657 - root - INFO - lr: 3.1905e-05 gnorm: 1.01 [20:50:42< 1:09:59] +[titan] 2025-07-10 10:25:50,553 - root - INFO - step: 94705 loss: 16.2686 memory: 44.58GiB(31.99%) tps: 84,116 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4871 global_avg_mtp_loss: 13.7816 +[titan] 2025-07-10 10:25:50,553 - root - INFO - lr: 3.1901e-05 gnorm: 1.03 [20:50:46< 1:09:55] +[titan] 2025-07-10 10:25:54,483 - root - INFO - step: 94710 loss: 16.3170 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.4897 global_avg_mtp_loss: 13.8273 +[titan] 2025-07-10 10:25:54,483 - root - INFO - lr: 3.1898e-05 gnorm: 1.05 [20:50:50< 1:09:51] +[titan] 2025-07-10 10:25:58,454 - root - INFO - step: 94715 loss: 16.1444 memory: 44.58GiB(31.99%) tps: 82,534 tflops: 284.84 mfu: 28.80% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6854 +[titan] 2025-07-10 10:25:58,454 - root - INFO - lr: 3.1894e-05 gnorm: 1.03 [20:50:54< 1:09:47] +[titan] 2025-07-10 10:26:02,458 - root - INFO - step: 94720 loss: 16.2971 memory: 44.58GiB(31.99%) tps: 81,849 tflops: 282.47 mfu: 28.56% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.8117 +[titan] 2025-07-10 10:26:02,458 - root - INFO - lr: 3.1890e-05 gnorm: 1.04 [20:50:58< 1:09:43] +[titan] 2025-07-10 10:26:02,615 - root - INFO - Dumping profiler traces at step 94720 +[titan] 2025-07-10 10:26:02,650 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:26:06,555 - root - INFO - step: 94725 loss: 16.2620 memory: 44.58GiB(31.99%) tps: 79,987 tflops: 276.05 mfu: 27.91% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.7754 +[titan] 2025-07-10 10:26:06,555 - root - INFO - lr: 3.1887e-05 gnorm: 1.07 [20:51:02< 1:09:40] +[titan] 2025-07-10 10:26:10,465 - root - INFO - step: 94730 loss: 16.0220 memory: 44.58GiB(31.99%) tps: 83,803 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.5763 +[titan] 2025-07-10 10:26:10,466 - root - INFO - lr: 3.1883e-05 gnorm: 1.08 [20:51:06< 1:09:36] +[titan] 2025-07-10 10:26:14,386 - root - INFO - step: 94735 loss: 16.1117 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4573 global_avg_mtp_loss: 13.6544 +[titan] 2025-07-10 10:26:14,387 - root - INFO - lr: 3.1880e-05 gnorm: 1.00 [20:51:10< 1:09:32] +[titan] 2025-07-10 10:26:18,303 - root - INFO - step: 94740 loss: 16.1450 memory: 44.58GiB(31.99%) tps: 83,679 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4540 global_avg_mtp_loss: 13.6909 +[titan] 2025-07-10 10:26:18,303 - root - INFO - lr: 3.1876e-05 gnorm: 1.03 [20:51:14< 1:09:28] +[titan] 2025-07-10 10:26:22,211 - root - INFO - step: 94745 loss: 15.9584 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4262 global_avg_mtp_loss: 13.5321 +[titan] 2025-07-10 10:26:22,211 - root - INFO - lr: 3.1873e-05 gnorm: 1.06 [20:51:18< 1:09:24] +[titan] 2025-07-10 10:26:25,371 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:26:26,162 - root - INFO - step: 94750 loss: 16.2847 memory: 44.58GiB(31.99%) tps: 82,940 tflops: 286.24 mfu: 28.94% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.7952 +[titan] 2025-07-10 10:26:26,162 - root - INFO - lr: 3.1869e-05 gnorm: 1.01 [20:51:21< 1:09:20] +[titan] 2025-07-10 10:26:30,073 - root - INFO - step: 94755 loss: 16.2403 memory: 44.58GiB(31.99%) tps: 83,802 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4869 global_avg_mtp_loss: 13.7535 +[titan] 2025-07-10 10:26:30,073 - root - INFO - lr: 3.1866e-05 gnorm: 1.10 [20:51:25< 1:09:16] +[titan] 2025-07-10 10:26:33,977 - root - INFO - step: 94760 loss: 15.8209 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.3856 global_avg_mtp_loss: 13.4353 +[titan] 2025-07-10 10:26:33,977 - root - INFO - lr: 3.1862e-05 gnorm: 1.11 [20:51:29< 1:09:12] +[titan] 2025-07-10 10:26:37,900 - root - INFO - step: 94765 loss: 16.0755 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4354 global_avg_mtp_loss: 13.6401 +[titan] 2025-07-10 10:26:37,900 - root - INFO - lr: 3.1858e-05 gnorm: 1.08 [20:51:33< 1:09:08] +[titan] 2025-07-10 10:26:41,878 - root - INFO - step: 94770 loss: 15.9710 memory: 44.58GiB(31.99%) tps: 82,374 tflops: 284.29 mfu: 28.74% global_avg_ntp_loss: 2.4300 global_avg_mtp_loss: 13.5411 +[titan] 2025-07-10 10:26:41,878 - root - INFO - lr: 3.1855e-05 gnorm: 1.00 [20:51:37< 1:09:04] +[titan] 2025-07-10 10:26:45,798 - root - INFO - step: 94775 loss: 16.0979 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.6485 +[titan] 2025-07-10 10:26:45,798 - root - INFO - lr: 3.1851e-05 gnorm: 1.05 [20:51:41< 1:09:00] +[titan] 2025-07-10 10:26:49,717 - root - INFO - step: 94780 loss: 15.9991 memory: 44.58GiB(31.99%) tps: 83,627 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4299 global_avg_mtp_loss: 13.5692 +[titan] 2025-07-10 10:26:49,717 - root - INFO - lr: 3.1848e-05 gnorm: 1.12 [20:51:45< 1:08:56] +[titan] 2025-07-10 10:26:53,642 - root - INFO - step: 94785 loss: 15.7931 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.3972 global_avg_mtp_loss: 13.3959 +[titan] 2025-07-10 10:26:53,642 - root - INFO - lr: 3.1844e-05 gnorm: 1.02 [20:51:49< 1:08:52] +[titan] 2025-07-10 10:26:57,563 - root - INFO - step: 94790 loss: 16.0571 memory: 44.58GiB(31.99%) tps: 83,587 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.4497 global_avg_mtp_loss: 13.6073 +[titan] 2025-07-10 10:26:57,563 - root - INFO - lr: 3.1841e-05 gnorm: 1.09 [20:51:53< 1:08:48] +[titan] 2025-07-10 10:27:01,478 - root - INFO - step: 94795 loss: 16.1191 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4545 global_avg_mtp_loss: 13.6645 +[titan] 2025-07-10 10:27:01,478 - root - INFO - lr: 3.1837e-05 gnorm: 1.01 [20:51:57< 1:08:44] +[titan] 2025-07-10 10:27:04,597 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:27:05,401 - root - INFO - step: 94800 loss: 16.3235 memory: 44.58GiB(31.99%) tps: 83,530 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5065 global_avg_mtp_loss: 13.8171 +[titan] 2025-07-10 10:27:05,401 - root - INFO - lr: 3.1834e-05 gnorm: 1.03 [20:52:01< 1:08:40] +[titan] 2025-07-10 10:27:09,332 - root - INFO - step: 94805 loss: 15.9602 memory: 44.58GiB(31.99%) tps: 83,362 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4292 global_avg_mtp_loss: 13.5310 +[titan] 2025-07-10 10:27:09,333 - root - INFO - lr: 3.1830e-05 gnorm: 1.03 [20:52:05< 1:08:36] +[titan] 2025-07-10 10:27:13,263 - root - INFO - step: 94810 loss: 16.3060 memory: 44.58GiB(31.99%) tps: 83,364 tflops: 287.70 mfu: 29.09% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8110 +[titan] 2025-07-10 10:27:13,264 - root - INFO - lr: 3.1827e-05 gnorm: 1.04 [20:52:09< 1:08:32] +[titan] 2025-07-10 10:27:17,187 - root - INFO - step: 94815 loss: 16.0095 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4395 global_avg_mtp_loss: 13.5700 +[titan] 2025-07-10 10:27:17,187 - root - INFO - lr: 3.1823e-05 gnorm: 1.05 [20:52:13< 1:08:28] +[titan] 2025-07-10 10:27:21,120 - root - INFO - step: 94820 loss: 16.1780 memory: 44.58GiB(31.99%) tps: 83,324 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7115 +[titan] 2025-07-10 10:27:21,120 - root - INFO - lr: 3.1820e-05 gnorm: 1.02 [20:52:16< 1:08:24] +[titan] 2025-07-10 10:27:25,039 - root - INFO - step: 94825 loss: 16.1323 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.6546 +[titan] 2025-07-10 10:27:25,040 - root - INFO - lr: 3.1816e-05 gnorm: 1.05 [20:52:20< 1:08:20] +[titan] 2025-07-10 10:27:28,992 - root - INFO - step: 94830 loss: 15.9204 memory: 44.58GiB(31.99%) tps: 82,910 tflops: 286.14 mfu: 28.93% global_avg_ntp_loss: 2.4247 global_avg_mtp_loss: 13.4957 +[titan] 2025-07-10 10:27:28,992 - root - INFO - lr: 3.1813e-05 gnorm: 1.09 [20:52:24< 1:08:16] +[titan] 2025-07-10 10:27:32,896 - root - INFO - step: 94835 loss: 16.1314 memory: 44.58GiB(31.99%) tps: 83,947 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6794 +[titan] 2025-07-10 10:27:32,896 - root - INFO - lr: 3.1809e-05 gnorm: 1.00 [20:52:28< 1:08:12] +[titan] 2025-07-10 10:27:36,799 - root - INFO - step: 94840 loss: 16.1785 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.7083 +[titan] 2025-07-10 10:27:36,799 - root - INFO - lr: 3.1806e-05 gnorm: 1.02 [20:52:32< 1:08:08] +[titan] 2025-07-10 10:27:40,701 - root - INFO - step: 94845 loss: 16.1283 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4571 global_avg_mtp_loss: 13.6712 +[titan] 2025-07-10 10:27:40,701 - root - INFO - lr: 3.1802e-05 gnorm: 1.06 [20:52:36< 1:08:04] +[titan] 2025-07-10 10:27:43,820 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:27:44,606 - root - INFO - step: 94850 loss: 15.8727 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4149 global_avg_mtp_loss: 13.4578 +[titan] 2025-07-10 10:27:44,606 - root - INFO - lr: 3.1799e-05 gnorm: 1.05 [20:52:40< 1:08:00] +[titan] 2025-07-10 10:27:48,519 - root - INFO - step: 94855 loss: 16.0494 memory: 44.58GiB(31.99%) tps: 83,743 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.6044 +[titan] 2025-07-10 10:27:48,520 - root - INFO - lr: 3.1795e-05 gnorm: 1.08 [20:52:44< 1:07:56] +[titan] 2025-07-10 10:27:52,416 - root - INFO - step: 94860 loss: 16.4127 memory: 44.58GiB(31.99%) tps: 84,099 tflops: 290.24 mfu: 29.35% global_avg_ntp_loss: 2.5096 global_avg_mtp_loss: 13.9030 +[titan] 2025-07-10 10:27:52,416 - root - INFO - lr: 3.1792e-05 gnorm: 1.06 [20:52:48< 1:07:52] +[titan] 2025-07-10 10:27:56,313 - root - INFO - step: 94865 loss: 16.1522 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 10:27:56,313 - root - INFO - lr: 3.1788e-05 gnorm: 1.03 [20:52:52< 1:07:49] +[titan] 2025-07-10 10:28:00,222 - root - INFO - step: 94870 loss: 16.1229 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.6621 +[titan] 2025-07-10 10:28:00,223 - root - INFO - lr: 3.1785e-05 gnorm: 1.05 [20:52:56< 1:07:45] +[titan] 2025-07-10 10:28:04,160 - root - INFO - step: 94875 loss: 16.2479 memory: 44.58GiB(31.99%) tps: 83,220 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4847 global_avg_mtp_loss: 13.7632 +[titan] 2025-07-10 10:28:04,161 - root - INFO - lr: 3.1781e-05 gnorm: 1.06 [20:52:59< 1:07:41] +[titan] 2025-07-10 10:28:08,067 - root - INFO - step: 94880 loss: 15.9739 memory: 44.58GiB(31.99%) tps: 83,876 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4370 global_avg_mtp_loss: 13.5370 +[titan] 2025-07-10 10:28:08,068 - root - INFO - lr: 3.1778e-05 gnorm: 1.06 [20:53:03< 1:07:37] +[titan] 2025-07-10 10:28:11,988 - root - INFO - step: 94885 loss: 16.1622 memory: 44.58GiB(31.99%) tps: 83,591 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6961 +[titan] 2025-07-10 10:28:11,988 - root - INFO - lr: 3.1774e-05 gnorm: 1.01 [20:53:07< 1:07:33] +[titan] 2025-07-10 10:28:15,900 - root - INFO - step: 94890 loss: 16.1510 memory: 44.58GiB(31.99%) tps: 83,771 tflops: 289.11 mfu: 29.23% global_avg_ntp_loss: 2.4712 global_avg_mtp_loss: 13.6798 +[titan] 2025-07-10 10:28:15,900 - root - INFO - lr: 3.1771e-05 gnorm: 1.01 [20:53:11< 1:07:29] +[titan] 2025-07-10 10:28:19,863 - root - INFO - step: 94895 loss: 16.3513 memory: 44.58GiB(31.99%) tps: 82,689 tflops: 285.37 mfu: 28.85% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8534 +[titan] 2025-07-10 10:28:19,863 - root - INFO - lr: 3.1768e-05 gnorm: 1.07 [20:53:15< 1:07:25] +[titan] 2025-07-10 10:28:23,012 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:28:23,805 - root - INFO - step: 94900 loss: 16.1292 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4771 global_avg_mtp_loss: 13.6520 +[titan] 2025-07-10 10:28:23,806 - root - INFO - lr: 3.1764e-05 gnorm: 1.13 [20:53:19< 1:07:21] +[titan] 2025-07-10 10:28:27,723 - root - INFO - step: 94905 loss: 16.1022 memory: 44.58GiB(31.99%) tps: 83,644 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6495 +[titan] 2025-07-10 10:28:27,723 - root - INFO - lr: 3.1761e-05 gnorm: 1.06 [20:53:23< 1:07:17] +[titan] 2025-07-10 10:28:31,646 - root - INFO - step: 94910 loss: 16.1167 memory: 44.58GiB(31.99%) tps: 83,534 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6648 +[titan] 2025-07-10 10:28:31,647 - root - INFO - lr: 3.1757e-05 gnorm: 1.02 [20:53:27< 1:07:13] +[titan] 2025-07-10 10:28:35,559 - root - INFO - step: 94915 loss: 15.9020 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4206 global_avg_mtp_loss: 13.4814 +[titan] 2025-07-10 10:28:35,559 - root - INFO - lr: 3.1754e-05 gnorm: 1.05 [20:53:31< 1:07:09] +[titan] 2025-07-10 10:28:39,506 - root - INFO - step: 94920 loss: 16.2676 memory: 44.58GiB(31.99%) tps: 83,024 tflops: 286.53 mfu: 28.97% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7838 +[titan] 2025-07-10 10:28:39,506 - root - INFO - lr: 3.1750e-05 gnorm: 1.05 [20:53:35< 1:07:05] +[titan] 2025-07-10 10:28:43,448 - root - INFO - step: 94925 loss: 16.2123 memory: 44.58GiB(31.99%) tps: 83,135 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4881 global_avg_mtp_loss: 13.7242 +[titan] 2025-07-10 10:28:43,448 - root - INFO - lr: 3.1747e-05 gnorm: 1.03 [20:53:39< 1:07:01] +[titan] 2025-07-10 10:28:47,392 - root - INFO - step: 94930 loss: 16.4737 memory: 44.58GiB(31.99%) tps: 83,090 tflops: 286.76 mfu: 28.99% global_avg_ntp_loss: 2.5358 global_avg_mtp_loss: 13.9379 +[titan] 2025-07-10 10:28:47,392 - root - INFO - lr: 3.1743e-05 gnorm: 1.06 [20:53:43< 1:06:57] +[titan] 2025-07-10 10:28:51,315 - root - INFO - step: 94935 loss: 16.0657 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.6195 +[titan] 2025-07-10 10:28:51,315 - root - INFO - lr: 3.1740e-05 gnorm: 1.04 [20:53:47< 1:06:53] +[titan] 2025-07-10 10:28:55,212 - root - INFO - step: 94940 loss: 16.4764 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.5486 global_avg_mtp_loss: 13.9278 +[titan] 2025-07-10 10:28:55,213 - root - INFO - lr: 3.1737e-05 gnorm: 1.06 [20:53:51< 1:06:49] +[titan] 2025-07-10 10:28:59,157 - root - INFO - step: 94945 loss: 16.3763 memory: 44.58GiB(31.99%) tps: 83,082 tflops: 286.73 mfu: 28.99% global_avg_ntp_loss: 2.5095 global_avg_mtp_loss: 13.8668 +[titan] 2025-07-10 10:28:59,157 - root - INFO - lr: 3.1733e-05 gnorm: 1.04 [20:53:54< 1:06:45] +[titan] 2025-07-10 10:29:02,277 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:29:03,064 - root - INFO - step: 94950 loss: 16.0860 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4440 global_avg_mtp_loss: 13.6420 +[titan] 2025-07-10 10:29:03,064 - root - INFO - lr: 3.1730e-05 gnorm: 1.03 [20:53:58< 1:06:41] +[titan] 2025-07-10 10:29:06,998 - root - INFO - step: 94955 loss: 16.1329 memory: 44.58GiB(31.99%) tps: 83,298 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6698 +[titan] 2025-07-10 10:29:06,998 - root - INFO - lr: 3.1726e-05 gnorm: 1.03 [20:54:02< 1:06:37] +[titan] 2025-07-10 10:29:10,937 - root - INFO - step: 94960 loss: 16.4506 memory: 44.58GiB(31.99%) tps: 83,190 tflops: 287.10 mfu: 29.03% global_avg_ntp_loss: 2.5304 global_avg_mtp_loss: 13.9202 +[titan] 2025-07-10 10:29:10,938 - root - INFO - lr: 3.1723e-05 gnorm: 1.02 [20:54:06< 1:06:33] +[titan] 2025-07-10 10:29:14,858 - root - INFO - step: 94965 loss: 16.4204 memory: 44.58GiB(31.99%) tps: 83,586 tflops: 288.47 mfu: 29.17% global_avg_ntp_loss: 2.5242 global_avg_mtp_loss: 13.8962 +[titan] 2025-07-10 10:29:14,858 - root - INFO - lr: 3.1719e-05 gnorm: 1.08 [20:54:10< 1:06:29] +[titan] 2025-07-10 10:29:18,769 - root - INFO - step: 94970 loss: 16.0439 memory: 44.58GiB(31.99%) tps: 83,785 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4506 global_avg_mtp_loss: 13.5933 +[titan] 2025-07-10 10:29:18,770 - root - INFO - lr: 3.1716e-05 gnorm: 1.08 [20:54:14< 1:06:25] +[titan] 2025-07-10 10:29:22,671 - root - INFO - step: 94975 loss: 16.1348 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.6615 +[titan] 2025-07-10 10:29:22,672 - root - INFO - lr: 3.1713e-05 gnorm: 1.02 [20:54:18< 1:06:21] +[titan] 2025-07-10 10:29:26,592 - root - INFO - step: 94980 loss: 15.8744 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4139 global_avg_mtp_loss: 13.4606 +[titan] 2025-07-10 10:29:26,592 - root - INFO - lr: 3.1709e-05 gnorm: 1.07 [20:54:22< 1:06:17] +[titan] 2025-07-10 10:29:30,520 - root - INFO - step: 94985 loss: 16.0481 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.4449 global_avg_mtp_loss: 13.6033 +[titan] 2025-07-10 10:29:30,520 - root - INFO - lr: 3.1706e-05 gnorm: 1.11 [20:54:26< 1:06:13] +[titan] 2025-07-10 10:29:34,435 - root - INFO - step: 94990 loss: 16.2555 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7716 +[titan] 2025-07-10 10:29:34,436 - root - INFO - lr: 3.1702e-05 gnorm: 1.05 [20:54:30< 1:06:09] +[titan] 2025-07-10 10:29:38,347 - root - INFO - step: 94995 loss: 16.0620 memory: 44.58GiB(31.99%) tps: 83,786 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.4516 global_avg_mtp_loss: 13.6104 +[titan] 2025-07-10 10:29:38,347 - root - INFO - lr: 3.1699e-05 gnorm: 1.04 [20:54:34< 1:06:05] +[titan] 2025-07-10 10:29:41,470 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:29:42,261 - root - INFO - step: 95000 loss: 16.0238 memory: 44.58GiB(31.99%) tps: 83,725 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4385 global_avg_mtp_loss: 13.5853 +[titan] 2025-07-10 10:29:42,261 - root - INFO - lr: 3.1696e-05 gnorm: 1.08 [20:54:38< 1:06:02] +[titan] 2025-07-10 10:29:46,175 - root - INFO - step: 95005 loss: 16.3454 memory: 44.58GiB(31.99%) tps: 83,724 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.5012 global_avg_mtp_loss: 13.8442 +[titan] 2025-07-10 10:29:46,175 - root - INFO - lr: 3.1692e-05 gnorm: 1.08 [20:54:41< 1:05:58] +[titan] 2025-07-10 10:29:50,067 - root - INFO - step: 95010 loss: 15.8609 memory: 44.58GiB(31.99%) tps: 84,202 tflops: 290.60 mfu: 29.38% global_avg_ntp_loss: 2.4184 global_avg_mtp_loss: 13.4425 +[titan] 2025-07-10 10:29:50,067 - root - INFO - lr: 3.1689e-05 gnorm: 1.08 [20:54:45< 1:05:54] +[titan] 2025-07-10 10:29:53,964 - root - INFO - step: 95015 loss: 16.2896 memory: 44.58GiB(31.99%) tps: 84,090 tflops: 290.21 mfu: 29.34% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.7875 +[titan] 2025-07-10 10:29:53,965 - root - INFO - lr: 3.1686e-05 gnorm: 1.11 [20:54:49< 1:05:50] +[titan] 2025-07-10 10:29:57,891 - root - INFO - step: 95020 loss: 15.9262 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4294 global_avg_mtp_loss: 13.4968 +[titan] 2025-07-10 10:29:57,891 - root - INFO - lr: 3.1682e-05 gnorm: 1.06 [20:54:53< 1:05:46] +[titan] 2025-07-10 10:30:01,792 - root - INFO - step: 95025 loss: 16.1714 memory: 44.58GiB(31.99%) tps: 84,000 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.7108 +[titan] 2025-07-10 10:30:01,792 - root - INFO - lr: 3.1679e-05 gnorm: 1.06 [20:54:57< 1:05:42] +[titan] 2025-07-10 10:30:05,711 - root - INFO - step: 95030 loss: 16.1785 memory: 44.58GiB(31.99%) tps: 83,615 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4644 global_avg_mtp_loss: 13.7141 +[titan] 2025-07-10 10:30:05,712 - root - INFO - lr: 3.1675e-05 gnorm: 1.13 [20:55:01< 1:05:38] +[titan] 2025-07-10 10:30:09,639 - root - INFO - step: 95035 loss: 16.3254 memory: 44.58GiB(31.99%) tps: 83,440 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-10 10:30:09,639 - root - INFO - lr: 3.1672e-05 gnorm: 1.06 [20:55:05< 1:05:34] +[titan] 2025-07-10 10:30:13,538 - root - INFO - step: 95040 loss: 15.7988 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.3919 global_avg_mtp_loss: 13.4069 +[titan] 2025-07-10 10:30:13,538 - root - INFO - lr: 3.1669e-05 gnorm: 1.10 [20:55:09< 1:05:30] +[titan] 2025-07-10 10:30:17,465 - root - INFO - step: 95045 loss: 16.2425 memory: 44.58GiB(31.99%) tps: 83,449 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7634 +[titan] 2025-07-10 10:30:17,465 - root - INFO - lr: 3.1665e-05 gnorm: 1.06 [20:55:13< 1:05:26] +[titan] 2025-07-10 10:30:20,603 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:30:21,434 - root - INFO - step: 95050 loss: 16.1727 memory: 44.58GiB(31.99%) tps: 82,563 tflops: 284.94 mfu: 28.81% global_avg_ntp_loss: 2.4771 global_avg_mtp_loss: 13.6957 +[titan] 2025-07-10 10:30:21,434 - root - INFO - lr: 3.1662e-05 gnorm: 1.07 [20:55:17< 1:05:22] +[titan] 2025-07-10 10:30:25,338 - root - INFO - step: 95055 loss: 16.3201 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8248 +[titan] 2025-07-10 10:30:25,339 - root - INFO - lr: 3.1659e-05 gnorm: 1.06 [20:55:21< 1:05:18] +[titan] 2025-07-10 10:30:29,256 - root - INFO - step: 95060 loss: 16.1167 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4587 global_avg_mtp_loss: 13.6580 +[titan] 2025-07-10 10:30:29,257 - root - INFO - lr: 3.1655e-05 gnorm: 1.01 [20:55:25< 1:05:14] +[titan] 2025-07-10 10:30:33,163 - root - INFO - step: 95065 loss: 15.9588 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.5226 +[titan] 2025-07-10 10:30:33,164 - root - INFO - lr: 3.1652e-05 gnorm: 1.03 [20:55:28< 1:05:10] +[titan] 2025-07-10 10:30:37,093 - root - INFO - step: 95070 loss: 16.1721 memory: 44.58GiB(31.99%) tps: 83,396 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.7144 +[titan] 2025-07-10 10:30:37,093 - root - INFO - lr: 3.1649e-05 gnorm: 1.02 [20:55:32< 1:05:06] +[titan] 2025-07-10 10:30:41,032 - root - INFO - step: 95075 loss: 16.4114 memory: 44.58GiB(31.99%) tps: 83,204 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.9014 +[titan] 2025-07-10 10:30:41,032 - root - INFO - lr: 3.1645e-05 gnorm: 1.12 [20:55:36< 1:05:02] +[titan] 2025-07-10 10:30:44,973 - root - INFO - step: 95080 loss: 16.1389 memory: 44.58GiB(31.99%) tps: 83,152 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.6736 +[titan] 2025-07-10 10:30:44,973 - root - INFO - lr: 3.1642e-05 gnorm: 1.04 [20:55:40< 1:04:58] +[titan] 2025-07-10 10:30:48,916 - root - INFO - step: 95085 loss: 15.8818 memory: 44.58GiB(31.99%) tps: 83,103 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.4149 global_avg_mtp_loss: 13.4670 +[titan] 2025-07-10 10:30:48,916 - root - INFO - lr: 3.1639e-05 gnorm: 1.09 [20:55:44< 1:04:54] +[titan] 2025-07-10 10:30:52,826 - root - INFO - step: 95090 loss: 16.1310 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4626 global_avg_mtp_loss: 13.6684 +[titan] 2025-07-10 10:30:52,827 - root - INFO - lr: 3.1635e-05 gnorm: 1.05 [20:55:48< 1:04:50] +[titan] 2025-07-10 10:30:56,755 - root - INFO - step: 95095 loss: 16.0667 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.6208 +[titan] 2025-07-10 10:30:56,755 - root - INFO - lr: 3.1632e-05 gnorm: 1.04 [20:55:52< 1:04:46] +[titan] 2025-07-10 10:30:59,883 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:31:00,685 - root - INFO - step: 95100 loss: 16.2561 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7667 +[titan] 2025-07-10 10:31:00,685 - root - INFO - lr: 3.1629e-05 gnorm: 1.01 [20:55:56< 1:04:42] +[titan] 2025-07-10 10:31:04,617 - root - INFO - step: 95105 loss: 15.9391 memory: 44.58GiB(31.99%) tps: 83,336 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4312 global_avg_mtp_loss: 13.5080 +[titan] 2025-07-10 10:31:04,617 - root - INFO - lr: 3.1625e-05 gnorm: 1.05 [20:56:00< 1:04:38] +[titan] 2025-07-10 10:31:08,537 - root - INFO - step: 95110 loss: 16.1875 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.7266 +[titan] 2025-07-10 10:31:08,537 - root - INFO - lr: 3.1622e-05 gnorm: 0.98 [20:56:04< 1:04:34] +[titan] 2025-07-10 10:31:12,491 - root - INFO - step: 95115 loss: 16.4947 memory: 44.58GiB(31.99%) tps: 82,865 tflops: 285.98 mfu: 28.92% global_avg_ntp_loss: 2.5330 global_avg_mtp_loss: 13.9616 +[titan] 2025-07-10 10:31:12,492 - root - INFO - lr: 3.1619e-05 gnorm: 1.07 [20:56:08< 1:04:30] +[titan] 2025-07-10 10:31:16,426 - root - INFO - step: 95120 loss: 16.4552 memory: 44.58GiB(31.99%) tps: 83,281 tflops: 287.42 mfu: 29.06% global_avg_ntp_loss: 2.5259 global_avg_mtp_loss: 13.9293 +[titan] 2025-07-10 10:31:16,427 - root - INFO - lr: 3.1615e-05 gnorm: 1.11 [20:56:12< 1:04:26] +[titan] 2025-07-10 10:31:20,368 - root - INFO - step: 95125 loss: 15.8835 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4167 global_avg_mtp_loss: 13.4668 +[titan] 2025-07-10 10:31:20,368 - root - INFO - lr: 3.1612e-05 gnorm: 1.03 [20:56:16< 1:04:22] +[titan] 2025-07-10 10:31:24,324 - root - INFO - step: 95130 loss: 16.2507 memory: 44.58GiB(31.99%) tps: 82,824 tflops: 285.84 mfu: 28.90% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.7832 +[titan] 2025-07-10 10:31:24,325 - root - INFO - lr: 3.1609e-05 gnorm: 1.02 [20:56:20< 1:04:18] +[titan] 2025-07-10 10:31:28,284 - root - INFO - step: 95135 loss: 16.0200 memory: 44.58GiB(31.99%) tps: 82,763 tflops: 285.63 mfu: 28.88% global_avg_ntp_loss: 2.4323 global_avg_mtp_loss: 13.5877 +[titan] 2025-07-10 10:31:28,284 - root - INFO - lr: 3.1606e-05 gnorm: 1.17 [20:56:24< 1:04:14] +[titan] 2025-07-10 10:31:32,227 - root - INFO - step: 95140 loss: 16.1998 memory: 44.58GiB(31.99%) tps: 83,119 tflops: 286.86 mfu: 29.00% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7120 +[titan] 2025-07-10 10:31:32,227 - root - INFO - lr: 3.1602e-05 gnorm: 1.02 [20:56:28< 1:04:11] +[titan] 2025-07-10 10:31:36,152 - root - INFO - step: 95145 loss: 16.0919 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.6128 +[titan] 2025-07-10 10:31:36,153 - root - INFO - lr: 3.1599e-05 gnorm: 1.07 [20:56:31< 1:04:07] +[titan] 2025-07-10 10:31:39,260 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:31:40,048 - root - INFO - step: 95150 loss: 16.2270 memory: 44.58GiB(31.99%) tps: 84,128 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.7598 +[titan] 2025-07-10 10:31:40,048 - root - INFO - lr: 3.1596e-05 gnorm: 1.05 [20:56:35< 1:04:03] +[titan] 2025-07-10 10:31:44,002 - root - INFO - step: 95155 loss: 16.2996 memory: 44.58GiB(31.99%) tps: 82,883 tflops: 286.04 mfu: 28.92% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.8213 +[titan] 2025-07-10 10:31:44,002 - root - INFO - lr: 3.1592e-05 gnorm: 1.03 [20:56:39< 1:03:59] +[titan] 2025-07-10 10:31:47,924 - root - INFO - step: 95160 loss: 15.8434 memory: 44.58GiB(31.99%) tps: 83,543 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4111 global_avg_mtp_loss: 13.4323 +[titan] 2025-07-10 10:31:47,925 - root - INFO - lr: 3.1589e-05 gnorm: 1.04 [20:56:43< 1:03:55] +[titan] 2025-07-10 10:31:51,877 - root - INFO - step: 95165 loss: 16.4297 memory: 44.58GiB(31.99%) tps: 82,905 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.9297 +[titan] 2025-07-10 10:31:51,878 - root - INFO - lr: 3.1586e-05 gnorm: 1.05 [20:56:47< 1:03:51] +[titan] 2025-07-10 10:31:55,801 - root - INFO - step: 95170 loss: 16.4498 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.5249 global_avg_mtp_loss: 13.9249 +[titan] 2025-07-10 10:31:55,801 - root - INFO - lr: 3.1583e-05 gnorm: 1.05 [20:56:51< 1:03:47] +[titan] 2025-07-10 10:31:59,701 - root - INFO - step: 95175 loss: 16.3572 memory: 44.58GiB(31.99%) tps: 84,024 tflops: 289.98 mfu: 29.32% global_avg_ntp_loss: 2.5127 global_avg_mtp_loss: 13.8444 +[titan] 2025-07-10 10:31:59,701 - root - INFO - lr: 3.1579e-05 gnorm: 1.04 [20:56:55< 1:03:43] +[titan] 2025-07-10 10:32:03,606 - root - INFO - step: 95180 loss: 16.2758 memory: 44.58GiB(31.99%) tps: 83,928 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4823 global_avg_mtp_loss: 13.7935 +[titan] 2025-07-10 10:32:03,606 - root - INFO - lr: 3.1576e-05 gnorm: 1.06 [20:56:59< 1:03:39] +[titan] 2025-07-10 10:32:07,543 - root - INFO - step: 95185 loss: 16.2460 memory: 44.58GiB(31.99%) tps: 83,244 tflops: 287.29 mfu: 29.05% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7716 +[titan] 2025-07-10 10:32:07,543 - root - INFO - lr: 3.1573e-05 gnorm: 1.03 [20:57:03< 1:03:35] +[titan] 2025-07-10 10:32:11,466 - root - INFO - step: 95190 loss: 16.1566 memory: 44.58GiB(31.99%) tps: 83,523 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6938 +[titan] 2025-07-10 10:32:11,467 - root - INFO - lr: 3.1570e-05 gnorm: 1.04 [20:57:07< 1:03:31] +[titan] 2025-07-10 10:32:15,396 - root - INFO - step: 95195 loss: 16.4096 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.9046 +[titan] 2025-07-10 10:32:15,396 - root - INFO - lr: 3.1566e-05 gnorm: 1.00 [20:57:11< 1:03:27] +[titan] 2025-07-10 10:32:18,530 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:32:19,317 - root - INFO - step: 95200 loss: 16.3065 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4981 global_avg_mtp_loss: 13.8085 +[titan] 2025-07-10 10:32:19,318 - root - INFO - lr: 3.1563e-05 gnorm: 1.08 [20:57:15< 1:03:23] +[titan] 2025-07-10 10:32:23,232 - root - INFO - step: 95205 loss: 16.1680 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.94 mfu: 29.21% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.7030 +[titan] 2025-07-10 10:32:23,232 - root - INFO - lr: 3.1560e-05 gnorm: 1.04 [20:57:19< 1:03:19] +[titan] 2025-07-10 10:32:27,159 - root - INFO - step: 95210 loss: 16.3758 memory: 44.58GiB(31.99%) tps: 83,443 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.5000 global_avg_mtp_loss: 13.8758 +[titan] 2025-07-10 10:32:27,159 - root - INFO - lr: 3.1557e-05 gnorm: 1.03 [20:57:22< 1:03:15] +[titan] 2025-07-10 10:32:31,075 - root - INFO - step: 95215 loss: 15.9745 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4246 global_avg_mtp_loss: 13.5499 +[titan] 2025-07-10 10:32:31,076 - root - INFO - lr: 3.1553e-05 gnorm: 1.04 [20:57:26< 1:03:11] +[titan] 2025-07-10 10:32:34,984 - root - INFO - step: 95220 loss: 16.1861 memory: 44.58GiB(31.99%) tps: 83,847 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.7164 +[titan] 2025-07-10 10:32:34,984 - root - INFO - lr: 3.1550e-05 gnorm: 1.04 [20:57:30< 1:03:07] +[titan] 2025-07-10 10:32:38,899 - root - INFO - step: 95225 loss: 16.1866 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7178 +[titan] 2025-07-10 10:32:38,900 - root - INFO - lr: 3.1547e-05 gnorm: 1.06 [20:57:34< 1:03:03] +[titan] 2025-07-10 10:32:42,895 - root - INFO - step: 95230 loss: 16.0047 memory: 44.58GiB(31.99%) tps: 82,015 tflops: 283.05 mfu: 28.62% global_avg_ntp_loss: 2.4410 global_avg_mtp_loss: 13.5637 +[titan] 2025-07-10 10:32:42,895 - root - INFO - lr: 3.1544e-05 gnorm: 1.07 [20:57:38< 1:02:59] +[titan] 2025-07-10 10:32:44,616 - root - INFO - Dumping profiler traces at step 95232 +[titan] 2025-07-10 10:32:44,650 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:32:46,998 - root - INFO - step: 95235 loss: 15.9478 memory: 44.58GiB(31.99%) tps: 79,871 tflops: 275.65 mfu: 27.87% global_avg_ntp_loss: 2.4172 global_avg_mtp_loss: 13.5305 +[titan] 2025-07-10 10:32:46,998 - root - INFO - lr: 3.1540e-05 gnorm: 1.05 [20:57:42< 1:02:55] +[titan] 2025-07-10 10:32:50,922 - root - INFO - step: 95240 loss: 16.1235 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4585 global_avg_mtp_loss: 13.6650 +[titan] 2025-07-10 10:32:50,923 - root - INFO - lr: 3.1537e-05 gnorm: 1.04 [20:57:46< 1:02:51] +[titan] 2025-07-10 10:32:54,829 - root - INFO - step: 95245 loss: 16.1720 memory: 44.58GiB(31.99%) tps: 83,888 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.7078 +[titan] 2025-07-10 10:32:54,829 - root - INFO - lr: 3.1534e-05 gnorm: 1.03 [20:57:50< 1:02:47] +[titan] 2025-07-10 10:32:57,944 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:32:58,734 - root - INFO - step: 95250 loss: 16.1042 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6460 +[titan] 2025-07-10 10:32:58,734 - root - INFO - lr: 3.1531e-05 gnorm: 1.06 [20:57:54< 1:02:43] +[titan] 2025-07-10 10:33:02,627 - root - INFO - step: 95255 loss: 16.0890 memory: 44.58GiB(31.99%) tps: 84,177 tflops: 290.51 mfu: 29.37% global_avg_ntp_loss: 2.4485 global_avg_mtp_loss: 13.6405 +[titan] 2025-07-10 10:33:02,627 - root - INFO - lr: 3.1527e-05 gnorm: 1.05 [20:57:58< 1:02:39] +[titan] 2025-07-10 10:33:06,542 - root - INFO - step: 95260 loss: 16.3485 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.5069 global_avg_mtp_loss: 13.8415 +[titan] 2025-07-10 10:33:06,542 - root - INFO - lr: 3.1524e-05 gnorm: 1.06 [20:58:02< 1:02:35] +[titan] 2025-07-10 10:33:10,471 - root - INFO - step: 95265 loss: 16.1659 memory: 44.58GiB(31.99%) tps: 83,404 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.6909 +[titan] 2025-07-10 10:33:10,471 - root - INFO - lr: 3.1521e-05 gnorm: 1.14 [20:58:06< 1:02:31] +[titan] 2025-07-10 10:33:14,382 - root - INFO - step: 95270 loss: 16.2213 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4783 global_avg_mtp_loss: 13.7430 +[titan] 2025-07-10 10:33:14,382 - root - INFO - lr: 3.1518e-05 gnorm: 1.14 [20:58:10< 1:02:27] +[titan] 2025-07-10 10:33:18,281 - root - INFO - step: 95275 loss: 16.4285 memory: 44.58GiB(31.99%) tps: 84,047 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.5135 global_avg_mtp_loss: 13.9151 +[titan] 2025-07-10 10:33:18,281 - root - INFO - lr: 3.1515e-05 gnorm: 1.05 [20:58:14< 1:02:23] +[titan] 2025-07-10 10:33:22,205 - root - INFO - step: 95280 loss: 16.1512 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4572 global_avg_mtp_loss: 13.6939 +[titan] 2025-07-10 10:33:22,206 - root - INFO - lr: 3.1511e-05 gnorm: 1.11 [20:58:17< 1:02:20] +[titan] 2025-07-10 10:33:26,104 - root - INFO - step: 95285 loss: 16.1469 memory: 44.58GiB(31.99%) tps: 84,068 tflops: 290.13 mfu: 29.34% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.6743 +[titan] 2025-07-10 10:33:26,104 - root - INFO - lr: 3.1508e-05 gnorm: 1.06 [20:58:21< 1:02:16] +[titan] 2025-07-10 10:33:30,024 - root - INFO - step: 95290 loss: 16.2364 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7589 +[titan] 2025-07-10 10:33:30,024 - root - INFO - lr: 3.1505e-05 gnorm: 1.03 [20:58:25< 1:02:12] +[titan] 2025-07-10 10:33:33,941 - root - INFO - step: 95295 loss: 16.1693 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.6905 +[titan] 2025-07-10 10:33:33,941 - root - INFO - lr: 3.1502e-05 gnorm: 1.10 [20:58:29< 1:02:08] +[titan] 2025-07-10 10:33:37,042 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:33:37,831 - root - INFO - step: 95300 loss: 16.2401 memory: 44.58GiB(31.99%) tps: 84,255 tflops: 290.78 mfu: 29.40% global_avg_ntp_loss: 2.4778 global_avg_mtp_loss: 13.7623 +[titan] 2025-07-10 10:33:37,831 - root - INFO - lr: 3.1499e-05 gnorm: 1.12 [20:58:33< 1:02:04] +[titan] 2025-07-10 10:33:41,806 - root - INFO - step: 95305 loss: 16.0600 memory: 44.58GiB(31.99%) tps: 82,446 tflops: 284.53 mfu: 28.77% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.6091 +[titan] 2025-07-10 10:33:41,806 - root - INFO - lr: 3.1496e-05 gnorm: 1.08 [20:58:37< 1:02:00] +[titan] 2025-07-10 10:33:45,717 - root - INFO - step: 95310 loss: 16.1455 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4590 global_avg_mtp_loss: 13.6865 +[titan] 2025-07-10 10:33:45,718 - root - INFO - lr: 3.1492e-05 gnorm: 1.07 [20:58:41< 1:01:56] +[titan] 2025-07-10 10:33:49,636 - root - INFO - step: 95315 loss: 16.0973 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6369 +[titan] 2025-07-10 10:33:49,636 - root - INFO - lr: 3.1489e-05 gnorm: 1.02 [20:58:45< 1:01:52] +[titan] 2025-07-10 10:33:53,562 - root - INFO - step: 95320 loss: 15.9940 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.5517 +[titan] 2025-07-10 10:33:53,562 - root - INFO - lr: 3.1486e-05 gnorm: 1.07 [20:58:49< 1:01:48] +[titan] 2025-07-10 10:33:57,460 - root - INFO - step: 95325 loss: 15.9639 memory: 44.58GiB(31.99%) tps: 84,063 tflops: 290.12 mfu: 29.33% global_avg_ntp_loss: 2.4327 global_avg_mtp_loss: 13.5312 +[titan] 2025-07-10 10:33:57,461 - root - INFO - lr: 3.1483e-05 gnorm: 1.05 [20:58:53< 1:01:44] +[titan] 2025-07-10 10:34:01,354 - root - INFO - step: 95330 loss: 16.2049 memory: 44.58GiB(31.99%) tps: 84,163 tflops: 290.46 mfu: 29.37% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.7399 +[titan] 2025-07-10 10:34:01,354 - root - INFO - lr: 3.1480e-05 gnorm: 1.00 [20:58:57< 1:01:40] +[titan] 2025-07-10 10:34:05,263 - root - INFO - step: 95335 loss: 16.2454 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.7532 +[titan] 2025-07-10 10:34:05,263 - root - INFO - lr: 3.1477e-05 gnorm: 1.04 [20:59:01< 1:01:36] +[titan] 2025-07-10 10:34:09,193 - root - INFO - step: 95340 loss: 16.2115 memory: 44.58GiB(31.99%) tps: 83,388 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7253 +[titan] 2025-07-10 10:34:09,193 - root - INFO - lr: 3.1473e-05 gnorm: 1.08 [20:59:04< 1:01:32] +[titan] 2025-07-10 10:34:13,124 - root - INFO - step: 95345 loss: 16.0790 memory: 44.58GiB(31.99%) tps: 83,370 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4507 global_avg_mtp_loss: 13.6283 +[titan] 2025-07-10 10:34:13,124 - root - INFO - lr: 3.1470e-05 gnorm: 1.04 [20:59:08< 1:01:28] +[titan] 2025-07-10 10:34:16,250 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:34:17,049 - root - INFO - step: 95350 loss: 16.2275 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7411 +[titan] 2025-07-10 10:34:17,050 - root - INFO - lr: 3.1467e-05 gnorm: 1.02 [20:59:12< 1:01:24] +[titan] 2025-07-10 10:34:20,993 - root - INFO - step: 95355 loss: 16.4807 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.5455 global_avg_mtp_loss: 13.9352 +[titan] 2025-07-10 10:34:20,993 - root - INFO - lr: 3.1464e-05 gnorm: 1.11 [20:59:16< 1:01:20] +[titan] 2025-07-10 10:34:24,912 - root - INFO - step: 95360 loss: 16.2248 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.7441 +[titan] 2025-07-10 10:34:24,912 - root - INFO - lr: 3.1461e-05 gnorm: 1.05 [20:59:20< 1:01:16] +[titan] 2025-07-10 10:34:28,846 - root - INFO - step: 95365 loss: 16.3281 memory: 44.58GiB(31.99%) tps: 83,296 tflops: 287.47 mfu: 29.07% global_avg_ntp_loss: 2.4934 global_avg_mtp_loss: 13.8347 +[titan] 2025-07-10 10:34:28,847 - root - INFO - lr: 3.1458e-05 gnorm: 1.03 [20:59:24< 1:01:12] +[titan] 2025-07-10 10:34:32,795 - root - INFO - step: 95370 loss: 16.1958 memory: 44.58GiB(31.99%) tps: 82,995 tflops: 286.43 mfu: 28.96% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.7291 +[titan] 2025-07-10 10:34:32,795 - root - INFO - lr: 3.1454e-05 gnorm: 1.04 [20:59:28< 1:01:08] +[titan] 2025-07-10 10:34:36,771 - root - INFO - step: 95375 loss: 16.1169 memory: 44.58GiB(31.99%) tps: 82,428 tflops: 284.47 mfu: 28.76% global_avg_ntp_loss: 2.4657 global_avg_mtp_loss: 13.6512 +[titan] 2025-07-10 10:34:36,771 - root - INFO - lr: 3.1451e-05 gnorm: 1.05 [20:59:32< 1:01:04] +[titan] 2025-07-10 10:34:40,682 - root - INFO - step: 95380 loss: 16.1013 memory: 44.58GiB(31.99%) tps: 83,796 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4452 global_avg_mtp_loss: 13.6561 +[titan] 2025-07-10 10:34:40,682 - root - INFO - lr: 3.1448e-05 gnorm: 1.05 [20:59:36< 1:01:00] +[titan] 2025-07-10 10:34:44,595 - root - INFO - step: 95385 loss: 16.0646 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4558 global_avg_mtp_loss: 13.6088 +[titan] 2025-07-10 10:34:44,596 - root - INFO - lr: 3.1445e-05 gnorm: 1.05 [20:59:40< 1:00:56] +[titan] 2025-07-10 10:34:48,516 - root - INFO - step: 95390 loss: 16.3145 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.5056 global_avg_mtp_loss: 13.8089 +[titan] 2025-07-10 10:34:48,516 - root - INFO - lr: 3.1442e-05 gnorm: 1.06 [20:59:44< 1:00:52] +[titan] 2025-07-10 10:34:52,461 - root - INFO - step: 95395 loss: 16.2206 memory: 44.58GiB(31.99%) tps: 83,062 tflops: 286.66 mfu: 28.99% global_avg_ntp_loss: 2.4771 global_avg_mtp_loss: 13.7435 +[titan] 2025-07-10 10:34:52,462 - root - INFO - lr: 3.1439e-05 gnorm: 1.04 [20:59:48< 1:00:48] +[titan] 2025-07-10 10:34:55,651 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:34:56,443 - root - INFO - step: 95400 loss: 15.9875 memory: 44.58GiB(31.99%) tps: 82,312 tflops: 284.07 mfu: 28.72% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.5413 +[titan] 2025-07-10 10:34:56,443 - root - INFO - lr: 3.1436e-05 gnorm: 1.21 [20:59:52< 1:00:44] +[titan] 2025-07-10 10:35:00,396 - root - INFO - step: 95405 loss: 15.9993 memory: 44.58GiB(31.99%) tps: 82,891 tflops: 286.07 mfu: 28.93% global_avg_ntp_loss: 2.4451 global_avg_mtp_loss: 13.5542 +[titan] 2025-07-10 10:35:00,397 - root - INFO - lr: 3.1433e-05 gnorm: 1.10 [20:59:56< 1:00:40] +[titan] 2025-07-10 10:35:04,345 - root - INFO - step: 95410 loss: 16.3396 memory: 44.58GiB(31.99%) tps: 82,993 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8440 +[titan] 2025-07-10 10:35:04,345 - root - INFO - lr: 3.1429e-05 gnorm: 1.06 [21:00:00< 1:00:36] +[titan] 2025-07-10 10:35:08,260 - root - INFO - step: 95415 loss: 16.0389 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.5957 +[titan] 2025-07-10 10:35:08,260 - root - INFO - lr: 3.1426e-05 gnorm: 1.03 [21:00:04< 1:00:33] +[titan] 2025-07-10 10:35:12,166 - root - INFO - step: 95420 loss: 16.1073 memory: 44.58GiB(31.99%) tps: 83,904 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6502 +[titan] 2025-07-10 10:35:12,166 - root - INFO - lr: 3.1423e-05 gnorm: 1.07 [21:00:07< 1:00:29] +[titan] 2025-07-10 10:35:16,103 - root - INFO - step: 95425 loss: 16.0291 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4418 global_avg_mtp_loss: 13.5873 +[titan] 2025-07-10 10:35:16,103 - root - INFO - lr: 3.1420e-05 gnorm: 1.09 [21:00:11< 1:00:25] +[titan] 2025-07-10 10:35:20,069 - root - INFO - step: 95430 loss: 16.1034 memory: 44.58GiB(31.99%) tps: 82,620 tflops: 285.13 mfu: 28.83% global_avg_ntp_loss: 2.4482 global_avg_mtp_loss: 13.6552 +[titan] 2025-07-10 10:35:20,070 - root - INFO - lr: 3.1417e-05 gnorm: 1.07 [21:00:15< 1:00:21] +[titan] 2025-07-10 10:35:23,978 - root - INFO - step: 95435 loss: 15.7543 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4048 global_avg_mtp_loss: 13.3495 +[titan] 2025-07-10 10:35:23,978 - root - INFO - lr: 3.1414e-05 gnorm: 1.09 [21:00:19< 1:00:17] +[titan] 2025-07-10 10:35:27,883 - root - INFO - step: 95440 loss: 16.1108 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6520 +[titan] 2025-07-10 10:35:27,883 - root - INFO - lr: 3.1411e-05 gnorm: 1.06 [21:00:23< 1:00:13] +[titan] 2025-07-10 10:35:31,801 - root - INFO - step: 95445 loss: 16.1862 memory: 44.58GiB(31.99%) tps: 83,626 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.7233 +[titan] 2025-07-10 10:35:31,802 - root - INFO - lr: 3.1408e-05 gnorm: 1.04 [21:00:27< 1:00:09] +[titan] 2025-07-10 10:35:34,939 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:35:35,735 - root - INFO - step: 95450 loss: 16.1899 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4629 global_avg_mtp_loss: 13.7270 +[titan] 2025-07-10 10:35:35,735 - root - INFO - lr: 3.1405e-05 gnorm: 1.02 [21:00:31< 1:00:05] +[titan] 2025-07-10 10:35:39,666 - root - INFO - step: 95455 loss: 16.3605 memory: 44.58GiB(31.99%) tps: 83,359 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8594 +[titan] 2025-07-10 10:35:39,666 - root - INFO - lr: 3.1402e-05 gnorm: 1.05 [21:00:35< 1:00:01] +[titan] 2025-07-10 10:35:43,615 - root - INFO - step: 95460 loss: 16.2782 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7965 +[titan] 2025-07-10 10:35:43,615 - root - INFO - lr: 3.1399e-05 gnorm: 1.06 [21:00:39< 0:59:57] +[titan] 2025-07-10 10:35:47,540 - root - INFO - step: 95465 loss: 15.9167 memory: 44.58GiB(31.99%) tps: 83,496 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4206 global_avg_mtp_loss: 13.4961 +[titan] 2025-07-10 10:35:47,540 - root - INFO - lr: 3.1395e-05 gnorm: 1.03 [21:00:43< 0:59:53] +[titan] 2025-07-10 10:35:51,459 - root - INFO - step: 95470 loss: 16.1980 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4721 global_avg_mtp_loss: 13.7259 +[titan] 2025-07-10 10:35:51,459 - root - INFO - lr: 3.1392e-05 gnorm: 1.03 [21:00:47< 0:59:49] +[titan] 2025-07-10 10:35:55,379 - root - INFO - step: 95475 loss: 15.9960 memory: 44.58GiB(31.99%) tps: 83,609 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4379 global_avg_mtp_loss: 13.5581 +[titan] 2025-07-10 10:35:55,379 - root - INFO - lr: 3.1389e-05 gnorm: 1.11 [21:00:51< 0:59:45] +[titan] 2025-07-10 10:35:59,311 - root - INFO - step: 95480 loss: 16.1815 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4693 global_avg_mtp_loss: 13.7122 +[titan] 2025-07-10 10:35:59,312 - root - INFO - lr: 3.1386e-05 gnorm: 1.02 [21:00:55< 0:59:41] +[titan] 2025-07-10 10:36:03,267 - root - INFO - step: 95485 loss: 16.3489 memory: 44.58GiB(31.99%) tps: 82,846 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.5036 global_avg_mtp_loss: 13.8453 +[titan] 2025-07-10 10:36:03,267 - root - INFO - lr: 3.1383e-05 gnorm: 1.00 [21:00:59< 0:59:37] +[titan] 2025-07-10 10:36:07,163 - root - INFO - step: 95490 loss: 16.3232 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4927 global_avg_mtp_loss: 13.8306 +[titan] 2025-07-10 10:36:07,163 - root - INFO - lr: 3.1380e-05 gnorm: 1.05 [21:01:02< 0:59:33] +[titan] 2025-07-10 10:36:11,081 - root - INFO - step: 95495 loss: 16.0919 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6415 +[titan] 2025-07-10 10:36:11,081 - root - INFO - lr: 3.1377e-05 gnorm: 1.04 [21:01:06< 0:59:29] +[titan] 2025-07-10 10:36:14,206 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:36:14,992 - root - INFO - step: 95500 loss: 16.1124 memory: 44.58GiB(31.99%) tps: 83,794 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6443 +[titan] 2025-07-10 10:36:14,992 - root - INFO - lr: 3.1374e-05 gnorm: 1.04 [21:01:10< 0:59:25] +[titan] 2025-07-10 10:36:18,922 - root - INFO - step: 95505 loss: 16.3040 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.8131 +[titan] 2025-07-10 10:36:18,922 - root - INFO - lr: 3.1371e-05 gnorm: 0.98 [21:01:14< 0:59:21] +[titan] 2025-07-10 10:36:22,867 - root - INFO - step: 95510 loss: 16.0930 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6312 +[titan] 2025-07-10 10:36:22,867 - root - INFO - lr: 3.1368e-05 gnorm: 1.03 [21:01:18< 0:59:17] +[titan] 2025-07-10 10:36:26,784 - root - INFO - step: 95515 loss: 16.2555 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7845 +[titan] 2025-07-10 10:36:26,784 - root - INFO - lr: 3.1365e-05 gnorm: 1.03 [21:01:22< 0:59:13] +[titan] 2025-07-10 10:36:30,707 - root - INFO - step: 95520 loss: 16.1671 memory: 44.58GiB(31.99%) tps: 83,541 tflops: 288.31 mfu: 29.15% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.6999 +[titan] 2025-07-10 10:36:30,707 - root - INFO - lr: 3.1362e-05 gnorm: 1.02 [21:01:26< 0:59:09] +[titan] 2025-07-10 10:36:34,660 - root - INFO - step: 95525 loss: 16.3975 memory: 44.58GiB(31.99%) tps: 82,904 tflops: 286.12 mfu: 28.93% global_avg_ntp_loss: 2.5097 global_avg_mtp_loss: 13.8878 +[titan] 2025-07-10 10:36:34,660 - root - INFO - lr: 3.1359e-05 gnorm: 1.06 [21:01:30< 0:59:05] +[titan] 2025-07-10 10:36:38,609 - root - INFO - step: 95530 loss: 15.9711 memory: 44.58GiB(31.99%) tps: 82,971 tflops: 286.35 mfu: 28.95% global_avg_ntp_loss: 2.4125 global_avg_mtp_loss: 13.5586 +[titan] 2025-07-10 10:36:38,610 - root - INFO - lr: 3.1356e-05 gnorm: 1.07 [21:01:34< 0:59:01] +[titan] 2025-07-10 10:36:42,575 - root - INFO - step: 95535 loss: 16.0210 memory: 44.58GiB(31.99%) tps: 82,630 tflops: 285.17 mfu: 28.83% global_avg_ntp_loss: 2.4336 global_avg_mtp_loss: 13.5874 +[titan] 2025-07-10 10:36:42,576 - root - INFO - lr: 3.1353e-05 gnorm: 1.04 [21:01:38< 0:58:57] +[titan] 2025-07-10 10:36:46,513 - root - INFO - step: 95540 loss: 15.9163 memory: 44.58GiB(31.99%) tps: 83,231 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4209 global_avg_mtp_loss: 13.4954 +[titan] 2025-07-10 10:36:46,513 - root - INFO - lr: 3.1350e-05 gnorm: 1.06 [21:01:42< 0:58:53] +[titan] 2025-07-10 10:36:50,431 - root - INFO - step: 95545 loss: 16.2510 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4878 global_avg_mtp_loss: 13.7632 +[titan] 2025-07-10 10:36:50,431 - root - INFO - lr: 3.1347e-05 gnorm: 1.07 [21:01:46< 0:58:49] +[titan] 2025-07-10 10:36:53,606 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:36:54,408 - root - INFO - step: 95550 loss: 15.9376 memory: 44.58GiB(31.99%) tps: 82,409 tflops: 284.41 mfu: 28.76% global_avg_ntp_loss: 2.4405 global_avg_mtp_loss: 13.4971 +[titan] 2025-07-10 10:36:54,408 - root - INFO - lr: 3.1344e-05 gnorm: 1.08 [21:01:50< 0:58:46] +[titan] 2025-07-10 10:36:58,348 - root - INFO - step: 95555 loss: 16.1772 memory: 44.58GiB(31.99%) tps: 83,172 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.6971 +[titan] 2025-07-10 10:36:58,348 - root - INFO - lr: 3.1341e-05 gnorm: 1.09 [21:01:54< 0:58:42] +[titan] 2025-07-10 10:37:02,291 - root - INFO - step: 95560 loss: 15.9963 memory: 44.58GiB(31.99%) tps: 83,115 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4269 global_avg_mtp_loss: 13.5694 +[titan] 2025-07-10 10:37:02,291 - root - INFO - lr: 3.1338e-05 gnorm: 1.11 [21:01:58< 0:58:38] +[titan] 2025-07-10 10:37:06,223 - root - INFO - step: 95565 loss: 15.9827 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4400 global_avg_mtp_loss: 13.5427 +[titan] 2025-07-10 10:37:06,223 - root - INFO - lr: 3.1335e-05 gnorm: 1.14 [21:02:01< 0:58:34] +[titan] 2025-07-10 10:37:10,144 - root - INFO - step: 95570 loss: 16.1435 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6793 +[titan] 2025-07-10 10:37:10,144 - root - INFO - lr: 3.1332e-05 gnorm: 1.06 [21:02:05< 0:58:30] +[titan] 2025-07-10 10:37:14,065 - root - INFO - step: 95575 loss: 16.0403 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4365 global_avg_mtp_loss: 13.6038 +[titan] 2025-07-10 10:37:14,065 - root - INFO - lr: 3.1329e-05 gnorm: 1.02 [21:02:09< 0:58:26] +[titan] 2025-07-10 10:37:17,982 - root - INFO - step: 95580 loss: 16.0068 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.5748 +[titan] 2025-07-10 10:37:17,982 - root - INFO - lr: 3.1326e-05 gnorm: 1.08 [21:02:13< 0:58:22] +[titan] 2025-07-10 10:37:21,912 - root - INFO - step: 95585 loss: 16.1025 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6488 +[titan] 2025-07-10 10:37:21,912 - root - INFO - lr: 3.1323e-05 gnorm: 1.10 [21:02:17< 0:58:18] +[titan] 2025-07-10 10:37:25,819 - root - INFO - step: 95590 loss: 16.2436 memory: 44.58GiB(31.99%) tps: 83,884 tflops: 289.50 mfu: 29.27% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7693 +[titan] 2025-07-10 10:37:25,819 - root - INFO - lr: 3.1320e-05 gnorm: 1.06 [21:02:21< 0:58:14] +[titan] 2025-07-10 10:37:29,720 - root - INFO - step: 95595 loss: 16.3441 memory: 44.58GiB(31.99%) tps: 84,002 tflops: 289.90 mfu: 29.31% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8435 +[titan] 2025-07-10 10:37:29,720 - root - INFO - lr: 3.1317e-05 gnorm: 1.02 [21:02:25< 0:58:10] +[titan] 2025-07-10 10:37:32,833 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:37:33,622 - root - INFO - step: 95600 loss: 16.2427 memory: 44.58GiB(31.99%) tps: 83,981 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7607 +[titan] 2025-07-10 10:37:33,623 - root - INFO - lr: 3.1314e-05 gnorm: 1.09 [21:02:29< 0:58:06] +[titan] 2025-07-10 10:37:37,527 - root - INFO - step: 95605 loss: 16.1322 memory: 44.58GiB(31.99%) tps: 83,924 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.6669 +[titan] 2025-07-10 10:37:37,528 - root - INFO - lr: 3.1311e-05 gnorm: 1.08 [21:02:33< 0:58:02] +[titan] 2025-07-10 10:37:41,424 - root - INFO - step: 95610 loss: 16.1392 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6828 +[titan] 2025-07-10 10:37:41,424 - root - INFO - lr: 3.1308e-05 gnorm: 1.12 [21:02:37< 0:57:58] +[titan] 2025-07-10 10:37:45,372 - root - INFO - step: 95615 loss: 16.0849 memory: 44.58GiB(31.99%) tps: 83,009 tflops: 286.48 mfu: 28.97% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6319 +[titan] 2025-07-10 10:37:45,372 - root - INFO - lr: 3.1305e-05 gnorm: 1.03 [21:02:41< 0:57:54] +[titan] 2025-07-10 10:37:49,334 - root - INFO - step: 95620 loss: 16.1260 memory: 44.58GiB(31.99%) tps: 82,710 tflops: 285.44 mfu: 28.86% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6671 +[titan] 2025-07-10 10:37:49,334 - root - INFO - lr: 3.1302e-05 gnorm: 1.06 [21:02:45< 0:57:50] +[titan] 2025-07-10 10:37:53,267 - root - INFO - step: 95625 loss: 16.3572 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4982 global_avg_mtp_loss: 13.8590 +[titan] 2025-07-10 10:37:53,267 - root - INFO - lr: 3.1299e-05 gnorm: 1.10 [21:02:49< 0:57:46] +[titan] 2025-07-10 10:37:57,184 - root - INFO - step: 95630 loss: 16.1521 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6890 +[titan] 2025-07-10 10:37:57,184 - root - INFO - lr: 3.1296e-05 gnorm: 1.02 [21:02:52< 0:57:42] +[titan] 2025-07-10 10:38:01,142 - root - INFO - step: 95635 loss: 16.2533 memory: 44.58GiB(31.99%) tps: 82,795 tflops: 285.74 mfu: 28.89% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7717 +[titan] 2025-07-10 10:38:01,142 - root - INFO - lr: 3.1293e-05 gnorm: 1.17 [21:02:56< 0:57:38] +[titan] 2025-07-10 10:38:05,067 - root - INFO - step: 95640 loss: 16.0250 memory: 44.58GiB(31.99%) tps: 83,486 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.5874 +[titan] 2025-07-10 10:38:05,067 - root - INFO - lr: 3.1290e-05 gnorm: 1.04 [21:03:00< 0:57:34] +[titan] 2025-07-10 10:38:08,978 - root - INFO - step: 95645 loss: 16.0218 memory: 44.58GiB(31.99%) tps: 83,791 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4320 global_avg_mtp_loss: 13.5898 +[titan] 2025-07-10 10:38:08,979 - root - INFO - lr: 3.1287e-05 gnorm: 1.04 [21:03:04< 0:57:30] +[titan] 2025-07-10 10:38:12,089 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:38:12,876 - root - INFO - step: 95650 loss: 16.2356 memory: 44.58GiB(31.99%) tps: 84,080 tflops: 290.17 mfu: 29.34% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.7513 +[titan] 2025-07-10 10:38:12,876 - root - INFO - lr: 3.1284e-05 gnorm: 1.02 [21:03:08< 0:57:26] +[titan] 2025-07-10 10:38:16,796 - root - INFO - step: 95655 loss: 16.3755 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.5107 global_avg_mtp_loss: 13.8648 +[titan] 2025-07-10 10:38:16,796 - root - INFO - lr: 3.1281e-05 gnorm: 1.03 [21:03:12< 0:57:22] +[titan] 2025-07-10 10:38:20,725 - root - INFO - step: 95660 loss: 16.1288 memory: 44.58GiB(31.99%) tps: 83,403 tflops: 287.84 mfu: 29.10% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.6554 +[titan] 2025-07-10 10:38:20,725 - root - INFO - lr: 3.1278e-05 gnorm: 1.10 [21:03:16< 0:57:18] +[titan] 2025-07-10 10:38:24,616 - root - INFO - step: 95665 loss: 16.1559 memory: 44.58GiB(31.99%) tps: 84,232 tflops: 290.70 mfu: 29.39% global_avg_ntp_loss: 2.4610 global_avg_mtp_loss: 13.6950 +[titan] 2025-07-10 10:38:24,616 - root - INFO - lr: 3.1275e-05 gnorm: 1.09 [21:03:20< 0:57:14] +[titan] 2025-07-10 10:38:28,517 - root - INFO - step: 95670 loss: 16.3510 memory: 44.58GiB(31.99%) tps: 83,988 tflops: 289.86 mfu: 29.31% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8541 +[titan] 2025-07-10 10:38:28,518 - root - INFO - lr: 3.1272e-05 gnorm: 1.01 [21:03:24< 0:57:10] +[titan] 2025-07-10 10:38:32,461 - root - INFO - step: 95675 loss: 16.1790 memory: 44.58GiB(31.99%) tps: 83,102 tflops: 286.80 mfu: 29.00% global_avg_ntp_loss: 2.4602 global_avg_mtp_loss: 13.7189 +[titan] 2025-07-10 10:38:32,461 - root - INFO - lr: 3.1269e-05 gnorm: 1.10 [21:03:28< 0:57:06] +[titan] 2025-07-10 10:38:36,425 - root - INFO - step: 95680 loss: 16.0695 memory: 44.58GiB(31.99%) tps: 82,664 tflops: 285.29 mfu: 28.85% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.6314 +[titan] 2025-07-10 10:38:36,426 - root - INFO - lr: 3.1267e-05 gnorm: 1.01 [21:03:32< 0:57:02] +[titan] 2025-07-10 10:38:40,356 - root - INFO - step: 95685 loss: 16.6290 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.5475 global_avg_mtp_loss: 14.0815 +[titan] 2025-07-10 10:38:40,356 - root - INFO - lr: 3.1264e-05 gnorm: 1.07 [21:03:36< 0:56:58] +[titan] 2025-07-10 10:38:44,279 - root - INFO - step: 95690 loss: 16.1038 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4547 global_avg_mtp_loss: 13.6491 +[titan] 2025-07-10 10:38:44,279 - root - INFO - lr: 3.1261e-05 gnorm: 1.03 [21:03:40< 0:56:55] +[titan] 2025-07-10 10:38:48,244 - root - INFO - step: 95695 loss: 16.1246 memory: 44.58GiB(31.99%) tps: 82,659 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.4635 global_avg_mtp_loss: 13.6611 +[titan] 2025-07-10 10:38:48,244 - root - INFO - lr: 3.1258e-05 gnorm: 0.98 [21:03:43< 0:56:51] +[titan] 2025-07-10 10:38:51,385 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:38:52,176 - root - INFO - step: 95700 loss: 16.3725 memory: 44.58GiB(31.99%) tps: 83,345 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8725 +[titan] 2025-07-10 10:38:52,176 - root - INFO - lr: 3.1255e-05 gnorm: 1.03 [21:03:47< 0:56:47] +[titan] 2025-07-10 10:38:56,096 - root - INFO - step: 95705 loss: 16.1956 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4636 global_avg_mtp_loss: 13.7320 +[titan] 2025-07-10 10:38:56,096 - root - INFO - lr: 3.1252e-05 gnorm: 1.13 [21:03:51< 0:56:43] +[titan] 2025-07-10 10:39:00,013 - root - INFO - step: 95710 loss: 16.0896 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4456 global_avg_mtp_loss: 13.6440 +[titan] 2025-07-10 10:39:00,014 - root - INFO - lr: 3.1249e-05 gnorm: 1.06 [21:03:55< 0:56:39] +[titan] 2025-07-10 10:39:03,940 - root - INFO - step: 95715 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 83,453 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7314 +[titan] 2025-07-10 10:39:03,940 - root - INFO - lr: 3.1246e-05 gnorm: 1.08 [21:03:59< 0:56:35] +[titan] 2025-07-10 10:39:07,857 - root - INFO - step: 95720 loss: 16.4894 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.5268 global_avg_mtp_loss: 13.9626 +[titan] 2025-07-10 10:39:07,857 - root - INFO - lr: 3.1243e-05 gnorm: 1.08 [21:04:03< 0:56:31] +[titan] 2025-07-10 10:39:11,762 - root - INFO - step: 95725 loss: 16.0912 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4515 global_avg_mtp_loss: 13.6397 +[titan] 2025-07-10 10:39:11,763 - root - INFO - lr: 3.1240e-05 gnorm: 1.08 [21:04:07< 0:56:27] +[titan] 2025-07-10 10:39:15,670 - root - INFO - step: 95730 loss: 16.2104 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7358 +[titan] 2025-07-10 10:39:15,670 - root - INFO - lr: 3.1237e-05 gnorm: 1.04 [21:04:11< 0:56:23] +[titan] 2025-07-10 10:39:19,577 - root - INFO - step: 95735 loss: 16.1134 memory: 44.58GiB(31.99%) tps: 83,874 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4429 global_avg_mtp_loss: 13.6705 +[titan] 2025-07-10 10:39:19,577 - root - INFO - lr: 3.1235e-05 gnorm: 1.10 [21:04:15< 0:56:19] +[titan] 2025-07-10 10:39:23,493 - root - INFO - step: 95740 loss: 16.1005 memory: 44.58GiB(31.99%) tps: 83,682 tflops: 288.80 mfu: 29.20% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.6345 +[titan] 2025-07-10 10:39:23,493 - root - INFO - lr: 3.1232e-05 gnorm: 1.15 [21:04:19< 0:56:15] +[titan] 2025-07-10 10:39:26,862 - root - INFO - Dumping profiler traces at step 95744 +[titan] 2025-07-10 10:39:26,896 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:39:27,698 - root - INFO - step: 95745 loss: 16.1439 memory: 44.58GiB(31.99%) tps: 77,944 tflops: 269.00 mfu: 27.20% global_avg_ntp_loss: 2.4725 global_avg_mtp_loss: 13.6714 +[titan] 2025-07-10 10:39:27,698 - root - INFO - lr: 3.1229e-05 gnorm: 1.24 [21:04:23< 0:56:11] +[titan] 2025-07-10 10:39:30,829 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:39:31,614 - root - INFO - step: 95750 loss: 15.8223 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4071 global_avg_mtp_loss: 13.4152 +[titan] 2025-07-10 10:39:31,615 - root - INFO - lr: 3.1226e-05 gnorm: 1.04 [21:04:27< 0:56:07] +[titan] 2025-07-10 10:39:35,559 - root - INFO - step: 95755 loss: 16.3710 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.8610 +[titan] 2025-07-10 10:39:35,559 - root - INFO - lr: 3.1223e-05 gnorm: 1.10 [21:04:31< 0:56:03] +[titan] 2025-07-10 10:39:39,504 - root - INFO - step: 95760 loss: 16.0806 memory: 44.58GiB(31.99%) tps: 83,070 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6117 +[titan] 2025-07-10 10:39:39,504 - root - INFO - lr: 3.1220e-05 gnorm: 1.02 [21:04:35< 0:55:59] +[titan] 2025-07-10 10:39:43,456 - root - INFO - step: 95765 loss: 16.1315 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.4433 global_avg_mtp_loss: 13.6882 +[titan] 2025-07-10 10:39:43,456 - root - INFO - lr: 3.1217e-05 gnorm: 1.04 [21:04:39< 0:55:55] +[titan] 2025-07-10 10:39:47,357 - root - INFO - step: 95770 loss: 16.3718 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5145 global_avg_mtp_loss: 13.8573 +[titan] 2025-07-10 10:39:47,357 - root - INFO - lr: 3.1214e-05 gnorm: 1.04 [21:04:43< 0:55:51] +[titan] 2025-07-10 10:39:51,275 - root - INFO - step: 95775 loss: 16.4383 memory: 44.58GiB(31.99%) tps: 83,640 tflops: 288.66 mfu: 29.19% global_avg_ntp_loss: 2.5214 global_avg_mtp_loss: 13.9169 +[titan] 2025-07-10 10:39:51,276 - root - INFO - lr: 3.1212e-05 gnorm: 1.07 [21:04:47< 0:55:47] +[titan] 2025-07-10 10:39:55,245 - root - INFO - step: 95780 loss: 15.9012 memory: 44.58GiB(31.99%) tps: 82,544 tflops: 284.87 mfu: 28.80% global_avg_ntp_loss: 2.4193 global_avg_mtp_loss: 13.4819 +[titan] 2025-07-10 10:39:55,246 - root - INFO - lr: 3.1209e-05 gnorm: 1.03 [21:04:50< 0:55:43] +[titan] 2025-07-10 10:39:59,162 - root - INFO - step: 95785 loss: 16.2032 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4773 global_avg_mtp_loss: 13.7259 +[titan] 2025-07-10 10:39:59,162 - root - INFO - lr: 3.1206e-05 gnorm: 1.05 [21:04:54< 0:55:39] +[titan] 2025-07-10 10:40:03,153 - root - INFO - step: 95790 loss: 16.0831 memory: 44.58GiB(31.99%) tps: 82,114 tflops: 283.39 mfu: 28.65% global_avg_ntp_loss: 2.4490 global_avg_mtp_loss: 13.6340 +[titan] 2025-07-10 10:40:03,153 - root - INFO - lr: 3.1203e-05 gnorm: 1.04 [21:04:58< 0:55:35] +[titan] 2025-07-10 10:40:07,075 - root - INFO - step: 95795 loss: 16.2457 memory: 44.58GiB(31.99%) tps: 83,554 tflops: 288.36 mfu: 29.16% global_avg_ntp_loss: 2.4846 global_avg_mtp_loss: 13.7611 +[titan] 2025-07-10 10:40:07,075 - root - INFO - lr: 3.1200e-05 gnorm: 1.11 [21:05:02< 0:55:31] +[titan] 2025-07-10 10:40:10,198 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:40:10,987 - root - INFO - step: 95800 loss: 16.3852 memory: 44.58GiB(31.99%) tps: 83,770 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.8894 +[titan] 2025-07-10 10:40:10,987 - root - INFO - lr: 3.1197e-05 gnorm: 1.04 [21:05:06< 0:55:27] +[titan] 2025-07-10 10:40:14,916 - root - INFO - step: 95805 loss: 15.9213 memory: 44.58GiB(31.99%) tps: 83,400 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4378 global_avg_mtp_loss: 13.4835 +[titan] 2025-07-10 10:40:14,917 - root - INFO - lr: 3.1194e-05 gnorm: 1.07 [21:05:10< 0:55:23] +[titan] 2025-07-10 10:40:18,845 - root - INFO - step: 95810 loss: 16.1738 memory: 44.58GiB(31.99%) tps: 83,414 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.7169 +[titan] 2025-07-10 10:40:18,846 - root - INFO - lr: 3.1192e-05 gnorm: 1.05 [21:05:14< 0:55:19] +[titan] 2025-07-10 10:40:22,776 - root - INFO - step: 95815 loss: 16.3327 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.5053 global_avg_mtp_loss: 13.8274 +[titan] 2025-07-10 10:40:22,776 - root - INFO - lr: 3.1189e-05 gnorm: 1.05 [21:05:18< 0:55:15] +[titan] 2025-07-10 10:40:26,677 - root - INFO - step: 95820 loss: 16.0852 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.6304 +[titan] 2025-07-10 10:40:26,677 - root - INFO - lr: 3.1186e-05 gnorm: 1.06 [21:05:22< 0:55:11] +[titan] 2025-07-10 10:40:30,588 - root - INFO - step: 95825 loss: 16.4778 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.5170 global_avg_mtp_loss: 13.9608 +[titan] 2025-07-10 10:40:30,588 - root - INFO - lr: 3.1183e-05 gnorm: 1.09 [21:05:26< 0:55:08] +[titan] 2025-07-10 10:40:34,510 - root - INFO - step: 95830 loss: 16.0317 memory: 44.58GiB(31.99%) tps: 83,556 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.5813 +[titan] 2025-07-10 10:40:34,510 - root - INFO - lr: 3.1180e-05 gnorm: 1.08 [21:05:30< 0:55:04] +[titan] 2025-07-10 10:40:38,429 - root - INFO - step: 95835 loss: 16.1947 memory: 44.58GiB(31.99%) tps: 83,619 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7232 +[titan] 2025-07-10 10:40:38,429 - root - INFO - lr: 3.1177e-05 gnorm: 1.09 [21:05:34< 0:55:00] +[titan] 2025-07-10 10:40:42,352 - root - INFO - step: 95840 loss: 16.3429 memory: 44.58GiB(31.99%) tps: 83,546 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4922 global_avg_mtp_loss: 13.8507 +[titan] 2025-07-10 10:40:42,352 - root - INFO - lr: 3.1175e-05 gnorm: 1.05 [21:05:38< 0:54:56] +[titan] 2025-07-10 10:40:46,241 - root - INFO - step: 95845 loss: 16.1931 memory: 44.58GiB(31.99%) tps: 84,254 tflops: 290.77 mfu: 29.40% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7224 +[titan] 2025-07-10 10:40:46,242 - root - INFO - lr: 3.1172e-05 gnorm: 1.06 [21:05:41< 0:54:52] +[titan] 2025-07-10 10:40:49,357 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:40:50,147 - root - INFO - step: 95850 loss: 15.8413 memory: 44.58GiB(31.99%) tps: 83,905 tflops: 289.57 mfu: 29.28% global_avg_ntp_loss: 2.4093 global_avg_mtp_loss: 13.4320 +[titan] 2025-07-10 10:40:50,147 - root - INFO - lr: 3.1169e-05 gnorm: 1.13 [21:05:45< 0:54:48] +[titan] 2025-07-10 10:40:54,070 - root - INFO - step: 95855 loss: 16.3545 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-10 10:40:54,070 - root - INFO - lr: 3.1166e-05 gnorm: 1.08 [21:05:49< 0:54:44] +[titan] 2025-07-10 10:40:58,019 - root - INFO - step: 95860 loss: 16.1144 memory: 44.58GiB(31.99%) tps: 82,976 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4652 global_avg_mtp_loss: 13.6492 +[titan] 2025-07-10 10:40:58,019 - root - INFO - lr: 3.1163e-05 gnorm: 1.05 [21:05:53< 0:54:40] +[titan] 2025-07-10 10:41:01,932 - root - INFO - step: 95865 loss: 16.3506 memory: 44.58GiB(31.99%) tps: 83,763 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.4933 global_avg_mtp_loss: 13.8573 +[titan] 2025-07-10 10:41:01,932 - root - INFO - lr: 3.1161e-05 gnorm: 1.05 [21:05:57< 0:54:36] +[titan] 2025-07-10 10:41:05,849 - root - INFO - step: 95870 loss: 16.1501 memory: 44.58GiB(31.99%) tps: 83,661 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.6913 +[titan] 2025-07-10 10:41:05,849 - root - INFO - lr: 3.1158e-05 gnorm: 1.05 [21:06:01< 0:54:32] +[titan] 2025-07-10 10:41:09,760 - root - INFO - step: 95875 loss: 16.1205 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4501 global_avg_mtp_loss: 13.6704 +[titan] 2025-07-10 10:41:09,761 - root - INFO - lr: 3.1155e-05 gnorm: 1.05 [21:06:05< 0:54:28] +[titan] 2025-07-10 10:41:13,685 - root - INFO - step: 95880 loss: 16.2382 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7587 +[titan] 2025-07-10 10:41:13,685 - root - INFO - lr: 3.1152e-05 gnorm: 1.05 [21:06:09< 0:54:24] +[titan] 2025-07-10 10:41:17,622 - root - INFO - step: 95885 loss: 16.0019 memory: 44.58GiB(31.99%) tps: 83,241 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4306 global_avg_mtp_loss: 13.5714 +[titan] 2025-07-10 10:41:17,622 - root - INFO - lr: 3.1149e-05 gnorm: 1.08 [21:06:13< 0:54:20] +[titan] 2025-07-10 10:41:21,564 - root - INFO - step: 95890 loss: 16.3138 memory: 44.58GiB(31.99%) tps: 83,137 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.8321 +[titan] 2025-07-10 10:41:21,564 - root - INFO - lr: 3.1147e-05 gnorm: 1.05 [21:06:17< 0:54:16] +[titan] 2025-07-10 10:41:25,507 - root - INFO - step: 95895 loss: 16.0188 memory: 44.58GiB(31.99%) tps: 83,121 tflops: 286.86 mfu: 29.01% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.5826 +[titan] 2025-07-10 10:41:25,507 - root - INFO - lr: 3.1144e-05 gnorm: 1.00 [21:06:21< 0:54:12] +[titan] 2025-07-10 10:41:28,651 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:41:29,438 - root - INFO - step: 95900 loss: 16.2033 memory: 44.58GiB(31.99%) tps: 83,360 tflops: 287.69 mfu: 29.09% global_avg_ntp_loss: 2.4791 global_avg_mtp_loss: 13.7242 +[titan] 2025-07-10 10:41:29,438 - root - INFO - lr: 3.1141e-05 gnorm: 1.02 [21:06:25< 0:54:08] +[titan] 2025-07-10 10:41:33,377 - root - INFO - step: 95905 loss: 16.2531 memory: 44.58GiB(31.99%) tps: 83,197 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4784 global_avg_mtp_loss: 13.7747 +[titan] 2025-07-10 10:41:33,377 - root - INFO - lr: 3.1138e-05 gnorm: 1.09 [21:06:29< 0:54:04] +[titan] 2025-07-10 10:41:37,293 - root - INFO - step: 95910 loss: 15.8403 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4086 global_avg_mtp_loss: 13.4317 +[titan] 2025-07-10 10:41:37,293 - root - INFO - lr: 3.1135e-05 gnorm: 1.08 [21:06:33< 0:54:00] +[titan] 2025-07-10 10:41:41,210 - root - INFO - step: 95915 loss: 16.1904 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4667 global_avg_mtp_loss: 13.7237 +[titan] 2025-07-10 10:41:41,210 - root - INFO - lr: 3.1133e-05 gnorm: 1.08 [21:06:36< 0:53:56] +[titan] 2025-07-10 10:41:45,120 - root - INFO - step: 95920 loss: 16.3346 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8406 +[titan] 2025-07-10 10:41:45,120 - root - INFO - lr: 3.1130e-05 gnorm: 1.00 [21:06:40< 0:53:52] +[titan] 2025-07-10 10:41:49,036 - root - INFO - step: 95925 loss: 15.9291 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4152 global_avg_mtp_loss: 13.5139 +[titan] 2025-07-10 10:41:49,037 - root - INFO - lr: 3.1127e-05 gnorm: 1.02 [21:06:44< 0:53:48] +[titan] 2025-07-10 10:41:52,954 - root - INFO - step: 95930 loss: 16.3272 memory: 44.58GiB(31.99%) tps: 83,655 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4969 global_avg_mtp_loss: 13.8304 +[titan] 2025-07-10 10:41:52,954 - root - INFO - lr: 3.1124e-05 gnorm: 1.03 [21:06:48< 0:53:44] +[titan] 2025-07-10 10:41:56,858 - root - INFO - step: 95935 loss: 16.0603 memory: 44.58GiB(31.99%) tps: 83,944 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6084 +[titan] 2025-07-10 10:41:56,858 - root - INFO - lr: 3.1122e-05 gnorm: 1.04 [21:06:52< 0:53:40] +[titan] 2025-07-10 10:42:00,802 - root - INFO - step: 95940 loss: 16.1214 memory: 44.58GiB(31.99%) tps: 83,085 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4684 global_avg_mtp_loss: 13.6531 +[titan] 2025-07-10 10:42:00,802 - root - INFO - lr: 3.1119e-05 gnorm: 1.08 [21:06:56< 0:53:36] +[titan] 2025-07-10 10:42:04,727 - root - INFO - step: 95945 loss: 16.2564 memory: 44.58GiB(31.99%) tps: 83,502 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4789 global_avg_mtp_loss: 13.7775 +[titan] 2025-07-10 10:42:04,727 - root - INFO - lr: 3.1116e-05 gnorm: 1.04 [21:07:00< 0:53:32] +[titan] 2025-07-10 10:42:07,843 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:42:08,633 - root - INFO - step: 95950 loss: 15.8825 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4088 global_avg_mtp_loss: 13.4737 +[titan] 2025-07-10 10:42:08,633 - root - INFO - lr: 3.1113e-05 gnorm: 1.05 [21:07:04< 0:53:28] +[titan] 2025-07-10 10:42:12,573 - root - INFO - step: 95955 loss: 16.3178 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.8217 +[titan] 2025-07-10 10:42:12,574 - root - INFO - lr: 3.1111e-05 gnorm: 1.00 [21:07:08< 0:53:24] +[titan] 2025-07-10 10:42:16,502 - root - INFO - step: 95960 loss: 16.3681 memory: 44.58GiB(31.99%) tps: 83,411 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.8725 +[titan] 2025-07-10 10:42:16,502 - root - INFO - lr: 3.1108e-05 gnorm: 1.08 [21:07:12< 0:53:21] +[titan] 2025-07-10 10:42:20,422 - root - INFO - step: 95965 loss: 16.0850 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4622 global_avg_mtp_loss: 13.6228 +[titan] 2025-07-10 10:42:20,422 - root - INFO - lr: 3.1105e-05 gnorm: 1.08 [21:07:16< 0:53:17] +[titan] 2025-07-10 10:42:24,338 - root - INFO - step: 95970 loss: 16.0373 memory: 44.58GiB(31.99%) tps: 83,675 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4377 global_avg_mtp_loss: 13.5996 +[titan] 2025-07-10 10:42:24,338 - root - INFO - lr: 3.1102e-05 gnorm: 1.03 [21:07:20< 0:53:13] +[titan] 2025-07-10 10:42:28,256 - root - INFO - step: 95975 loss: 15.7469 memory: 44.58GiB(31.99%) tps: 83,653 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4232 global_avg_mtp_loss: 13.3237 +[titan] 2025-07-10 10:42:28,256 - root - INFO - lr: 3.1100e-05 gnorm: 1.08 [21:07:23< 0:53:09] +[titan] 2025-07-10 10:42:32,217 - root - INFO - step: 95980 loss: 16.5896 memory: 44.58GiB(31.99%) tps: 82,730 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.5522 global_avg_mtp_loss: 14.0374 +[titan] 2025-07-10 10:42:32,217 - root - INFO - lr: 3.1097e-05 gnorm: 1.07 [21:07:27< 0:53:05] +[titan] 2025-07-10 10:42:36,158 - root - INFO - step: 95985 loss: 16.3323 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4856 global_avg_mtp_loss: 13.8467 +[titan] 2025-07-10 10:42:36,158 - root - INFO - lr: 3.1094e-05 gnorm: 1.01 [21:07:31< 0:53:01] +[titan] 2025-07-10 10:42:40,072 - root - INFO - step: 95990 loss: 15.9468 memory: 44.58GiB(31.99%) tps: 83,732 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.5107 +[titan] 2025-07-10 10:42:40,072 - root - INFO - lr: 3.1092e-05 gnorm: 1.04 [21:07:35< 0:52:57] +[titan] 2025-07-10 10:42:44,010 - root - INFO - step: 95995 loss: 16.4840 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.5386 global_avg_mtp_loss: 13.9454 +[titan] 2025-07-10 10:42:44,010 - root - INFO - lr: 3.1089e-05 gnorm: 1.15 [21:07:39< 0:52:53] +[titan] 2025-07-10 10:42:47,127 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:42:47,923 - root - INFO - step: 96000 loss: 16.1144 memory: 44.58GiB(31.99%) tps: 83,750 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4481 global_avg_mtp_loss: 13.6663 +[titan] 2025-07-10 10:42:47,923 - root - INFO - lr: 3.1086e-05 gnorm: 1.05 [21:07:43< 0:52:49] +[titan] 2025-07-10 10:42:51,868 - root - INFO - step: 96005 loss: 16.0476 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4464 global_avg_mtp_loss: 13.6012 +[titan] 2025-07-10 10:42:51,868 - root - INFO - lr: 3.1083e-05 gnorm: 1.07 [21:07:47< 0:52:45] +[titan] 2025-07-10 10:42:55,782 - root - INFO - step: 96010 loss: 16.2374 memory: 44.58GiB(31.99%) tps: 83,718 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7545 +[titan] 2025-07-10 10:42:55,783 - root - INFO - lr: 3.1081e-05 gnorm: 1.05 [21:07:51< 0:52:41] +[titan] 2025-07-10 10:42:59,695 - root - INFO - step: 96015 loss: 16.2106 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7366 +[titan] 2025-07-10 10:42:59,695 - root - INFO - lr: 3.1078e-05 gnorm: 1.07 [21:07:55< 0:52:37] +[titan] 2025-07-10 10:43:03,606 - root - INFO - step: 96020 loss: 15.9591 memory: 44.58GiB(31.99%) tps: 83,788 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4441 global_avg_mtp_loss: 13.5150 +[titan] 2025-07-10 10:43:03,607 - root - INFO - lr: 3.1075e-05 gnorm: 1.10 [21:07:59< 0:52:33] +[titan] 2025-07-10 10:43:07,606 - root - INFO - step: 96025 loss: 16.0641 memory: 44.58GiB(31.99%) tps: 81,935 tflops: 282.77 mfu: 28.59% global_avg_ntp_loss: 2.4444 global_avg_mtp_loss: 13.6197 +[titan] 2025-07-10 10:43:07,606 - root - INFO - lr: 3.1073e-05 gnorm: 1.04 [21:08:03< 0:52:29] +[titan] 2025-07-10 10:43:11,546 - root - INFO - step: 96030 loss: 15.8958 memory: 44.58GiB(31.99%) tps: 83,178 tflops: 287.06 mfu: 29.03% global_avg_ntp_loss: 2.4177 global_avg_mtp_loss: 13.4781 +[titan] 2025-07-10 10:43:11,546 - root - INFO - lr: 3.1070e-05 gnorm: 1.11 [21:08:07< 0:52:25] +[titan] 2025-07-10 10:43:15,462 - root - INFO - step: 96035 loss: 16.0631 memory: 44.58GiB(31.99%) tps: 83,680 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.6204 +[titan] 2025-07-10 10:43:15,462 - root - INFO - lr: 3.1067e-05 gnorm: 1.05 [21:08:11< 0:52:21] +[titan] 2025-07-10 10:43:19,361 - root - INFO - step: 96040 loss: 16.1890 memory: 44.58GiB(31.99%) tps: 84,054 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4686 global_avg_mtp_loss: 13.7204 +[titan] 2025-07-10 10:43:19,361 - root - INFO - lr: 3.1064e-05 gnorm: 1.07 [21:08:15< 0:52:17] +[titan] 2025-07-10 10:43:23,265 - root - INFO - step: 96045 loss: 16.0088 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.4366 global_avg_mtp_loss: 13.5722 +[titan] 2025-07-10 10:43:23,265 - root - INFO - lr: 3.1062e-05 gnorm: 1.09 [21:08:18< 0:52:13] +[titan] 2025-07-10 10:43:26,392 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:43:27,179 - root - INFO - step: 96050 loss: 15.6994 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.3983 global_avg_mtp_loss: 13.3011 +[titan] 2025-07-10 10:43:27,180 - root - INFO - lr: 3.1059e-05 gnorm: 1.19 [21:08:22< 0:52:09] +[titan] 2025-07-10 10:43:31,089 - root - INFO - step: 96055 loss: 16.5009 memory: 44.58GiB(31.99%) tps: 83,833 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.5347 global_avg_mtp_loss: 13.9662 +[titan] 2025-07-10 10:43:31,089 - root - INFO - lr: 3.1056e-05 gnorm: 1.13 [21:08:26< 0:52:05] +[titan] 2025-07-10 10:43:34,990 - root - INFO - step: 96060 loss: 16.0308 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5817 +[titan] 2025-07-10 10:43:34,991 - root - INFO - lr: 3.1054e-05 gnorm: 1.09 [21:08:30< 0:52:01] +[titan] 2025-07-10 10:43:38,901 - root - INFO - step: 96065 loss: 16.2415 memory: 44.58GiB(31.99%) tps: 83,809 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7625 +[titan] 2025-07-10 10:43:38,901 - root - INFO - lr: 3.1051e-05 gnorm: 1.06 [21:08:34< 0:51:57] +[titan] 2025-07-10 10:43:42,900 - root - INFO - step: 96070 loss: 16.1278 memory: 44.58GiB(31.99%) tps: 81,935 tflops: 282.77 mfu: 28.59% global_avg_ntp_loss: 2.4518 global_avg_mtp_loss: 13.6761 +[titan] 2025-07-10 10:43:42,901 - root - INFO - lr: 3.1048e-05 gnorm: 1.09 [21:08:38< 0:51:53] +[titan] 2025-07-10 10:43:46,834 - root - INFO - step: 96075 loss: 16.2807 memory: 44.58GiB(31.99%) tps: 83,317 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4803 global_avg_mtp_loss: 13.8004 +[titan] 2025-07-10 10:43:46,834 - root - INFO - lr: 3.1046e-05 gnorm: 1.02 [21:08:42< 0:51:49] +[titan] 2025-07-10 10:43:50,776 - root - INFO - step: 96080 loss: 16.3008 memory: 44.58GiB(31.99%) tps: 83,132 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4998 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-10 10:43:50,776 - root - INFO - lr: 3.1043e-05 gnorm: 1.04 [21:08:46< 0:51:45] +[titan] 2025-07-10 10:43:54,701 - root - INFO - step: 96085 loss: 16.5208 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5270 global_avg_mtp_loss: 13.9938 +[titan] 2025-07-10 10:43:54,701 - root - INFO - lr: 3.1040e-05 gnorm: 1.08 [21:08:50< 0:51:41] +[titan] 2025-07-10 10:43:58,622 - root - INFO - step: 96090 loss: 15.9221 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4104 global_avg_mtp_loss: 13.5117 +[titan] 2025-07-10 10:43:58,622 - root - INFO - lr: 3.1038e-05 gnorm: 1.02 [21:08:54< 0:51:37] +[titan] 2025-07-10 10:44:02,538 - root - INFO - step: 96095 loss: 16.3391 memory: 44.58GiB(31.99%) tps: 83,673 tflops: 288.77 mfu: 29.20% global_avg_ntp_loss: 2.4878 global_avg_mtp_loss: 13.8513 +[titan] 2025-07-10 10:44:02,539 - root - INFO - lr: 3.1035e-05 gnorm: 1.04 [21:08:58< 0:51:34] +[titan] 2025-07-10 10:44:05,661 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:44:06,456 - root - INFO - step: 96100 loss: 15.9458 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.5096 +[titan] 2025-07-10 10:44:06,457 - root - INFO - lr: 3.1033e-05 gnorm: 1.05 [21:09:02< 0:51:30] +[titan] 2025-07-10 10:44:10,373 - root - INFO - step: 96105 loss: 16.2113 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.74 mfu: 29.19% global_avg_ntp_loss: 2.4586 global_avg_mtp_loss: 13.7527 +[titan] 2025-07-10 10:44:10,374 - root - INFO - lr: 3.1030e-05 gnorm: 1.06 [21:09:06< 0:51:26] +[titan] 2025-07-10 10:44:14,303 - root - INFO - step: 96110 loss: 15.5962 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.3758 global_avg_mtp_loss: 13.2203 +[titan] 2025-07-10 10:44:14,304 - root - INFO - lr: 3.1027e-05 gnorm: 1.06 [21:09:09< 0:51:22] +[titan] 2025-07-10 10:44:18,230 - root - INFO - step: 96115 loss: 16.1007 memory: 44.58GiB(31.99%) tps: 83,456 tflops: 288.02 mfu: 29.12% global_avg_ntp_loss: 2.4493 global_avg_mtp_loss: 13.6514 +[titan] 2025-07-10 10:44:18,230 - root - INFO - lr: 3.1025e-05 gnorm: 1.10 [21:09:13< 0:51:18] +[titan] 2025-07-10 10:44:22,135 - root - INFO - step: 96120 loss: 16.0946 memory: 44.58GiB(31.99%) tps: 83,933 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4496 global_avg_mtp_loss: 13.6450 +[titan] 2025-07-10 10:44:22,135 - root - INFO - lr: 3.1022e-05 gnorm: 1.16 [21:09:17< 0:51:14] +[titan] 2025-07-10 10:44:26,019 - root - INFO - step: 96125 loss: 15.9524 memory: 44.58GiB(31.99%) tps: 84,359 tflops: 291.14 mfu: 29.44% global_avg_ntp_loss: 2.4301 global_avg_mtp_loss: 13.5224 +[titan] 2025-07-10 10:44:26,020 - root - INFO - lr: 3.1019e-05 gnorm: 1.03 [21:09:21< 0:51:10] +[titan] 2025-07-10 10:44:29,941 - root - INFO - step: 96130 loss: 16.4302 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.5144 global_avg_mtp_loss: 13.9157 +[titan] 2025-07-10 10:44:29,941 - root - INFO - lr: 3.1017e-05 gnorm: 1.06 [21:09:25< 0:51:06] +[titan] 2025-07-10 10:44:33,849 - root - INFO - step: 96135 loss: 16.0512 memory: 44.58GiB(31.99%) tps: 83,851 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.5999 +[titan] 2025-07-10 10:44:33,849 - root - INFO - lr: 3.1014e-05 gnorm: 1.04 [21:09:29< 0:51:02] +[titan] 2025-07-10 10:44:37,784 - root - INFO - step: 96140 loss: 16.3568 memory: 44.58GiB(31.99%) tps: 83,270 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.5014 global_avg_mtp_loss: 13.8554 +[titan] 2025-07-10 10:44:37,785 - root - INFO - lr: 3.1011e-05 gnorm: 1.03 [21:09:33< 0:50:58] +[titan] 2025-07-10 10:44:41,691 - root - INFO - step: 96145 loss: 16.0670 memory: 44.58GiB(31.99%) tps: 83,889 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.4377 global_avg_mtp_loss: 13.6294 +[titan] 2025-07-10 10:44:41,691 - root - INFO - lr: 3.1009e-05 gnorm: 1.05 [21:09:37< 0:50:54] +[titan] 2025-07-10 10:44:44,830 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:44:45,622 - root - INFO - step: 96150 loss: 16.1323 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4600 global_avg_mtp_loss: 13.6723 +[titan] 2025-07-10 10:44:45,622 - root - INFO - lr: 3.1006e-05 gnorm: 1.01 [21:09:41< 0:50:50] +[titan] 2025-07-10 10:44:49,564 - root - INFO - step: 96155 loss: 16.1980 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.7360 +[titan] 2025-07-10 10:44:49,565 - root - INFO - lr: 3.1004e-05 gnorm: 1.06 [21:09:45< 0:50:46] +[titan] 2025-07-10 10:44:53,497 - root - INFO - step: 96160 loss: 16.1954 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.7323 +[titan] 2025-07-10 10:44:53,497 - root - INFO - lr: 3.1001e-05 gnorm: 1.06 [21:09:49< 0:50:42] +[titan] 2025-07-10 10:44:57,459 - root - INFO - step: 96165 loss: 15.8396 memory: 44.58GiB(31.99%) tps: 82,706 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4180 global_avg_mtp_loss: 13.4216 +[titan] 2025-07-10 10:44:57,459 - root - INFO - lr: 3.0998e-05 gnorm: 1.05 [21:09:53< 0:50:38] +[titan] 2025-07-10 10:45:01,377 - root - INFO - step: 96170 loss: 16.2213 memory: 44.58GiB(31.99%) tps: 83,650 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4701 global_avg_mtp_loss: 13.7512 +[titan] 2025-07-10 10:45:01,377 - root - INFO - lr: 3.0996e-05 gnorm: 1.08 [21:09:57< 0:50:34] +[titan] 2025-07-10 10:45:05,281 - root - INFO - step: 96175 loss: 16.2653 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7777 +[titan] 2025-07-10 10:45:05,281 - root - INFO - lr: 3.0993e-05 gnorm: 1.05 [21:10:00< 0:50:30] +[titan] 2025-07-10 10:45:09,213 - root - INFO - step: 96180 loss: 16.1031 memory: 44.58GiB(31.99%) tps: 83,346 tflops: 287.64 mfu: 29.08% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.6470 +[titan] 2025-07-10 10:45:09,213 - root - INFO - lr: 3.0991e-05 gnorm: 1.05 [21:10:04< 0:50:26] +[titan] 2025-07-10 10:45:13,141 - root - INFO - step: 96185 loss: 16.4893 memory: 44.58GiB(31.99%) tps: 83,416 tflops: 287.88 mfu: 29.11% global_avg_ntp_loss: 2.5176 global_avg_mtp_loss: 13.9717 +[titan] 2025-07-10 10:45:13,142 - root - INFO - lr: 3.0988e-05 gnorm: 1.03 [21:10:08< 0:50:22] +[titan] 2025-07-10 10:45:17,082 - root - INFO - step: 96190 loss: 16.2525 memory: 44.58GiB(31.99%) tps: 83,157 tflops: 286.99 mfu: 29.02% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7811 +[titan] 2025-07-10 10:45:17,083 - root - INFO - lr: 3.0985e-05 gnorm: 1.06 [21:10:12< 0:50:18] +[titan] 2025-07-10 10:45:21,110 - root - INFO - step: 96195 loss: 16.3993 memory: 44.58GiB(31.99%) tps: 81,369 tflops: 280.82 mfu: 28.39% global_avg_ntp_loss: 2.5031 global_avg_mtp_loss: 13.8961 +[titan] 2025-07-10 10:45:21,110 - root - INFO - lr: 3.0983e-05 gnorm: 1.05 [21:10:16< 0:50:14] +[titan] 2025-07-10 10:45:24,231 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:45:25,019 - root - INFO - step: 96200 loss: 16.0857 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.6421 +[titan] 2025-07-10 10:45:25,020 - root - INFO - lr: 3.0980e-05 gnorm: 1.04 [21:10:20< 0:50:10] +[titan] 2025-07-10 10:45:28,921 - root - INFO - step: 96205 loss: 15.8868 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.4127 global_avg_mtp_loss: 13.4741 +[titan] 2025-07-10 10:45:28,922 - root - INFO - lr: 3.0978e-05 gnorm: 1.04 [21:10:24< 0:50:06] +[titan] 2025-07-10 10:45:32,826 - root - INFO - step: 96210 loss: 16.2340 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7487 +[titan] 2025-07-10 10:45:32,826 - root - INFO - lr: 3.0975e-05 gnorm: 1.06 [21:10:28< 0:50:02] +[titan] 2025-07-10 10:45:36,729 - root - INFO - step: 96215 loss: 15.9774 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4366 global_avg_mtp_loss: 13.5408 +[titan] 2025-07-10 10:45:36,729 - root - INFO - lr: 3.0973e-05 gnorm: 1.08 [21:10:32< 0:49:58] +[titan] 2025-07-10 10:45:40,650 - root - INFO - step: 96220 loss: 16.1639 memory: 44.58GiB(31.99%) tps: 83,574 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4558 global_avg_mtp_loss: 13.7082 +[titan] 2025-07-10 10:45:40,650 - root - INFO - lr: 3.0970e-05 gnorm: 1.06 [21:10:36< 0:49:54] +[titan] 2025-07-10 10:45:44,568 - root - INFO - step: 96225 loss: 16.2322 memory: 44.58GiB(31.99%) tps: 83,654 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4635 global_avg_mtp_loss: 13.7686 +[titan] 2025-07-10 10:45:44,568 - root - INFO - lr: 3.0967e-05 gnorm: 1.07 [21:10:40< 0:49:50] +[titan] 2025-07-10 10:45:48,466 - root - INFO - step: 96230 loss: 16.3405 memory: 44.58GiB(31.99%) tps: 84,062 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4975 global_avg_mtp_loss: 13.8430 +[titan] 2025-07-10 10:45:48,466 - root - INFO - lr: 3.0965e-05 gnorm: 1.05 [21:10:44< 0:49:47] +[titan] 2025-07-10 10:45:52,378 - root - INFO - step: 96235 loss: 16.4574 memory: 44.58GiB(31.99%) tps: 83,764 tflops: 289.08 mfu: 29.23% global_avg_ntp_loss: 2.5089 global_avg_mtp_loss: 13.9485 +[titan] 2025-07-10 10:45:52,379 - root - INFO - lr: 3.0962e-05 gnorm: 1.12 [21:10:48< 0:49:43] +[titan] 2025-07-10 10:45:56,304 - root - INFO - step: 96240 loss: 16.2473 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7715 +[titan] 2025-07-10 10:45:56,304 - root - INFO - lr: 3.0960e-05 gnorm: 1.04 [21:10:51< 0:49:39] +[titan] 2025-07-10 10:46:00,218 - root - INFO - step: 96245 loss: 16.1519 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4699 global_avg_mtp_loss: 13.6820 +[titan] 2025-07-10 10:46:00,218 - root - INFO - lr: 3.0957e-05 gnorm: 1.04 [21:10:55< 0:49:35] +[titan] 2025-07-10 10:46:03,358 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:46:04,147 - root - INFO - step: 96250 loss: 16.1764 memory: 44.58GiB(31.99%) tps: 83,393 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.7155 +[titan] 2025-07-10 10:46:04,148 - root - INFO - lr: 3.0955e-05 gnorm: 1.05 [21:10:59< 0:49:31] +[titan] 2025-07-10 10:46:08,135 - root - INFO - step: 96255 loss: 16.3922 memory: 44.58GiB(31.99%) tps: 82,183 tflops: 283.63 mfu: 28.68% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8893 +[titan] 2025-07-10 10:46:08,135 - root - INFO - lr: 3.0952e-05 gnorm: 1.07 [21:11:03< 0:49:27] +[titan] 2025-07-10 10:46:09,072 - root - INFO - Dumping profiler traces at step 96256 +[titan] 2025-07-10 10:46:09,104 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:46:12,229 - root - INFO - step: 96260 loss: 16.1400 memory: 44.58GiB(31.99%) tps: 80,045 tflops: 276.25 mfu: 27.93% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 10:46:12,229 - root - INFO - lr: 3.0950e-05 gnorm: 1.03 [21:11:07< 0:49:23] +[titan] 2025-07-10 10:46:16,163 - root - INFO - step: 96265 loss: 16.2479 memory: 44.58GiB(31.99%) tps: 83,304 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.7745 +[titan] 2025-07-10 10:46:16,163 - root - INFO - lr: 3.0947e-05 gnorm: 1.04 [21:11:11< 0:49:19] +[titan] 2025-07-10 10:46:20,067 - root - INFO - step: 96270 loss: 16.1105 memory: 44.58GiB(31.99%) tps: 83,948 tflops: 289.72 mfu: 29.29% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6504 +[titan] 2025-07-10 10:46:20,067 - root - INFO - lr: 3.0945e-05 gnorm: 1.04 [21:11:15< 0:49:15] +[titan] 2025-07-10 10:46:23,967 - root - INFO - step: 96275 loss: 16.2571 memory: 44.58GiB(31.99%) tps: 84,022 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.4813 global_avg_mtp_loss: 13.7758 +[titan] 2025-07-10 10:46:23,967 - root - INFO - lr: 3.0942e-05 gnorm: 1.02 [21:11:19< 0:49:11] +[titan] 2025-07-10 10:46:27,923 - root - INFO - step: 96280 loss: 16.2562 memory: 44.58GiB(31.99%) tps: 82,842 tflops: 285.90 mfu: 28.91% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.7710 +[titan] 2025-07-10 10:46:27,923 - root - INFO - lr: 3.0940e-05 gnorm: 0.99 [21:11:23< 0:49:07] +[titan] 2025-07-10 10:46:31,874 - root - INFO - step: 96285 loss: 15.9757 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4267 global_avg_mtp_loss: 13.5490 +[titan] 2025-07-10 10:46:31,874 - root - INFO - lr: 3.0937e-05 gnorm: 1.07 [21:11:27< 0:49:03] +[titan] 2025-07-10 10:46:35,803 - root - INFO - step: 96290 loss: 16.2569 memory: 44.58GiB(31.99%) tps: 83,392 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4890 global_avg_mtp_loss: 13.7679 +[titan] 2025-07-10 10:46:35,804 - root - INFO - lr: 3.0934e-05 gnorm: 1.03 [21:11:31< 0:48:59] +[titan] 2025-07-10 10:46:39,741 - root - INFO - step: 96295 loss: 16.1919 memory: 44.58GiB(31.99%) tps: 83,216 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4599 global_avg_mtp_loss: 13.7320 +[titan] 2025-07-10 10:46:39,742 - root - INFO - lr: 3.0932e-05 gnorm: 1.03 [21:11:35< 0:48:55] +[titan] 2025-07-10 10:46:42,917 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:46:43,706 - root - INFO - step: 96300 loss: 16.2066 memory: 44.58GiB(31.99%) tps: 82,658 tflops: 285.27 mfu: 28.84% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7241 +[titan] 2025-07-10 10:46:43,706 - root - INFO - lr: 3.0929e-05 gnorm: 1.04 [21:11:39< 0:48:51] +[titan] 2025-07-10 10:46:47,644 - root - INFO - step: 96305 loss: 16.1775 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4887 global_avg_mtp_loss: 13.6889 +[titan] 2025-07-10 10:46:47,644 - root - INFO - lr: 3.0927e-05 gnorm: 1.07 [21:11:43< 0:48:47] +[titan] 2025-07-10 10:46:51,562 - root - INFO - step: 96310 loss: 16.2801 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.7817 +[titan] 2025-07-10 10:46:51,562 - root - INFO - lr: 3.0924e-05 gnorm: 1.05 [21:11:47< 0:48:43] +[titan] 2025-07-10 10:46:55,522 - root - INFO - step: 96315 loss: 16.0236 memory: 44.58GiB(31.99%) tps: 82,752 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.5682 +[titan] 2025-07-10 10:46:55,522 - root - INFO - lr: 3.0922e-05 gnorm: 1.06 [21:11:51< 0:48:39] +[titan] 2025-07-10 10:46:59,444 - root - INFO - step: 96320 loss: 16.2445 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7515 +[titan] 2025-07-10 10:46:59,444 - root - INFO - lr: 3.0919e-05 gnorm: 1.05 [21:11:55< 0:48:35] +[titan] 2025-07-10 10:47:03,353 - root - INFO - step: 96325 loss: 16.2483 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7666 +[titan] 2025-07-10 10:47:03,353 - root - INFO - lr: 3.0917e-05 gnorm: 1.04 [21:11:59< 0:48:31] +[titan] 2025-07-10 10:47:07,272 - root - INFO - step: 96330 loss: 16.2584 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4936 global_avg_mtp_loss: 13.7648 +[titan] 2025-07-10 10:47:07,273 - root - INFO - lr: 3.0914e-05 gnorm: 1.56 [21:12:02< 0:48:27] +[titan] 2025-07-10 10:47:11,250 - root - INFO - step: 96335 loss: 16.0363 memory: 44.58GiB(31.99%) tps: 82,383 tflops: 284.32 mfu: 28.75% global_avg_ntp_loss: 2.4240 global_avg_mtp_loss: 13.6123 +[titan] 2025-07-10 10:47:11,250 - root - INFO - lr: 3.0912e-05 gnorm: 1.07 [21:12:06< 0:48:23] +[titan] 2025-07-10 10:47:15,198 - root - INFO - step: 96340 loss: 16.1824 memory: 44.58GiB(31.99%) tps: 83,018 tflops: 286.51 mfu: 28.97% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.7154 +[titan] 2025-07-10 10:47:15,198 - root - INFO - lr: 3.0909e-05 gnorm: 1.05 [21:12:10< 0:48:19] +[titan] 2025-07-10 10:47:19,115 - root - INFO - step: 96345 loss: 16.3021 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.5032 global_avg_mtp_loss: 13.7990 +[titan] 2025-07-10 10:47:19,115 - root - INFO - lr: 3.0907e-05 gnorm: 1.04 [21:12:14< 0:48:15] +[titan] 2025-07-10 10:47:22,237 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:47:23,025 - root - INFO - step: 96350 loss: 16.2534 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7682 +[titan] 2025-07-10 10:47:23,025 - root - INFO - lr: 3.0905e-05 gnorm: 1.04 [21:12:18< 0:48:11] +[titan] 2025-07-10 10:47:26,943 - root - INFO - step: 96355 loss: 16.2584 memory: 44.58GiB(31.99%) tps: 83,643 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7784 +[titan] 2025-07-10 10:47:26,943 - root - INFO - lr: 3.0902e-05 gnorm: 1.09 [21:12:22< 0:48:07] +[titan] 2025-07-10 10:47:30,874 - root - INFO - step: 96360 loss: 16.0191 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4352 global_avg_mtp_loss: 13.5838 +[titan] 2025-07-10 10:47:30,874 - root - INFO - lr: 3.0900e-05 gnorm: 1.12 [21:12:26< 0:48:03] +[titan] 2025-07-10 10:47:34,803 - root - INFO - step: 96365 loss: 15.9916 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4393 global_avg_mtp_loss: 13.5523 +[titan] 2025-07-10 10:47:34,803 - root - INFO - lr: 3.0897e-05 gnorm: 1.07 [21:12:30< 0:48:00] +[titan] 2025-07-10 10:47:38,710 - root - INFO - step: 96370 loss: 16.2286 memory: 44.58GiB(31.99%) tps: 83,877 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7499 +[titan] 2025-07-10 10:47:38,710 - root - INFO - lr: 3.0895e-05 gnorm: 1.06 [21:12:34< 0:47:56] +[titan] 2025-07-10 10:47:42,631 - root - INFO - step: 96375 loss: 16.0890 memory: 44.58GiB(31.99%) tps: 83,581 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4462 global_avg_mtp_loss: 13.6427 +[titan] 2025-07-10 10:47:42,631 - root - INFO - lr: 3.0892e-05 gnorm: 1.05 [21:12:38< 0:47:52] +[titan] 2025-07-10 10:47:46,541 - root - INFO - step: 96380 loss: 16.2073 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7385 +[titan] 2025-07-10 10:47:46,541 - root - INFO - lr: 3.0890e-05 gnorm: 1.05 [21:12:42< 0:47:48] +[titan] 2025-07-10 10:47:50,451 - root - INFO - step: 96385 loss: 15.9445 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4315 global_avg_mtp_loss: 13.5130 +[titan] 2025-07-10 10:47:50,451 - root - INFO - lr: 3.0887e-05 gnorm: 1.05 [21:12:46< 0:47:44] +[titan] 2025-07-10 10:47:54,366 - root - INFO - step: 96390 loss: 16.1798 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7017 +[titan] 2025-07-10 10:47:54,366 - root - INFO - lr: 3.0885e-05 gnorm: 1.02 [21:12:50< 0:47:40] +[titan] 2025-07-10 10:47:58,309 - root - INFO - step: 96395 loss: 16.2117 memory: 44.58GiB(31.99%) tps: 83,105 tflops: 286.81 mfu: 29.00% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7245 +[titan] 2025-07-10 10:47:58,310 - root - INFO - lr: 3.0882e-05 gnorm: 1.11 [21:12:53< 0:47:36] +[titan] 2025-07-10 10:48:01,446 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:48:02,243 - root - INFO - step: 96400 loss: 16.1395 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6744 +[titan] 2025-07-10 10:48:02,243 - root - INFO - lr: 3.0880e-05 gnorm: 1.02 [21:12:57< 0:47:32] +[titan] 2025-07-10 10:48:06,160 - root - INFO - step: 96405 loss: 16.1897 memory: 44.58GiB(31.99%) tps: 83,662 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4619 global_avg_mtp_loss: 13.7279 +[titan] 2025-07-10 10:48:06,160 - root - INFO - lr: 3.0878e-05 gnorm: 1.01 [21:13:01< 0:47:28] +[titan] 2025-07-10 10:48:10,097 - root - INFO - step: 96410 loss: 16.3034 memory: 44.58GiB(31.99%) tps: 83,223 tflops: 287.22 mfu: 29.04% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.8198 +[titan] 2025-07-10 10:48:10,098 - root - INFO - lr: 3.0875e-05 gnorm: 1.03 [21:13:05< 0:47:24] +[titan] 2025-07-10 10:48:14,011 - root - INFO - step: 96415 loss: 16.2679 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4861 global_avg_mtp_loss: 13.7818 +[titan] 2025-07-10 10:48:14,012 - root - INFO - lr: 3.0873e-05 gnorm: 1.01 [21:13:09< 0:47:20] +[titan] 2025-07-10 10:48:17,918 - root - INFO - step: 96420 loss: 16.2035 memory: 44.58GiB(31.99%) tps: 83,890 tflops: 289.52 mfu: 29.27% global_avg_ntp_loss: 2.4743 global_avg_mtp_loss: 13.7292 +[titan] 2025-07-10 10:48:17,918 - root - INFO - lr: 3.0870e-05 gnorm: 1.13 [21:13:13< 0:47:16] +[titan] 2025-07-10 10:48:21,848 - root - INFO - step: 96425 loss: 16.1022 memory: 44.58GiB(31.99%) tps: 83,390 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4663 global_avg_mtp_loss: 13.6359 +[titan] 2025-07-10 10:48:21,848 - root - INFO - lr: 3.0868e-05 gnorm: 1.04 [21:13:17< 0:47:12] +[titan] 2025-07-10 10:48:25,768 - root - INFO - step: 96430 loss: 16.1876 memory: 44.58GiB(31.99%) tps: 83,596 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4762 global_avg_mtp_loss: 13.7113 +[titan] 2025-07-10 10:48:25,768 - root - INFO - lr: 3.0865e-05 gnorm: 1.00 [21:13:21< 0:47:08] +[titan] 2025-07-10 10:48:29,712 - root - INFO - step: 96435 loss: 16.1415 memory: 44.58GiB(31.99%) tps: 83,084 tflops: 286.74 mfu: 28.99% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6863 +[titan] 2025-07-10 10:48:29,713 - root - INFO - lr: 3.0863e-05 gnorm: 1.04 [21:13:25< 0:47:04] +[titan] 2025-07-10 10:48:33,628 - root - INFO - step: 96440 loss: 15.9413 memory: 44.58GiB(31.99%) tps: 83,693 tflops: 288.84 mfu: 29.20% global_avg_ntp_loss: 2.4160 global_avg_mtp_loss: 13.5254 +[titan] 2025-07-10 10:48:33,628 - root - INFO - lr: 3.0861e-05 gnorm: 1.02 [21:13:29< 0:47:00] +[titan] 2025-07-10 10:48:37,544 - root - INFO - step: 96445 loss: 16.2570 memory: 44.58GiB(31.99%) tps: 83,686 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4864 global_avg_mtp_loss: 13.7705 +[titan] 2025-07-10 10:48:37,544 - root - INFO - lr: 3.0858e-05 gnorm: 1.09 [21:13:33< 0:46:56] +[titan] 2025-07-10 10:48:40,667 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:48:41,459 - root - INFO - step: 96450 loss: 16.2771 memory: 44.58GiB(31.99%) tps: 83,717 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4752 global_avg_mtp_loss: 13.8019 +[titan] 2025-07-10 10:48:41,459 - root - INFO - lr: 3.0856e-05 gnorm: 1.10 [21:13:37< 0:46:52] +[titan] 2025-07-10 10:48:45,398 - root - INFO - step: 96455 loss: 16.0324 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4453 global_avg_mtp_loss: 13.5871 +[titan] 2025-07-10 10:48:45,399 - root - INFO - lr: 3.0853e-05 gnorm: 1.06 [21:13:41< 0:46:48] +[titan] 2025-07-10 10:48:49,341 - root - INFO - step: 96460 loss: 16.3744 memory: 44.58GiB(31.99%) tps: 83,128 tflops: 286.89 mfu: 29.01% global_avg_ntp_loss: 2.4962 global_avg_mtp_loss: 13.8782 +[titan] 2025-07-10 10:48:49,341 - root - INFO - lr: 3.0851e-05 gnorm: 1.05 [21:13:45< 0:46:44] +[titan] 2025-07-10 10:48:53,278 - root - INFO - step: 96465 loss: 16.1762 memory: 44.58GiB(31.99%) tps: 83,236 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.7093 +[titan] 2025-07-10 10:48:53,278 - root - INFO - lr: 3.0848e-05 gnorm: 1.07 [21:13:48< 0:46:40] +[titan] 2025-07-10 10:48:57,188 - root - INFO - step: 96470 loss: 16.3745 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5199 global_avg_mtp_loss: 13.8547 +[titan] 2025-07-10 10:48:57,188 - root - INFO - lr: 3.0846e-05 gnorm: 1.05 [21:13:52< 0:46:36] +[titan] 2025-07-10 10:49:01,149 - root - INFO - step: 96475 loss: 16.1352 memory: 44.58GiB(31.99%) tps: 82,738 tflops: 285.54 mfu: 28.87% global_avg_ntp_loss: 2.4513 global_avg_mtp_loss: 13.6840 +[titan] 2025-07-10 10:49:01,149 - root - INFO - lr: 3.0844e-05 gnorm: 1.09 [21:13:56< 0:46:32] +[titan] 2025-07-10 10:49:05,115 - root - INFO - step: 96480 loss: 16.0149 memory: 44.58GiB(31.99%) tps: 82,616 tflops: 285.12 mfu: 28.83% global_avg_ntp_loss: 2.4351 global_avg_mtp_loss: 13.5797 +[titan] 2025-07-10 10:49:05,116 - root - INFO - lr: 3.0841e-05 gnorm: 1.04 [21:14:00< 0:46:28] +[titan] 2025-07-10 10:49:09,040 - root - INFO - step: 96485 loss: 16.1839 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4664 global_avg_mtp_loss: 13.7176 +[titan] 2025-07-10 10:49:09,041 - root - INFO - lr: 3.0839e-05 gnorm: 1.05 [21:14:04< 0:46:24] +[titan] 2025-07-10 10:49:12,948 - root - INFO - step: 96490 loss: 16.2758 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7983 +[titan] 2025-07-10 10:49:12,949 - root - INFO - lr: 3.0837e-05 gnorm: 1.04 [21:14:08< 0:46:20] +[titan] 2025-07-10 10:49:16,898 - root - INFO - step: 96495 loss: 16.2207 memory: 44.58GiB(31.99%) tps: 82,970 tflops: 286.34 mfu: 28.95% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7520 +[titan] 2025-07-10 10:49:16,898 - root - INFO - lr: 3.0834e-05 gnorm: 1.07 [21:14:12< 0:46:16] +[titan] 2025-07-10 10:49:20,037 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:49:20,829 - root - INFO - step: 96500 loss: 16.1208 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6638 +[titan] 2025-07-10 10:49:20,829 - root - INFO - lr: 3.0832e-05 gnorm: 1.07 [21:14:16< 0:46:13] +[titan] 2025-07-10 10:49:24,786 - root - INFO - step: 96505 loss: 16.1784 memory: 44.58GiB(31.99%) tps: 82,815 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.4702 global_avg_mtp_loss: 13.7082 +[titan] 2025-07-10 10:49:24,787 - root - INFO - lr: 3.0829e-05 gnorm: 1.03 [21:14:20< 0:46:09] +[titan] 2025-07-10 10:49:28,689 - root - INFO - step: 96510 loss: 16.1544 memory: 44.58GiB(31.99%) tps: 83,961 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4716 global_avg_mtp_loss: 13.6828 +[titan] 2025-07-10 10:49:28,690 - root - INFO - lr: 3.0827e-05 gnorm: 1.03 [21:14:24< 0:46:05] +[titan] 2025-07-10 10:49:32,591 - root - INFO - step: 96515 loss: 16.4185 memory: 44.58GiB(31.99%) tps: 84,003 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.5122 global_avg_mtp_loss: 13.9063 +[titan] 2025-07-10 10:49:32,591 - root - INFO - lr: 3.0825e-05 gnorm: 1.07 [21:14:28< 0:46:01] +[titan] 2025-07-10 10:49:36,507 - root - INFO - step: 96520 loss: 16.3404 memory: 44.58GiB(31.99%) tps: 83,685 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8412 +[titan] 2025-07-10 10:49:36,507 - root - INFO - lr: 3.0822e-05 gnorm: 1.06 [21:14:32< 0:45:57] +[titan] 2025-07-10 10:49:40,432 - root - INFO - step: 96525 loss: 16.2649 memory: 44.58GiB(31.99%) tps: 83,493 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4919 global_avg_mtp_loss: 13.7730 +[titan] 2025-07-10 10:49:40,432 - root - INFO - lr: 3.0820e-05 gnorm: 1.00 [21:14:36< 0:45:53] +[titan] 2025-07-10 10:49:44,363 - root - INFO - step: 96530 loss: 16.6305 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.5432 global_avg_mtp_loss: 14.0873 +[titan] 2025-07-10 10:49:44,364 - root - INFO - lr: 3.0818e-05 gnorm: 1.03 [21:14:40< 0:45:49] +[titan] 2025-07-10 10:49:48,257 - root - INFO - step: 96535 loss: 16.2448 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7669 +[titan] 2025-07-10 10:49:48,257 - root - INFO - lr: 3.0815e-05 gnorm: 1.12 [21:14:43< 0:45:45] +[titan] 2025-07-10 10:49:52,152 - root - INFO - step: 96540 loss: 16.0574 memory: 44.58GiB(31.99%) tps: 84,138 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.5964 +[titan] 2025-07-10 10:49:52,152 - root - INFO - lr: 3.0813e-05 gnorm: 1.08 [21:14:47< 0:45:41] +[titan] 2025-07-10 10:49:56,080 - root - INFO - step: 96545 loss: 16.3846 memory: 44.58GiB(31.99%) tps: 83,431 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5011 global_avg_mtp_loss: 13.8835 +[titan] 2025-07-10 10:49:56,080 - root - INFO - lr: 3.0811e-05 gnorm: 1.02 [21:14:51< 0:45:37] +[titan] 2025-07-10 10:49:59,216 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:50:00,006 - root - INFO - step: 96550 loss: 16.0219 memory: 44.58GiB(31.99%) tps: 83,459 tflops: 288.03 mfu: 29.12% global_avg_ntp_loss: 2.4381 global_avg_mtp_loss: 13.5838 +[titan] 2025-07-10 10:50:00,007 - root - INFO - lr: 3.0808e-05 gnorm: 1.02 [21:14:55< 0:45:33] +[titan] 2025-07-10 10:50:03,924 - root - INFO - step: 96555 loss: 16.1978 memory: 44.58GiB(31.99%) tps: 83,659 tflops: 288.72 mfu: 29.19% global_avg_ntp_loss: 2.4634 global_avg_mtp_loss: 13.7344 +[titan] 2025-07-10 10:50:03,924 - root - INFO - lr: 3.0806e-05 gnorm: 1.07 [21:14:59< 0:45:29] +[titan] 2025-07-10 10:50:07,838 - root - INFO - step: 96560 loss: 16.1661 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.7042 +[titan] 2025-07-10 10:50:07,838 - root - INFO - lr: 3.0804e-05 gnorm: 1.04 [21:15:03< 0:45:25] +[titan] 2025-07-10 10:50:11,779 - root - INFO - step: 96565 loss: 15.6571 memory: 44.58GiB(31.99%) tps: 83,144 tflops: 286.94 mfu: 29.01% global_avg_ntp_loss: 2.3746 global_avg_mtp_loss: 13.2825 +[titan] 2025-07-10 10:50:11,780 - root - INFO - lr: 3.0801e-05 gnorm: 1.09 [21:15:07< 0:45:21] +[titan] 2025-07-10 10:50:15,729 - root - INFO - step: 96570 loss: 16.2315 memory: 44.58GiB(31.99%) tps: 82,965 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.4795 global_avg_mtp_loss: 13.7521 +[titan] 2025-07-10 10:50:15,730 - root - INFO - lr: 3.0799e-05 gnorm: 1.19 [21:15:11< 0:45:17] +[titan] 2025-07-10 10:50:19,660 - root - INFO - step: 96575 loss: 16.1382 memory: 44.58GiB(31.99%) tps: 83,366 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.6614 +[titan] 2025-07-10 10:50:19,661 - root - INFO - lr: 3.0797e-05 gnorm: 1.11 [21:15:15< 0:45:13] +[titan] 2025-07-10 10:50:23,572 - root - INFO - step: 96580 loss: 16.1198 memory: 44.58GiB(31.99%) tps: 83,784 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4483 global_avg_mtp_loss: 13.6715 +[titan] 2025-07-10 10:50:23,572 - root - INFO - lr: 3.0794e-05 gnorm: 1.06 [21:15:19< 0:45:09] +[titan] 2025-07-10 10:50:27,534 - root - INFO - step: 96585 loss: 16.4979 memory: 44.58GiB(31.99%) tps: 82,709 tflops: 285.44 mfu: 28.86% global_avg_ntp_loss: 2.5423 global_avg_mtp_loss: 13.9555 +[titan] 2025-07-10 10:50:27,534 - root - INFO - lr: 3.0792e-05 gnorm: 1.08 [21:15:23< 0:45:05] +[titan] 2025-07-10 10:50:31,472 - root - INFO - step: 96590 loss: 16.1144 memory: 44.58GiB(31.99%) tps: 83,215 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6606 +[titan] 2025-07-10 10:50:31,472 - root - INFO - lr: 3.0790e-05 gnorm: 1.07 [21:15:27< 0:45:01] +[titan] 2025-07-10 10:50:35,451 - root - INFO - step: 96595 loss: 16.2135 memory: 44.58GiB(31.99%) tps: 82,367 tflops: 284.26 mfu: 28.74% global_avg_ntp_loss: 2.4733 global_avg_mtp_loss: 13.7402 +[titan] 2025-07-10 10:50:35,451 - root - INFO - lr: 3.0787e-05 gnorm: 1.00 [21:15:31< 0:44:57] +[titan] 2025-07-10 10:50:38,577 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:50:39,363 - root - INFO - step: 96600 loss: 16.2713 memory: 44.58GiB(31.99%) tps: 83,776 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.7851 +[titan] 2025-07-10 10:50:39,363 - root - INFO - lr: 3.0785e-05 gnorm: 1.05 [21:15:35< 0:44:53] +[titan] 2025-07-10 10:50:43,280 - root - INFO - step: 96605 loss: 16.3060 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 10:50:43,280 - root - INFO - lr: 3.0783e-05 gnorm: 1.06 [21:15:38< 0:44:49] +[titan] 2025-07-10 10:50:47,201 - root - INFO - step: 96610 loss: 15.7056 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.3908 global_avg_mtp_loss: 13.3147 +[titan] 2025-07-10 10:50:47,201 - root - INFO - lr: 3.0780e-05 gnorm: 1.03 [21:15:42< 0:44:45] +[titan] 2025-07-10 10:50:51,127 - root - INFO - step: 96615 loss: 15.8497 memory: 44.58GiB(31.99%) tps: 83,474 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4074 global_avg_mtp_loss: 13.4423 +[titan] 2025-07-10 10:50:51,127 - root - INFO - lr: 3.0778e-05 gnorm: 1.06 [21:15:46< 0:44:41] +[titan] 2025-07-10 10:50:55,096 - root - INFO - step: 96620 loss: 16.1915 memory: 44.58GiB(31.99%) tps: 82,560 tflops: 284.93 mfu: 28.81% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7172 +[titan] 2025-07-10 10:50:55,096 - root - INFO - lr: 3.0776e-05 gnorm: 1.10 [21:15:50< 0:44:37] +[titan] 2025-07-10 10:50:59,034 - root - INFO - step: 96625 loss: 16.2475 memory: 44.58GiB(31.99%) tps: 83,228 tflops: 287.23 mfu: 29.04% global_avg_ntp_loss: 2.4853 global_avg_mtp_loss: 13.7623 +[titan] 2025-07-10 10:50:59,034 - root - INFO - lr: 3.0773e-05 gnorm: 1.05 [21:15:54< 0:44:33] +[titan] 2025-07-10 10:51:02,958 - root - INFO - step: 96630 loss: 16.2273 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4713 global_avg_mtp_loss: 13.7560 +[titan] 2025-07-10 10:51:02,959 - root - INFO - lr: 3.0771e-05 gnorm: 1.06 [21:15:58< 0:44:30] +[titan] 2025-07-10 10:51:06,883 - root - INFO - step: 96635 loss: 16.2932 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.5017 global_avg_mtp_loss: 13.7915 +[titan] 2025-07-10 10:51:06,883 - root - INFO - lr: 3.0769e-05 gnorm: 1.05 [21:16:02< 0:44:26] +[titan] 2025-07-10 10:51:10,811 - root - INFO - step: 96640 loss: 15.9805 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4307 global_avg_mtp_loss: 13.5498 +[titan] 2025-07-10 10:51:10,812 - root - INFO - lr: 3.0767e-05 gnorm: 1.05 [21:16:06< 0:44:22] +[titan] 2025-07-10 10:51:14,719 - root - INFO - step: 96645 loss: 16.0993 memory: 44.58GiB(31.99%) tps: 83,873 tflops: 289.46 mfu: 29.27% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6489 +[titan] 2025-07-10 10:51:14,719 - root - INFO - lr: 3.0764e-05 gnorm: 1.01 [21:16:10< 0:44:18] +[titan] 2025-07-10 10:51:17,844 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:51:18,633 - root - INFO - step: 96650 loss: 16.2356 memory: 44.58GiB(31.99%) tps: 83,710 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7577 +[titan] 2025-07-10 10:51:18,634 - root - INFO - lr: 3.0762e-05 gnorm: 1.02 [21:16:14< 0:44:14] +[titan] 2025-07-10 10:51:22,554 - root - INFO - step: 96655 loss: 15.5913 memory: 44.58GiB(31.99%) tps: 83,590 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.3739 global_avg_mtp_loss: 13.2174 +[titan] 2025-07-10 10:51:22,554 - root - INFO - lr: 3.0760e-05 gnorm: 1.10 [21:16:18< 0:44:10] +[titan] 2025-07-10 10:51:26,479 - root - INFO - step: 96660 loss: 16.2593 memory: 44.58GiB(31.99%) tps: 83,488 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4828 global_avg_mtp_loss: 13.7765 +[titan] 2025-07-10 10:51:26,479 - root - INFO - lr: 3.0758e-05 gnorm: 1.07 [21:16:22< 0:44:06] +[titan] 2025-07-10 10:51:30,398 - root - INFO - step: 96665 loss: 16.2898 memory: 44.58GiB(31.99%) tps: 83,618 tflops: 288.58 mfu: 29.18% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.7963 +[titan] 2025-07-10 10:51:30,399 - root - INFO - lr: 3.0755e-05 gnorm: 1.08 [21:16:26< 0:44:02] +[titan] 2025-07-10 10:51:34,329 - root - INFO - step: 96670 loss: 16.1875 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.7257 +[titan] 2025-07-10 10:51:34,330 - root - INFO - lr: 3.0753e-05 gnorm: 1.09 [21:16:29< 0:43:58] +[titan] 2025-07-10 10:51:38,240 - root - INFO - step: 96675 loss: 16.2524 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 10:51:38,240 - root - INFO - lr: 3.0751e-05 gnorm: 1.07 [21:16:33< 0:43:54] +[titan] 2025-07-10 10:51:42,159 - root - INFO - step: 96680 loss: 15.9732 memory: 44.58GiB(31.99%) tps: 83,601 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4209 global_avg_mtp_loss: 13.5523 +[titan] 2025-07-10 10:51:42,160 - root - INFO - lr: 3.0749e-05 gnorm: 1.10 [21:16:37< 0:43:50] +[titan] 2025-07-10 10:51:46,093 - root - INFO - step: 96685 loss: 16.3127 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 10:51:46,094 - root - INFO - lr: 3.0746e-05 gnorm: 1.05 [21:16:41< 0:43:46] +[titan] 2025-07-10 10:51:50,016 - root - INFO - step: 96690 loss: 15.9349 memory: 44.58GiB(31.99%) tps: 83,548 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4270 global_avg_mtp_loss: 13.5079 +[titan] 2025-07-10 10:51:50,016 - root - INFO - lr: 3.0744e-05 gnorm: 1.06 [21:16:45< 0:43:42] +[titan] 2025-07-10 10:51:53,937 - root - INFO - step: 96695 loss: 15.9562 memory: 44.58GiB(31.99%) tps: 83,568 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4294 global_avg_mtp_loss: 13.5268 +[titan] 2025-07-10 10:51:53,938 - root - INFO - lr: 3.0742e-05 gnorm: 1.11 [21:16:49< 0:43:38] +[titan] 2025-07-10 10:51:57,087 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:51:57,881 - root - INFO - step: 96700 loss: 16.1540 memory: 44.58GiB(31.99%) tps: 83,101 tflops: 286.79 mfu: 29.00% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 10:51:57,881 - root - INFO - lr: 3.0740e-05 gnorm: 1.03 [21:16:53< 0:43:34] +[titan] 2025-07-10 10:52:01,815 - root - INFO - step: 96705 loss: 16.4849 memory: 44.58GiB(31.99%) tps: 83,312 tflops: 287.52 mfu: 29.07% global_avg_ntp_loss: 2.5324 global_avg_mtp_loss: 13.9525 +[titan] 2025-07-10 10:52:01,815 - root - INFO - lr: 3.0737e-05 gnorm: 1.09 [21:16:57< 0:43:30] +[titan] 2025-07-10 10:52:05,748 - root - INFO - step: 96710 loss: 16.0773 memory: 44.58GiB(31.99%) tps: 83,318 tflops: 287.54 mfu: 29.07% global_avg_ntp_loss: 2.4536 global_avg_mtp_loss: 13.6237 +[titan] 2025-07-10 10:52:05,748 - root - INFO - lr: 3.0735e-05 gnorm: 1.04 [21:17:01< 0:43:26] +[titan] 2025-07-10 10:52:09,679 - root - INFO - step: 96715 loss: 16.1949 memory: 44.58GiB(31.99%) tps: 83,358 tflops: 287.68 mfu: 29.09% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.7348 +[titan] 2025-07-10 10:52:09,680 - root - INFO - lr: 3.0733e-05 gnorm: 1.04 [21:17:05< 0:43:22] +[titan] 2025-07-10 10:52:13,635 - root - INFO - step: 96720 loss: 16.2425 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4831 global_avg_mtp_loss: 13.7595 +[titan] 2025-07-10 10:52:13,635 - root - INFO - lr: 3.0731e-05 gnorm: 1.07 [21:17:09< 0:43:18] +[titan] 2025-07-10 10:52:17,565 - root - INFO - step: 96725 loss: 16.0903 memory: 44.58GiB(31.99%) tps: 83,385 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.6527 +[titan] 2025-07-10 10:52:17,565 - root - INFO - lr: 3.0728e-05 gnorm: 1.05 [21:17:13< 0:43:14] +[titan] 2025-07-10 10:52:21,476 - root - INFO - step: 96730 loss: 16.0099 memory: 44.58GiB(31.99%) tps: 83,790 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5697 +[titan] 2025-07-10 10:52:21,476 - root - INFO - lr: 3.0726e-05 gnorm: 1.11 [21:17:17< 0:43:10] +[titan] 2025-07-10 10:52:25,422 - root - INFO - step: 96735 loss: 16.0355 memory: 44.58GiB(31.99%) tps: 83,046 tflops: 286.61 mfu: 28.98% global_avg_ntp_loss: 2.4368 global_avg_mtp_loss: 13.5986 +[titan] 2025-07-10 10:52:25,423 - root - INFO - lr: 3.0724e-05 gnorm: 1.09 [21:17:21< 0:43:06] +[titan] 2025-07-10 10:52:29,333 - root - INFO - step: 96740 loss: 16.2822 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4872 global_avg_mtp_loss: 13.7950 +[titan] 2025-07-10 10:52:29,333 - root - INFO - lr: 3.0722e-05 gnorm: 1.01 [21:17:24< 0:43:02] +[titan] 2025-07-10 10:52:33,236 - root - INFO - step: 96745 loss: 16.2207 memory: 44.58GiB(31.99%) tps: 83,958 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7381 +[titan] 2025-07-10 10:52:33,236 - root - INFO - lr: 3.0720e-05 gnorm: 1.05 [21:17:28< 0:42:58] +[titan] 2025-07-10 10:52:36,358 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:52:37,156 - root - INFO - step: 96750 loss: 16.2075 memory: 44.58GiB(31.99%) tps: 83,597 tflops: 288.51 mfu: 29.17% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7298 +[titan] 2025-07-10 10:52:37,157 - root - INFO - lr: 3.0717e-05 gnorm: 1.06 [21:17:32< 0:42:54] +[titan] 2025-07-10 10:52:41,051 - root - INFO - step: 96755 loss: 16.2462 memory: 44.58GiB(31.99%) tps: 84,147 tflops: 290.41 mfu: 29.36% global_avg_ntp_loss: 2.4754 global_avg_mtp_loss: 13.7708 +[titan] 2025-07-10 10:52:41,051 - root - INFO - lr: 3.0715e-05 gnorm: 1.05 [21:17:36< 0:42:50] +[titan] 2025-07-10 10:52:44,984 - root - INFO - step: 96760 loss: 16.2887 memory: 44.58GiB(31.99%) tps: 83,320 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4866 global_avg_mtp_loss: 13.8021 +[titan] 2025-07-10 10:52:44,984 - root - INFO - lr: 3.0713e-05 gnorm: 1.08 [21:17:40< 0:42:46] +[titan] 2025-07-10 10:52:48,965 - root - INFO - step: 96765 loss: 15.8813 memory: 44.58GiB(31.99%) tps: 82,322 tflops: 284.11 mfu: 28.73% global_avg_ntp_loss: 2.4260 global_avg_mtp_loss: 13.4553 +[titan] 2025-07-10 10:52:48,965 - root - INFO - lr: 3.0711e-05 gnorm: 1.06 [21:17:44< 0:42:43] +[titan] 2025-07-10 10:52:51,460 - root - INFO - Dumping profiler traces at step 96768 +[titan] 2025-07-10 10:52:51,492 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 10:52:53,081 - root - INFO - step: 96770 loss: 16.3454 memory: 44.58GiB(31.99%) tps: 79,619 tflops: 274.78 mfu: 27.78% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.8477 +[titan] 2025-07-10 10:52:53,081 - root - INFO - lr: 3.0709e-05 gnorm: 1.03 [21:17:48< 0:42:39] +[titan] 2025-07-10 10:52:56,995 - root - INFO - step: 96775 loss: 15.9256 memory: 44.58GiB(31.99%) tps: 83,716 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4362 global_avg_mtp_loss: 13.4894 +[titan] 2025-07-10 10:52:56,996 - root - INFO - lr: 3.0706e-05 gnorm: 1.17 [21:17:52< 0:42:35] +[titan] 2025-07-10 10:53:00,902 - root - INFO - step: 96780 loss: 16.2883 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4765 global_avg_mtp_loss: 13.8118 +[titan] 2025-07-10 10:53:00,902 - root - INFO - lr: 3.0704e-05 gnorm: 1.10 [21:17:56< 0:42:31] +[titan] 2025-07-10 10:53:04,802 - root - INFO - step: 96785 loss: 15.9852 memory: 44.58GiB(31.99%) tps: 84,029 tflops: 290.00 mfu: 29.32% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5426 +[titan] 2025-07-10 10:53:04,802 - root - INFO - lr: 3.0702e-05 gnorm: 1.06 [21:18:00< 0:42:27] +[titan] 2025-07-10 10:53:08,719 - root - INFO - step: 96790 loss: 16.0467 memory: 44.58GiB(31.99%) tps: 83,652 tflops: 288.70 mfu: 29.19% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.6010 +[titan] 2025-07-10 10:53:08,720 - root - INFO - lr: 3.0700e-05 gnorm: 1.05 [21:18:04< 0:42:23] +[titan] 2025-07-10 10:53:12,636 - root - INFO - step: 96795 loss: 16.0093 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4409 global_avg_mtp_loss: 13.5685 +[titan] 2025-07-10 10:53:12,636 - root - INFO - lr: 3.0698e-05 gnorm: 1.08 [21:18:08< 0:42:19] +[titan] 2025-07-10 10:53:15,758 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:53:16,558 - root - INFO - step: 96800 loss: 16.3795 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4974 global_avg_mtp_loss: 13.8821 +[titan] 2025-07-10 10:53:16,558 - root - INFO - lr: 3.0695e-05 gnorm: 1.02 [21:18:12< 0:42:15] +[titan] 2025-07-10 10:53:20,468 - root - INFO - step: 96805 loss: 16.1033 memory: 44.58GiB(31.99%) tps: 83,811 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.6449 +[titan] 2025-07-10 10:53:20,468 - root - INFO - lr: 3.0693e-05 gnorm: 1.02 [21:18:16< 0:42:11] +[titan] 2025-07-10 10:53:24,373 - root - INFO - step: 96810 loss: 15.8578 memory: 44.58GiB(31.99%) tps: 83,910 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.3951 global_avg_mtp_loss: 13.4626 +[titan] 2025-07-10 10:53:24,374 - root - INFO - lr: 3.0691e-05 gnorm: 1.05 [21:18:19< 0:42:07] +[titan] 2025-07-10 10:53:28,287 - root - INFO - step: 96815 loss: 16.2114 memory: 44.58GiB(31.99%) tps: 83,727 tflops: 288.96 mfu: 29.22% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.7244 +[titan] 2025-07-10 10:53:28,288 - root - INFO - lr: 3.0689e-05 gnorm: 1.05 [21:18:23< 0:42:03] +[titan] 2025-07-10 10:53:32,209 - root - INFO - step: 96820 loss: 16.1809 memory: 44.58GiB(31.99%) tps: 83,566 tflops: 288.40 mfu: 29.16% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.7181 +[titan] 2025-07-10 10:53:32,209 - root - INFO - lr: 3.0687e-05 gnorm: 1.04 [21:18:27< 0:41:59] +[titan] 2025-07-10 10:53:36,125 - root - INFO - step: 96825 loss: 16.1339 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6806 +[titan] 2025-07-10 10:53:36,125 - root - INFO - lr: 3.0685e-05 gnorm: 1.05 [21:18:31< 0:41:55] +[titan] 2025-07-10 10:53:40,066 - root - INFO - step: 96830 loss: 16.4251 memory: 44.58GiB(31.99%) tps: 83,138 tflops: 286.92 mfu: 29.01% global_avg_ntp_loss: 2.5179 global_avg_mtp_loss: 13.9071 +[titan] 2025-07-10 10:53:40,067 - root - INFO - lr: 3.0682e-05 gnorm: 1.08 [21:18:35< 0:41:51] +[titan] 2025-07-10 10:53:43,977 - root - INFO - step: 96835 loss: 16.3424 memory: 44.58GiB(31.99%) tps: 83,808 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.8609 +[titan] 2025-07-10 10:53:43,977 - root - INFO - lr: 3.0680e-05 gnorm: 1.05 [21:18:39< 0:41:47] +[titan] 2025-07-10 10:53:47,885 - root - INFO - step: 96840 loss: 15.8848 memory: 44.58GiB(31.99%) tps: 83,863 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4105 global_avg_mtp_loss: 13.4743 +[titan] 2025-07-10 10:53:47,885 - root - INFO - lr: 3.0678e-05 gnorm: 1.12 [21:18:43< 0:41:43] +[titan] 2025-07-10 10:53:51,809 - root - INFO - step: 96845 loss: 16.0063 memory: 44.58GiB(31.99%) tps: 83,499 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.5613 +[titan] 2025-07-10 10:53:51,810 - root - INFO - lr: 3.0676e-05 gnorm: 1.05 [21:18:47< 0:41:39] +[titan] 2025-07-10 10:53:54,943 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:53:55,737 - root - INFO - step: 96850 loss: 16.1547 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4632 global_avg_mtp_loss: 13.6915 +[titan] 2025-07-10 10:53:55,738 - root - INFO - lr: 3.0674e-05 gnorm: 1.04 [21:18:51< 0:41:35] +[titan] 2025-07-10 10:53:59,718 - root - INFO - step: 96855 loss: 16.0296 memory: 44.58GiB(31.99%) tps: 82,330 tflops: 284.14 mfu: 28.73% global_avg_ntp_loss: 2.4593 global_avg_mtp_loss: 13.5703 +[titan] 2025-07-10 10:53:59,718 - root - INFO - lr: 3.0672e-05 gnorm: 1.09 [21:18:55< 0:41:31] +[titan] 2025-07-10 10:54:03,618 - root - INFO - step: 96860 loss: 16.4306 memory: 44.58GiB(31.99%) tps: 84,020 tflops: 289.97 mfu: 29.32% global_avg_ntp_loss: 2.5162 global_avg_mtp_loss: 13.9144 +[titan] 2025-07-10 10:54:03,618 - root - INFO - lr: 3.0670e-05 gnorm: 1.04 [21:18:59< 0:41:27] +[titan] 2025-07-10 10:54:07,528 - root - INFO - step: 96865 loss: 16.3110 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4806 global_avg_mtp_loss: 13.8305 +[titan] 2025-07-10 10:54:07,528 - root - INFO - lr: 3.0667e-05 gnorm: 1.02 [21:19:03< 0:41:23] +[titan] 2025-07-10 10:54:11,450 - root - INFO - step: 96870 loss: 16.0560 memory: 44.58GiB(31.99%) tps: 83,549 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4327 global_avg_mtp_loss: 13.6233 +[titan] 2025-07-10 10:54:11,451 - root - INFO - lr: 3.0665e-05 gnorm: 1.10 [21:19:07< 0:41:19] +[titan] 2025-07-10 10:54:15,349 - root - INFO - step: 96875 loss: 16.1891 memory: 44.58GiB(31.99%) tps: 84,057 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7186 +[titan] 2025-07-10 10:54:15,349 - root - INFO - lr: 3.0663e-05 gnorm: 1.07 [21:19:10< 0:41:15] +[titan] 2025-07-10 10:54:19,295 - root - INFO - step: 96880 loss: 16.1434 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.4606 global_avg_mtp_loss: 13.6827 +[titan] 2025-07-10 10:54:19,295 - root - INFO - lr: 3.0661e-05 gnorm: 1.07 [21:19:14< 0:41:11] +[titan] 2025-07-10 10:54:23,202 - root - INFO - step: 96885 loss: 15.7922 memory: 44.58GiB(31.99%) tps: 83,883 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.3974 global_avg_mtp_loss: 13.3947 +[titan] 2025-07-10 10:54:23,202 - root - INFO - lr: 3.0659e-05 gnorm: 1.07 [21:19:18< 0:41:07] +[titan] 2025-07-10 10:54:27,117 - root - INFO - step: 96890 loss: 16.1993 memory: 44.58GiB(31.99%) tps: 83,706 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.7120 +[titan] 2025-07-10 10:54:27,117 - root - INFO - lr: 3.0657e-05 gnorm: 1.05 [21:19:22< 0:41:03] +[titan] 2025-07-10 10:54:31,023 - root - INFO - step: 96895 loss: 16.2387 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4732 global_avg_mtp_loss: 13.7655 +[titan] 2025-07-10 10:54:31,023 - root - INFO - lr: 3.0655e-05 gnorm: 1.05 [21:19:26< 0:40:59] +[titan] 2025-07-10 10:54:34,145 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:54:34,930 - root - INFO - step: 96900 loss: 16.2133 memory: 44.58GiB(31.99%) tps: 83,880 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7406 +[titan] 2025-07-10 10:54:34,930 - root - INFO - lr: 3.0653e-05 gnorm: 1.04 [21:19:30< 0:40:56] +[titan] 2025-07-10 10:54:38,832 - root - INFO - step: 96905 loss: 16.2723 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4911 global_avg_mtp_loss: 13.7813 +[titan] 2025-07-10 10:54:38,832 - root - INFO - lr: 3.0651e-05 gnorm: 1.07 [21:19:34< 0:40:52] +[titan] 2025-07-10 10:54:42,743 - root - INFO - step: 96910 loss: 16.3426 memory: 44.58GiB(31.99%) tps: 83,795 tflops: 289.19 mfu: 29.24% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8385 +[titan] 2025-07-10 10:54:42,743 - root - INFO - lr: 3.0648e-05 gnorm: 1.05 [21:19:38< 0:40:48] +[titan] 2025-07-10 10:54:46,666 - root - INFO - step: 96915 loss: 16.3233 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8151 +[titan] 2025-07-10 10:54:46,666 - root - INFO - lr: 3.0646e-05 gnorm: 1.05 [21:19:42< 0:40:44] +[titan] 2025-07-10 10:54:50,580 - root - INFO - step: 96920 loss: 15.9994 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4408 global_avg_mtp_loss: 13.5586 +[titan] 2025-07-10 10:54:50,580 - root - INFO - lr: 3.0644e-05 gnorm: 1.13 [21:19:46< 0:40:40] +[titan] 2025-07-10 10:54:54,528 - root - INFO - step: 96925 loss: 16.2208 memory: 44.58GiB(31.99%) tps: 82,990 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7543 +[titan] 2025-07-10 10:54:54,529 - root - INFO - lr: 3.0642e-05 gnorm: 1.06 [21:19:50< 0:40:36] +[titan] 2025-07-10 10:54:58,468 - root - INFO - step: 96930 loss: 16.2936 memory: 44.58GiB(31.99%) tps: 83,184 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.8063 +[titan] 2025-07-10 10:54:58,468 - root - INFO - lr: 3.0640e-05 gnorm: 1.00 [21:19:54< 0:40:32] +[titan] 2025-07-10 10:55:02,384 - root - INFO - step: 96935 loss: 16.0955 memory: 44.58GiB(31.99%) tps: 83,691 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6265 +[titan] 2025-07-10 10:55:02,384 - root - INFO - lr: 3.0638e-05 gnorm: 1.02 [21:19:57< 0:40:28] +[titan] 2025-07-10 10:55:06,286 - root - INFO - step: 96940 loss: 16.1928 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4706 global_avg_mtp_loss: 13.7222 +[titan] 2025-07-10 10:55:06,286 - root - INFO - lr: 3.0636e-05 gnorm: 1.05 [21:20:01< 0:40:24] +[titan] 2025-07-10 10:55:10,211 - root - INFO - step: 96945 loss: 16.2300 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4745 global_avg_mtp_loss: 13.7555 +[titan] 2025-07-10 10:55:10,212 - root - INFO - lr: 3.0634e-05 gnorm: 1.06 [21:20:05< 0:40:20] +[titan] 2025-07-10 10:55:13,367 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:55:14,157 - root - INFO - step: 96950 loss: 16.1968 memory: 44.58GiB(31.99%) tps: 83,058 tflops: 286.65 mfu: 28.98% global_avg_ntp_loss: 2.4578 global_avg_mtp_loss: 13.7390 +[titan] 2025-07-10 10:55:14,157 - root - INFO - lr: 3.0632e-05 gnorm: 1.07 [21:20:09< 0:40:16] +[titan] 2025-07-10 10:55:18,065 - root - INFO - step: 96955 loss: 16.0355 memory: 44.58GiB(31.99%) tps: 83,862 tflops: 289.42 mfu: 29.26% global_avg_ntp_loss: 2.4459 global_avg_mtp_loss: 13.5897 +[titan] 2025-07-10 10:55:18,065 - root - INFO - lr: 3.0630e-05 gnorm: 1.03 [21:20:13< 0:40:12] +[titan] 2025-07-10 10:55:21,977 - root - INFO - step: 96960 loss: 15.9754 memory: 44.58GiB(31.99%) tps: 83,768 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4279 global_avg_mtp_loss: 13.5475 +[titan] 2025-07-10 10:55:21,977 - root - INFO - lr: 3.0628e-05 gnorm: 1.05 [21:20:17< 0:40:08] +[titan] 2025-07-10 10:55:25,906 - root - INFO - step: 96965 loss: 16.1238 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6656 +[titan] 2025-07-10 10:55:25,906 - root - INFO - lr: 3.0626e-05 gnorm: 1.05 [21:20:21< 0:40:04] +[titan] 2025-07-10 10:55:29,836 - root - INFO - step: 96970 loss: 16.0465 memory: 44.58GiB(31.99%) tps: 83,384 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.5832 +[titan] 2025-07-10 10:55:29,836 - root - INFO - lr: 3.0624e-05 gnorm: 1.10 [21:20:25< 0:40:00] +[titan] 2025-07-10 10:55:33,741 - root - INFO - step: 96975 loss: 16.2030 memory: 44.58GiB(31.99%) tps: 83,913 tflops: 289.60 mfu: 29.28% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.7381 +[titan] 2025-07-10 10:55:33,742 - root - INFO - lr: 3.0622e-05 gnorm: 1.05 [21:20:29< 0:39:56] +[titan] 2025-07-10 10:55:37,699 - root - INFO - step: 96980 loss: 16.4231 memory: 44.58GiB(31.99%) tps: 82,804 tflops: 285.77 mfu: 28.89% global_avg_ntp_loss: 2.5100 global_avg_mtp_loss: 13.9131 +[titan] 2025-07-10 10:55:37,699 - root - INFO - lr: 3.0619e-05 gnorm: 1.01 [21:20:33< 0:39:52] +[titan] 2025-07-10 10:55:41,616 - root - INFO - step: 96985 loss: 15.6172 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.3700 global_avg_mtp_loss: 13.2472 +[titan] 2025-07-10 10:55:41,616 - root - INFO - lr: 3.0617e-05 gnorm: 1.09 [21:20:37< 0:39:48] +[titan] 2025-07-10 10:55:45,568 - root - INFO - step: 96990 loss: 16.2231 memory: 44.58GiB(31.99%) tps: 82,932 tflops: 286.21 mfu: 28.94% global_avg_ntp_loss: 2.4715 global_avg_mtp_loss: 13.7515 +[titan] 2025-07-10 10:55:45,568 - root - INFO - lr: 3.0615e-05 gnorm: 1.04 [21:20:41< 0:39:44] +[titan] 2025-07-10 10:55:49,477 - root - INFO - step: 96995 loss: 15.9742 memory: 44.58GiB(31.99%) tps: 83,829 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4426 global_avg_mtp_loss: 13.5316 +[titan] 2025-07-10 10:55:49,477 - root - INFO - lr: 3.0613e-05 gnorm: 1.03 [21:20:45< 0:39:40] +[titan] 2025-07-10 10:55:52,607 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:55:53,398 - root - INFO - step: 97000 loss: 15.9790 memory: 44.58GiB(31.99%) tps: 83,583 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4347 global_avg_mtp_loss: 13.5443 +[titan] 2025-07-10 10:55:53,398 - root - INFO - lr: 3.0611e-05 gnorm: 1.05 [21:20:49< 0:39:36] +[titan] 2025-07-10 10:55:57,311 - root - INFO - step: 97005 loss: 16.1618 memory: 44.58GiB(31.99%) tps: 83,736 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.7052 +[titan] 2025-07-10 10:55:57,312 - root - INFO - lr: 3.0609e-05 gnorm: 1.07 [21:20:52< 0:39:32] +[titan] 2025-07-10 10:56:01,216 - root - INFO - step: 97010 loss: 16.0620 memory: 44.58GiB(31.99%) tps: 83,934 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4579 global_avg_mtp_loss: 13.6041 +[titan] 2025-07-10 10:56:01,216 - root - INFO - lr: 3.0607e-05 gnorm: 1.09 [21:20:56< 0:39:28] +[titan] 2025-07-10 10:56:05,130 - root - INFO - step: 97015 loss: 16.3287 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4968 global_avg_mtp_loss: 13.8319 +[titan] 2025-07-10 10:56:05,130 - root - INFO - lr: 3.0605e-05 gnorm: 1.07 [21:21:00< 0:39:24] +[titan] 2025-07-10 10:56:09,081 - root - INFO - step: 97020 loss: 16.1636 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.7106 +[titan] 2025-07-10 10:56:09,081 - root - INFO - lr: 3.0603e-05 gnorm: 1.02 [21:21:04< 0:39:20] +[titan] 2025-07-10 10:56:12,976 - root - INFO - step: 97025 loss: 16.2025 memory: 44.58GiB(31.99%) tps: 84,121 tflops: 290.32 mfu: 29.35% global_avg_ntp_loss: 2.4708 global_avg_mtp_loss: 13.7317 +[titan] 2025-07-10 10:56:12,977 - root - INFO - lr: 3.0601e-05 gnorm: 1.13 [21:21:08< 0:39:16] +[titan] 2025-07-10 10:56:16,892 - root - INFO - step: 97030 loss: 16.0686 memory: 44.58GiB(31.99%) tps: 83,703 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.4487 global_avg_mtp_loss: 13.6199 +[titan] 2025-07-10 10:56:16,892 - root - INFO - lr: 3.0599e-05 gnorm: 1.03 [21:21:12< 0:39:12] +[titan] 2025-07-10 10:56:20,809 - root - INFO - step: 97035 loss: 16.3705 memory: 44.58GiB(31.99%) tps: 83,646 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4964 global_avg_mtp_loss: 13.8740 +[titan] 2025-07-10 10:56:20,810 - root - INFO - lr: 3.0597e-05 gnorm: 1.03 [21:21:16< 0:39:09] +[titan] 2025-07-10 10:56:24,760 - root - INFO - step: 97040 loss: 15.9666 memory: 44.58GiB(31.99%) tps: 82,954 tflops: 286.29 mfu: 28.95% global_avg_ntp_loss: 2.4259 global_avg_mtp_loss: 13.5407 +[titan] 2025-07-10 10:56:24,760 - root - INFO - lr: 3.0595e-05 gnorm: 1.20 [21:21:20< 0:39:05] +[titan] 2025-07-10 10:56:28,671 - root - INFO - step: 97045 loss: 16.0639 memory: 44.58GiB(31.99%) tps: 83,797 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6135 +[titan] 2025-07-10 10:56:28,671 - root - INFO - lr: 3.0593e-05 gnorm: 1.06 [21:21:24< 0:39:01] +[titan] 2025-07-10 10:56:31,790 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:56:32,582 - root - INFO - step: 97050 loss: 16.4934 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.5192 global_avg_mtp_loss: 13.9742 +[titan] 2025-07-10 10:56:32,582 - root - INFO - lr: 3.0591e-05 gnorm: 1.01 [21:21:28< 0:38:57] +[titan] 2025-07-10 10:56:36,482 - root - INFO - step: 97055 loss: 15.8650 memory: 44.58GiB(31.99%) tps: 84,017 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4125 global_avg_mtp_loss: 13.4525 +[titan] 2025-07-10 10:56:36,483 - root - INFO - lr: 3.0589e-05 gnorm: 1.10 [21:21:32< 0:38:53] +[titan] 2025-07-10 10:56:40,400 - root - INFO - step: 97060 loss: 16.0608 memory: 44.58GiB(31.99%) tps: 83,648 tflops: 288.68 mfu: 29.19% global_avg_ntp_loss: 2.4603 global_avg_mtp_loss: 13.6005 +[titan] 2025-07-10 10:56:40,400 - root - INFO - lr: 3.0587e-05 gnorm: 1.04 [21:21:36< 0:38:49] +[titan] 2025-07-10 10:56:44,331 - root - INFO - step: 97065 loss: 16.0907 memory: 44.58GiB(31.99%) tps: 83,375 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4588 global_avg_mtp_loss: 13.6319 +[titan] 2025-07-10 10:56:44,331 - root - INFO - lr: 3.0585e-05 gnorm: 1.07 [21:21:39< 0:38:45] +[titan] 2025-07-10 10:56:48,234 - root - INFO - step: 97070 loss: 16.1099 memory: 44.58GiB(31.99%) tps: 83,950 tflops: 289.73 mfu: 29.29% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6595 +[titan] 2025-07-10 10:56:48,235 - root - INFO - lr: 3.0583e-05 gnorm: 1.05 [21:21:43< 0:38:41] +[titan] 2025-07-10 10:56:52,165 - root - INFO - step: 97075 loss: 16.0924 memory: 44.58GiB(31.99%) tps: 83,376 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.6460 +[titan] 2025-07-10 10:56:52,165 - root - INFO - lr: 3.0581e-05 gnorm: 1.05 [21:21:47< 0:38:37] +[titan] 2025-07-10 10:56:56,103 - root - INFO - step: 97080 loss: 16.1577 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6969 +[titan] 2025-07-10 10:56:56,103 - root - INFO - lr: 3.0579e-05 gnorm: 1.09 [21:21:51< 0:38:33] +[titan] 2025-07-10 10:57:00,061 - root - INFO - step: 97085 loss: 16.2987 memory: 44.58GiB(31.99%) tps: 82,793 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.7988 +[titan] 2025-07-10 10:57:00,061 - root - INFO - lr: 3.0577e-05 gnorm: 1.06 [21:21:55< 0:38:29] +[titan] 2025-07-10 10:57:04,072 - root - INFO - step: 97090 loss: 16.2649 memory: 44.58GiB(31.99%) tps: 81,712 tflops: 282.00 mfu: 28.51% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7812 +[titan] 2025-07-10 10:57:04,072 - root - INFO - lr: 3.0575e-05 gnorm: 1.08 [21:21:59< 0:38:25] +[titan] 2025-07-10 10:57:07,986 - root - INFO - step: 97095 loss: 16.1593 memory: 44.58GiB(31.99%) tps: 83,712 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4515 global_avg_mtp_loss: 13.7077 +[titan] 2025-07-10 10:57:07,987 - root - INFO - lr: 3.0573e-05 gnorm: 1.07 [21:22:03< 0:38:21] +[titan] 2025-07-10 10:57:11,160 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:57:11,971 - root - INFO - step: 97100 loss: 16.2864 memory: 44.58GiB(31.99%) tps: 82,242 tflops: 283.83 mfu: 28.70% global_avg_ntp_loss: 2.4841 global_avg_mtp_loss: 13.8024 +[titan] 2025-07-10 10:57:11,971 - root - INFO - lr: 3.0571e-05 gnorm: 1.02 [21:22:07< 0:38:17] +[titan] 2025-07-10 10:57:15,913 - root - INFO - step: 97105 loss: 16.2823 memory: 44.58GiB(31.99%) tps: 83,131 tflops: 286.90 mfu: 29.01% global_avg_ntp_loss: 2.4839 global_avg_mtp_loss: 13.7984 +[titan] 2025-07-10 10:57:15,914 - root - INFO - lr: 3.0569e-05 gnorm: 1.05 [21:22:11< 0:38:13] +[titan] 2025-07-10 10:57:19,807 - root - INFO - step: 97110 loss: 15.9770 memory: 44.58GiB(31.99%) tps: 84,171 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4266 global_avg_mtp_loss: 13.5503 +[titan] 2025-07-10 10:57:19,807 - root - INFO - lr: 3.0567e-05 gnorm: 1.13 [21:22:15< 0:38:09] +[titan] 2025-07-10 10:57:23,717 - root - INFO - step: 97115 loss: 15.6732 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.3682 global_avg_mtp_loss: 13.3050 +[titan] 2025-07-10 10:57:23,717 - root - INFO - lr: 3.0565e-05 gnorm: 1.07 [21:22:19< 0:38:05] +[titan] 2025-07-10 10:57:27,644 - root - INFO - step: 97120 loss: 16.2319 memory: 44.58GiB(31.99%) tps: 83,451 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.7654 +[titan] 2025-07-10 10:57:27,644 - root - INFO - lr: 3.0563e-05 gnorm: 1.06 [21:22:23< 0:38:01] +[titan] 2025-07-10 10:57:31,576 - root - INFO - step: 97125 loss: 16.2683 memory: 44.58GiB(31.99%) tps: 83,351 tflops: 287.66 mfu: 29.09% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7989 +[titan] 2025-07-10 10:57:31,576 - root - INFO - lr: 3.0561e-05 gnorm: 1.06 [21:22:27< 0:37:57] +[titan] 2025-07-10 10:57:35,483 - root - INFO - step: 97130 loss: 16.3444 memory: 44.58GiB(31.99%) tps: 83,875 tflops: 289.47 mfu: 29.27% global_avg_ntp_loss: 2.5091 global_avg_mtp_loss: 13.8353 +[titan] 2025-07-10 10:57:35,483 - root - INFO - lr: 3.0559e-05 gnorm: 1.06 [21:22:31< 0:37:53] +[titan] 2025-07-10 10:57:39,400 - root - INFO - step: 97135 loss: 15.8461 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4233 global_avg_mtp_loss: 13.4228 +[titan] 2025-07-10 10:57:39,400 - root - INFO - lr: 3.0558e-05 gnorm: 1.08 [21:22:34< 0:37:49] +[titan] 2025-07-10 10:57:43,331 - root - INFO - step: 97140 loss: 16.2347 memory: 44.58GiB(31.99%) tps: 83,367 tflops: 287.71 mfu: 29.09% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.7478 +[titan] 2025-07-10 10:57:43,331 - root - INFO - lr: 3.0556e-05 gnorm: 1.05 [21:22:38< 0:37:45] +[titan] 2025-07-10 10:57:47,279 - root - INFO - step: 97145 loss: 16.0573 memory: 44.58GiB(31.99%) tps: 83,008 tflops: 286.47 mfu: 28.97% global_avg_ntp_loss: 2.4389 global_avg_mtp_loss: 13.6184 +[titan] 2025-07-10 10:57:47,279 - root - INFO - lr: 3.0554e-05 gnorm: 1.07 [21:22:42< 0:37:41] +[titan] 2025-07-10 10:57:50,396 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:57:51,188 - root - INFO - step: 97150 loss: 15.8926 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.32 mfu: 29.25% global_avg_ntp_loss: 2.4149 global_avg_mtp_loss: 13.4777 +[titan] 2025-07-10 10:57:51,188 - root - INFO - lr: 3.0552e-05 gnorm: 1.04 [21:22:46< 0:37:37] +[titan] 2025-07-10 10:57:55,128 - root - INFO - step: 97155 loss: 16.2718 memory: 44.58GiB(31.99%) tps: 83,168 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4914 global_avg_mtp_loss: 13.7804 +[titan] 2025-07-10 10:57:55,129 - root - INFO - lr: 3.0550e-05 gnorm: 1.05 [21:22:50< 0:37:33] +[titan] 2025-07-10 10:57:59,050 - root - INFO - step: 97160 loss: 16.0343 memory: 44.58GiB(31.99%) tps: 83,557 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5946 +[titan] 2025-07-10 10:57:59,051 - root - INFO - lr: 3.0548e-05 gnorm: 1.04 [21:22:54< 0:37:29] +[titan] 2025-07-10 10:58:02,969 - root - INFO - step: 97165 loss: 16.0302 memory: 44.58GiB(31.99%) tps: 83,628 tflops: 288.61 mfu: 29.18% global_avg_ntp_loss: 2.4583 global_avg_mtp_loss: 13.5719 +[titan] 2025-07-10 10:58:02,969 - root - INFO - lr: 3.0546e-05 gnorm: 1.06 [21:22:58< 0:37:26] +[titan] 2025-07-10 10:58:06,875 - root - INFO - step: 97170 loss: 16.1701 memory: 44.58GiB(31.99%) tps: 83,902 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4659 global_avg_mtp_loss: 13.7042 +[titan] 2025-07-10 10:58:06,875 - root - INFO - lr: 3.0544e-05 gnorm: 1.11 [21:23:02< 0:37:22] +[titan] 2025-07-10 10:58:10,771 - root - INFO - step: 97175 loss: 16.0915 memory: 44.58GiB(31.99%) tps: 84,107 tflops: 290.27 mfu: 29.35% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6476 +[titan] 2025-07-10 10:58:10,772 - root - INFO - lr: 3.0542e-05 gnorm: 1.07 [21:23:06< 0:37:18] +[titan] 2025-07-10 10:58:14,677 - root - INFO - step: 97180 loss: 16.2098 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7340 +[titan] 2025-07-10 10:58:14,677 - root - INFO - lr: 3.0540e-05 gnorm: 1.05 [21:23:10< 0:37:14] +[titan] 2025-07-10 10:58:18,596 - root - INFO - step: 97185 loss: 15.9262 memory: 44.58GiB(31.99%) tps: 83,620 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.5009 +[titan] 2025-07-10 10:58:18,596 - root - INFO - lr: 3.0538e-05 gnorm: 1.07 [21:23:14< 0:37:10] +[titan] 2025-07-10 10:58:22,501 - root - INFO - step: 97190 loss: 16.1538 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6949 +[titan] 2025-07-10 10:58:22,502 - root - INFO - lr: 3.0536e-05 gnorm: 1.04 [21:23:18< 0:37:06] +[titan] 2025-07-10 10:58:26,446 - root - INFO - step: 97195 loss: 16.0277 memory: 44.58GiB(31.99%) tps: 83,073 tflops: 286.70 mfu: 28.99% global_avg_ntp_loss: 2.4461 global_avg_mtp_loss: 13.5817 +[titan] 2025-07-10 10:58:26,446 - root - INFO - lr: 3.0534e-05 gnorm: 1.03 [21:23:22< 0:37:02] +[titan] 2025-07-10 10:58:29,568 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:58:30,356 - root - INFO - step: 97200 loss: 15.8967 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4098 global_avg_mtp_loss: 13.4869 +[titan] 2025-07-10 10:58:30,357 - root - INFO - lr: 3.0533e-05 gnorm: 1.06 [21:23:25< 0:36:58] +[titan] 2025-07-10 10:58:34,282 - root - INFO - step: 97205 loss: 16.3853 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4960 global_avg_mtp_loss: 13.8892 +[titan] 2025-07-10 10:58:34,282 - root - INFO - lr: 3.0531e-05 gnorm: 1.02 [21:23:29< 0:36:54] +[titan] 2025-07-10 10:58:38,185 - root - INFO - step: 97210 loss: 16.0439 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.5935 +[titan] 2025-07-10 10:58:38,185 - root - INFO - lr: 3.0529e-05 gnorm: 1.03 [21:23:33< 0:36:50] +[titan] 2025-07-10 10:58:42,083 - root - INFO - step: 97215 loss: 16.1649 memory: 44.58GiB(31.99%) tps: 84,082 tflops: 290.18 mfu: 29.34% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.7031 +[titan] 2025-07-10 10:58:42,083 - root - INFO - lr: 3.0527e-05 gnorm: 1.06 [21:23:37< 0:36:46] +[titan] 2025-07-10 10:58:45,987 - root - INFO - step: 97220 loss: 16.3588 memory: 44.58GiB(31.99%) tps: 83,942 tflops: 289.70 mfu: 29.29% global_avg_ntp_loss: 2.5029 global_avg_mtp_loss: 13.8560 +[titan] 2025-07-10 10:58:45,987 - root - INFO - lr: 3.0525e-05 gnorm: 1.01 [21:23:41< 0:36:42] +[titan] 2025-07-10 10:58:49,896 - root - INFO - step: 97225 loss: 16.4548 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.5369 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 10:58:49,896 - root - INFO - lr: 3.0523e-05 gnorm: 1.06 [21:23:45< 0:36:38] +[titan] 2025-07-10 10:58:53,841 - root - INFO - step: 97230 loss: 16.0028 memory: 44.58GiB(31.99%) tps: 83,065 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.4391 global_avg_mtp_loss: 13.5637 +[titan] 2025-07-10 10:58:53,841 - root - INFO - lr: 3.0521e-05 gnorm: 1.01 [21:23:49< 0:36:34] +[titan] 2025-07-10 10:58:57,808 - root - INFO - step: 97235 loss: 16.1659 memory: 44.58GiB(31.99%) tps: 82,606 tflops: 285.09 mfu: 28.83% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6998 +[titan] 2025-07-10 10:58:57,808 - root - INFO - lr: 3.0519e-05 gnorm: 1.06 [21:23:53< 0:36:30] +[titan] 2025-07-10 10:59:01,730 - root - INFO - step: 97240 loss: 15.9890 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4372 global_avg_mtp_loss: 13.5518 +[titan] 2025-07-10 10:59:01,730 - root - INFO - lr: 3.0517e-05 gnorm: 1.07 [21:23:57< 0:36:26] +[titan] 2025-07-10 10:59:05,658 - root - INFO - step: 97245 loss: 15.9716 memory: 44.58GiB(31.99%) tps: 83,423 tflops: 287.91 mfu: 29.11% global_avg_ntp_loss: 2.4259 global_avg_mtp_loss: 13.5457 +[titan] 2025-07-10 10:59:05,658 - root - INFO - lr: 3.0516e-05 gnorm: 1.10 [21:24:01< 0:36:22] +[titan] 2025-07-10 10:59:08,785 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:59:09,575 - root - INFO - step: 97250 loss: 16.4514 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.5169 global_avg_mtp_loss: 13.9345 +[titan] 2025-07-10 10:59:09,575 - root - INFO - lr: 3.0514e-05 gnorm: 1.05 [21:24:05< 0:36:18] +[titan] 2025-07-10 10:59:13,484 - root - INFO - step: 97255 loss: 16.1151 memory: 44.58GiB(31.99%) tps: 83,831 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6507 +[titan] 2025-07-10 10:59:13,485 - root - INFO - lr: 3.0512e-05 gnorm: 1.08 [21:24:09< 0:36:14] +[titan] 2025-07-10 10:59:17,390 - root - INFO - step: 97260 loss: 15.8808 memory: 44.58GiB(31.99%) tps: 83,899 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4159 global_avg_mtp_loss: 13.4649 +[titan] 2025-07-10 10:59:17,391 - root - INFO - lr: 3.0510e-05 gnorm: 1.05 [21:24:12< 0:36:10] +[titan] 2025-07-10 10:59:21,311 - root - INFO - step: 97265 loss: 16.1523 memory: 44.58GiB(31.99%) tps: 83,579 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.6892 +[titan] 2025-07-10 10:59:21,312 - root - INFO - lr: 3.0508e-05 gnorm: 1.08 [21:24:16< 0:36:06] +[titan] 2025-07-10 10:59:25,263 - root - INFO - step: 97270 loss: 16.2405 memory: 44.58GiB(31.99%) tps: 82,934 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.7557 +[titan] 2025-07-10 10:59:25,263 - root - INFO - lr: 3.0506e-05 gnorm: 1.04 [21:24:20< 0:36:02] +[titan] 2025-07-10 10:59:29,186 - root - INFO - step: 97275 loss: 16.4018 memory: 44.58GiB(31.99%) tps: 83,532 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4855 global_avg_mtp_loss: 13.9163 +[titan] 2025-07-10 10:59:29,186 - root - INFO - lr: 3.0504e-05 gnorm: 1.00 [21:24:24< 0:35:58] +[titan] 2025-07-10 10:59:33,189 - root - INFO - step: 97280 loss: 16.3828 memory: 44.58GiB(31.99%) tps: 81,868 tflops: 282.54 mfu: 28.57% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8882 +[titan] 2025-07-10 10:59:33,189 - root - INFO - lr: 3.0503e-05 gnorm: 1.03 [21:24:28< 0:35:54] +[titan] 2025-07-10 10:59:33,347 - root - INFO - Dumping profiler traces at step 97280 +[titan] 2025-07-10 10:59:33,383 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 10:59:37,326 - root - INFO - step: 97285 loss: 16.1042 memory: 44.58GiB(31.99%) tps: 79,221 tflops: 273.40 mfu: 27.64% global_avg_ntp_loss: 2.4515 global_avg_mtp_loss: 13.6527 +[titan] 2025-07-10 10:59:37,326 - root - INFO - lr: 3.0501e-05 gnorm: 1.09 [21:24:32< 0:35:50] +[titan] 2025-07-10 10:59:41,249 - root - INFO - step: 97290 loss: 16.3295 memory: 44.58GiB(31.99%) tps: 83,538 tflops: 288.30 mfu: 29.15% global_avg_ntp_loss: 2.5003 global_avg_mtp_loss: 13.8291 +[titan] 2025-07-10 10:59:41,249 - root - INFO - lr: 3.0499e-05 gnorm: 1.06 [21:24:36< 0:35:46] +[titan] 2025-07-10 10:59:45,147 - root - INFO - step: 97295 loss: 16.1054 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4439 global_avg_mtp_loss: 13.6615 +[titan] 2025-07-10 10:59:45,148 - root - INFO - lr: 3.0497e-05 gnorm: 1.04 [21:24:40< 0:35:43] +[titan] 2025-07-10 10:59:48,294 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 10:59:49,079 - root - INFO - step: 97300 loss: 16.2913 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.4932 global_avg_mtp_loss: 13.7981 +[titan] 2025-07-10 10:59:49,080 - root - INFO - lr: 3.0495e-05 gnorm: 1.03 [21:24:44< 0:35:39] +[titan] 2025-07-10 10:59:53,005 - root - INFO - step: 97305 loss: 16.3492 memory: 44.58GiB(31.99%) tps: 83,487 tflops: 288.13 mfu: 29.13% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.8526 +[titan] 2025-07-10 10:59:53,005 - root - INFO - lr: 3.0493e-05 gnorm: 1.01 [21:24:48< 0:35:35] +[titan] 2025-07-10 10:59:56,913 - root - INFO - step: 97310 loss: 16.2121 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.7424 +[titan] 2025-07-10 10:59:56,914 - root - INFO - lr: 3.0492e-05 gnorm: 1.08 [21:24:52< 0:35:31] +[titan] 2025-07-10 11:00:00,824 - root - INFO - step: 97315 loss: 16.1932 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7192 +[titan] 2025-07-10 11:00:00,824 - root - INFO - lr: 3.0490e-05 gnorm: 1.12 [21:24:56< 0:35:27] +[titan] 2025-07-10 11:00:04,779 - root - INFO - step: 97320 loss: 16.1896 memory: 44.58GiB(31.99%) tps: 82,848 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4753 global_avg_mtp_loss: 13.7143 +[titan] 2025-07-10 11:00:04,779 - root - INFO - lr: 3.0488e-05 gnorm: 1.02 [21:25:00< 0:35:23] +[titan] 2025-07-10 11:00:08,678 - root - INFO - step: 97325 loss: 15.9811 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4281 global_avg_mtp_loss: 13.5530 +[titan] 2025-07-10 11:00:08,678 - root - INFO - lr: 3.0486e-05 gnorm: 1.02 [21:25:04< 0:35:19] +[titan] 2025-07-10 11:00:12,619 - root - INFO - step: 97330 loss: 16.0830 memory: 44.58GiB(31.99%) tps: 83,160 tflops: 287.00 mfu: 29.02% global_avg_ntp_loss: 2.4480 global_avg_mtp_loss: 13.6349 +[titan] 2025-07-10 11:00:12,619 - root - INFO - lr: 3.0484e-05 gnorm: 1.05 [21:25:08< 0:35:15] +[titan] 2025-07-10 11:00:16,528 - root - INFO - step: 97335 loss: 15.9923 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4330 global_avg_mtp_loss: 13.5593 +[titan] 2025-07-10 11:00:16,529 - root - INFO - lr: 3.0482e-05 gnorm: 1.11 [21:25:12< 0:35:11] +[titan] 2025-07-10 11:00:20,440 - root - INFO - step: 97340 loss: 16.0698 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6160 +[titan] 2025-07-10 11:00:20,441 - root - INFO - lr: 3.0481e-05 gnorm: 1.06 [21:25:16< 0:35:07] +[titan] 2025-07-10 11:00:24,348 - root - INFO - step: 97345 loss: 16.3261 memory: 44.58GiB(31.99%) tps: 83,865 tflops: 289.43 mfu: 29.26% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.8344 +[titan] 2025-07-10 11:00:24,348 - root - INFO - lr: 3.0479e-05 gnorm: 1.03 [21:25:19< 0:35:03] +[titan] 2025-07-10 11:00:27,481 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:00:28,268 - root - INFO - step: 97350 loss: 15.9577 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4207 global_avg_mtp_loss: 13.5369 +[titan] 2025-07-10 11:00:28,268 - root - INFO - lr: 3.0477e-05 gnorm: 1.05 [21:25:23< 0:34:59] +[titan] 2025-07-10 11:00:32,215 - root - INFO - step: 97355 loss: 15.9990 memory: 44.58GiB(31.99%) tps: 83,023 tflops: 286.52 mfu: 28.97% global_avg_ntp_loss: 2.4567 global_avg_mtp_loss: 13.5423 +[titan] 2025-07-10 11:00:32,215 - root - INFO - lr: 3.0475e-05 gnorm: 1.07 [21:25:27< 0:34:55] +[titan] 2025-07-10 11:00:36,181 - root - INFO - step: 97360 loss: 16.1363 memory: 44.58GiB(31.99%) tps: 82,629 tflops: 285.16 mfu: 28.83% global_avg_ntp_loss: 2.4816 global_avg_mtp_loss: 13.6547 +[titan] 2025-07-10 11:00:36,181 - root - INFO - lr: 3.0473e-05 gnorm: 1.11 [21:25:31< 0:34:51] +[titan] 2025-07-10 11:00:40,075 - root - INFO - step: 97365 loss: 16.2258 memory: 44.58GiB(31.99%) tps: 84,168 tflops: 290.48 mfu: 29.37% global_avg_ntp_loss: 2.4747 global_avg_mtp_loss: 13.7510 +[titan] 2025-07-10 11:00:40,075 - root - INFO - lr: 3.0472e-05 gnorm: 1.04 [21:25:35< 0:34:47] +[titan] 2025-07-10 11:00:43,997 - root - INFO - step: 97370 loss: 16.0605 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4403 global_avg_mtp_loss: 13.6202 +[titan] 2025-07-10 11:00:43,997 - root - INFO - lr: 3.0470e-05 gnorm: 1.05 [21:25:39< 0:34:43] +[titan] 2025-07-10 11:00:47,918 - root - INFO - step: 97375 loss: 15.9936 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4417 global_avg_mtp_loss: 13.5520 +[titan] 2025-07-10 11:00:47,919 - root - INFO - lr: 3.0468e-05 gnorm: 1.11 [21:25:43< 0:34:39] +[titan] 2025-07-10 11:00:51,847 - root - INFO - step: 97380 loss: 16.1335 memory: 44.58GiB(31.99%) tps: 83,412 tflops: 287.87 mfu: 29.11% global_avg_ntp_loss: 2.4630 global_avg_mtp_loss: 13.6706 +[titan] 2025-07-10 11:00:51,848 - root - INFO - lr: 3.0466e-05 gnorm: 1.02 [21:25:47< 0:34:35] +[titan] 2025-07-10 11:00:55,782 - root - INFO - step: 97385 loss: 16.0697 memory: 44.58GiB(31.99%) tps: 83,293 tflops: 287.46 mfu: 29.07% global_avg_ntp_loss: 2.4485 global_avg_mtp_loss: 13.6212 +[titan] 2025-07-10 11:00:55,782 - root - INFO - lr: 3.0465e-05 gnorm: 1.05 [21:25:51< 0:34:31] +[titan] 2025-07-10 11:00:59,706 - root - INFO - step: 97390 loss: 16.1102 memory: 44.58GiB(31.99%) tps: 83,514 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4620 global_avg_mtp_loss: 13.6483 +[titan] 2025-07-10 11:00:59,706 - root - INFO - lr: 3.0463e-05 gnorm: 1.03 [21:25:55< 0:34:27] +[titan] 2025-07-10 11:01:03,653 - root - INFO - step: 97395 loss: 16.2347 memory: 44.58GiB(31.99%) tps: 83,028 tflops: 286.54 mfu: 28.97% global_avg_ntp_loss: 2.4720 global_avg_mtp_loss: 13.7628 +[titan] 2025-07-10 11:01:03,653 - root - INFO - lr: 3.0461e-05 gnorm: 1.06 [21:25:59< 0:34:23] +[titan] 2025-07-10 11:01:06,797 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:01:07,608 - root - INFO - step: 97400 loss: 16.2168 memory: 44.58GiB(31.99%) tps: 82,862 tflops: 285.97 mfu: 28.92% global_avg_ntp_loss: 2.4995 global_avg_mtp_loss: 13.7173 +[titan] 2025-07-10 11:01:07,608 - root - INFO - lr: 3.0459e-05 gnorm: 1.12 [21:26:03< 0:34:19] +[titan] 2025-07-10 11:01:11,517 - root - INFO - step: 97405 loss: 16.4448 memory: 44.58GiB(31.99%) tps: 83,840 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.5208 global_avg_mtp_loss: 13.9240 +[titan] 2025-07-10 11:01:11,517 - root - INFO - lr: 3.0457e-05 gnorm: 1.06 [21:26:07< 0:34:15] +[titan] 2025-07-10 11:01:15,417 - root - INFO - step: 97410 loss: 16.1966 memory: 44.58GiB(31.99%) tps: 84,016 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7278 +[titan] 2025-07-10 11:01:15,417 - root - INFO - lr: 3.0456e-05 gnorm: 1.09 [21:26:10< 0:34:11] +[titan] 2025-07-10 11:01:19,349 - root - INFO - step: 97415 loss: 16.3689 memory: 44.58GiB(31.99%) tps: 83,341 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.5078 global_avg_mtp_loss: 13.8611 +[titan] 2025-07-10 11:01:19,350 - root - INFO - lr: 3.0454e-05 gnorm: 1.06 [21:26:14< 0:34:07] +[titan] 2025-07-10 11:01:23,313 - root - INFO - step: 97420 loss: 16.1169 memory: 44.58GiB(31.99%) tps: 82,679 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.4539 global_avg_mtp_loss: 13.6630 +[titan] 2025-07-10 11:01:23,313 - root - INFO - lr: 3.0452e-05 gnorm: 1.08 [21:26:18< 0:34:03] +[titan] 2025-07-10 11:01:27,233 - root - INFO - step: 97425 loss: 16.1181 memory: 44.58GiB(31.99%) tps: 83,600 tflops: 288.52 mfu: 29.17% global_avg_ntp_loss: 2.4529 global_avg_mtp_loss: 13.6652 +[titan] 2025-07-10 11:01:27,233 - root - INFO - lr: 3.0450e-05 gnorm: 1.08 [21:26:22< 0:33:59] +[titan] 2025-07-10 11:01:31,164 - root - INFO - step: 97430 loss: 16.2794 memory: 44.58GiB(31.99%) tps: 83,369 tflops: 287.72 mfu: 29.09% global_avg_ntp_loss: 2.4809 global_avg_mtp_loss: 13.7985 +[titan] 2025-07-10 11:01:31,164 - root - INFO - lr: 3.0449e-05 gnorm: 1.07 [21:26:26< 0:33:56] +[titan] 2025-07-10 11:01:35,074 - root - INFO - step: 97435 loss: 16.3072 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.8035 +[titan] 2025-07-10 11:01:35,074 - root - INFO - lr: 3.0447e-05 gnorm: 1.06 [21:26:30< 0:33:52] +[titan] 2025-07-10 11:01:39,028 - root - INFO - step: 97440 loss: 16.1149 memory: 44.58GiB(31.99%) tps: 82,875 tflops: 286.01 mfu: 28.92% global_avg_ntp_loss: 2.4528 global_avg_mtp_loss: 13.6621 +[titan] 2025-07-10 11:01:39,028 - root - INFO - lr: 3.0445e-05 gnorm: 1.08 [21:26:34< 0:33:48] +[titan] 2025-07-10 11:01:42,964 - root - INFO - step: 97445 loss: 15.8654 memory: 44.58GiB(31.99%) tps: 83,254 tflops: 287.32 mfu: 29.05% global_avg_ntp_loss: 2.4004 global_avg_mtp_loss: 13.4650 +[titan] 2025-07-10 11:01:42,964 - root - INFO - lr: 3.0443e-05 gnorm: 1.10 [21:26:38< 0:33:44] +[titan] 2025-07-10 11:01:46,096 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:01:46,883 - root - INFO - step: 97450 loss: 16.0983 memory: 44.58GiB(31.99%) tps: 83,631 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4523 global_avg_mtp_loss: 13.6460 +[titan] 2025-07-10 11:01:46,883 - root - INFO - lr: 3.0442e-05 gnorm: 1.08 [21:26:42< 0:33:40] +[titan] 2025-07-10 11:01:50,845 - root - INFO - step: 97455 loss: 16.1823 memory: 44.58GiB(31.99%) tps: 82,710 tflops: 285.45 mfu: 28.86% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7092 +[titan] 2025-07-10 11:01:50,845 - root - INFO - lr: 3.0440e-05 gnorm: 1.09 [21:26:46< 0:33:36] +[titan] 2025-07-10 11:01:54,766 - root - INFO - step: 97460 loss: 16.0024 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4330 global_avg_mtp_loss: 13.5694 +[titan] 2025-07-10 11:01:54,766 - root - INFO - lr: 3.0438e-05 gnorm: 1.07 [21:26:50< 0:33:32] +[titan] 2025-07-10 11:01:58,662 - root - INFO - step: 97465 loss: 16.4853 memory: 44.58GiB(31.99%) tps: 84,126 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.5271 global_avg_mtp_loss: 13.9582 +[titan] 2025-07-10 11:01:58,662 - root - INFO - lr: 3.0437e-05 gnorm: 1.02 [21:26:54< 0:33:28] +[titan] 2025-07-10 11:02:02,612 - root - INFO - step: 97470 loss: 16.4059 memory: 44.58GiB(31.99%) tps: 82,964 tflops: 286.32 mfu: 28.95% global_avg_ntp_loss: 2.5224 global_avg_mtp_loss: 13.8834 +[titan] 2025-07-10 11:02:02,612 - root - INFO - lr: 3.0435e-05 gnorm: 1.06 [21:26:58< 0:33:24] +[titan] 2025-07-10 11:02:06,516 - root - INFO - step: 97475 loss: 16.3381 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.8544 +[titan] 2025-07-10 11:02:06,517 - root - INFO - lr: 3.0433e-05 gnorm: 1.08 [21:27:02< 0:33:20] +[titan] 2025-07-10 11:02:10,414 - root - INFO - step: 97480 loss: 16.2453 memory: 44.58GiB(31.99%) tps: 84,073 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7676 +[titan] 2025-07-10 11:02:10,415 - root - INFO - lr: 3.0431e-05 gnorm: 1.08 [21:27:05< 0:33:16] +[titan] 2025-07-10 11:02:14,396 - root - INFO - step: 97485 loss: 16.3363 memory: 44.58GiB(31.99%) tps: 82,308 tflops: 284.06 mfu: 28.72% global_avg_ntp_loss: 2.4891 global_avg_mtp_loss: 13.8471 +[titan] 2025-07-10 11:02:14,396 - root - INFO - lr: 3.0430e-05 gnorm: 1.08 [21:27:09< 0:33:12] +[titan] 2025-07-10 11:02:18,328 - root - INFO - step: 97490 loss: 16.1085 memory: 44.58GiB(31.99%) tps: 83,337 tflops: 287.61 mfu: 29.08% global_avg_ntp_loss: 2.4553 global_avg_mtp_loss: 13.6532 +[titan] 2025-07-10 11:02:18,329 - root - INFO - lr: 3.0428e-05 gnorm: 1.08 [21:27:13< 0:33:08] +[titan] 2025-07-10 11:02:22,299 - root - INFO - step: 97495 loss: 16.2713 memory: 44.58GiB(31.99%) tps: 82,541 tflops: 284.86 mfu: 28.80% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7852 +[titan] 2025-07-10 11:02:22,299 - root - INFO - lr: 3.0426e-05 gnorm: 1.07 [21:27:17< 0:33:04] +[titan] 2025-07-10 11:02:25,412 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:02:26,207 - root - INFO - step: 97500 loss: 16.2541 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7750 +[titan] 2025-07-10 11:02:26,208 - root - INFO - lr: 3.0425e-05 gnorm: 1.06 [21:27:21< 0:33:00] +[titan] 2025-07-10 11:02:30,113 - root - INFO - step: 97505 loss: 15.9899 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.5525 +[titan] 2025-07-10 11:02:30,114 - root - INFO - lr: 3.0423e-05 gnorm: 0.99 [21:27:25< 0:32:56] +[titan] 2025-07-10 11:02:34,020 - root - INFO - step: 97510 loss: 16.1567 memory: 44.58GiB(31.99%) tps: 83,898 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4961 global_avg_mtp_loss: 13.6606 +[titan] 2025-07-10 11:02:34,020 - root - INFO - lr: 3.0421e-05 gnorm: 1.11 [21:27:29< 0:32:52] +[titan] 2025-07-10 11:02:38,005 - root - INFO - step: 97515 loss: 16.3623 memory: 44.58GiB(31.99%) tps: 82,227 tflops: 283.78 mfu: 28.69% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8652 +[titan] 2025-07-10 11:02:38,005 - root - INFO - lr: 3.0420e-05 gnorm: 1.08 [21:27:33< 0:32:48] +[titan] 2025-07-10 11:02:41,918 - root - INFO - step: 97520 loss: 16.1214 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4538 global_avg_mtp_loss: 13.6676 +[titan] 2025-07-10 11:02:41,918 - root - INFO - lr: 3.0418e-05 gnorm: 1.07 [21:27:37< 0:32:44] +[titan] 2025-07-10 11:02:45,819 - root - INFO - step: 97525 loss: 16.3635 memory: 44.58GiB(31.99%) tps: 83,998 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.8605 +[titan] 2025-07-10 11:02:45,820 - root - INFO - lr: 3.0416e-05 gnorm: 1.03 [21:27:41< 0:32:40] +[titan] 2025-07-10 11:02:49,748 - root - INFO - step: 97530 loss: 16.5468 memory: 44.58GiB(31.99%) tps: 83,419 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5391 global_avg_mtp_loss: 14.0077 +[titan] 2025-07-10 11:02:49,748 - root - INFO - lr: 3.0414e-05 gnorm: 1.01 [21:27:45< 0:32:36] +[titan] 2025-07-10 11:02:53,669 - root - INFO - step: 97535 loss: 16.3453 memory: 44.58GiB(31.99%) tps: 83,575 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4868 global_avg_mtp_loss: 13.8585 +[titan] 2025-07-10 11:02:53,669 - root - INFO - lr: 3.0413e-05 gnorm: 1.05 [21:27:49< 0:32:32] +[titan] 2025-07-10 11:02:57,586 - root - INFO - step: 97540 loss: 16.1509 memory: 44.58GiB(31.99%) tps: 83,665 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.6894 +[titan] 2025-07-10 11:02:57,586 - root - INFO - lr: 3.0411e-05 gnorm: 1.13 [21:27:53< 0:32:28] +[titan] 2025-07-10 11:03:01,495 - root - INFO - step: 97545 loss: 16.2596 memory: 44.58GiB(31.99%) tps: 83,827 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4787 global_avg_mtp_loss: 13.7809 +[titan] 2025-07-10 11:03:01,496 - root - INFO - lr: 3.0409e-05 gnorm: 1.09 [21:27:57< 0:32:24] +[titan] 2025-07-10 11:03:04,616 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:03:05,402 - root - INFO - step: 97550 loss: 16.3448 memory: 44.58GiB(31.99%) tps: 83,881 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4945 global_avg_mtp_loss: 13.8504 +[titan] 2025-07-10 11:03:05,403 - root - INFO - lr: 3.0408e-05 gnorm: 1.04 [21:28:00< 0:32:20] +[titan] 2025-07-10 11:03:09,309 - root - INFO - step: 97555 loss: 16.2810 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7952 +[titan] 2025-07-10 11:03:09,309 - root - INFO - lr: 3.0406e-05 gnorm: 1.04 [21:28:04< 0:32:16] +[titan] 2025-07-10 11:03:13,221 - root - INFO - step: 97560 loss: 16.0720 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4407 global_avg_mtp_loss: 13.6313 +[titan] 2025-07-10 11:03:13,221 - root - INFO - lr: 3.0404e-05 gnorm: 1.07 [21:28:08< 0:32:13] +[titan] 2025-07-10 11:03:17,175 - root - INFO - step: 97565 loss: 16.2007 memory: 44.58GiB(31.99%) tps: 82,877 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.4682 global_avg_mtp_loss: 13.7325 +[titan] 2025-07-10 11:03:17,176 - root - INFO - lr: 3.0403e-05 gnorm: 1.06 [21:28:12< 0:32:09] +[titan] 2025-07-10 11:03:21,113 - root - INFO - step: 97570 loss: 16.0497 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.6062 +[titan] 2025-07-10 11:03:21,113 - root - INFO - lr: 3.0401e-05 gnorm: 1.06 [21:28:16< 0:32:05] +[titan] 2025-07-10 11:03:25,016 - root - INFO - step: 97575 loss: 16.4378 memory: 44.58GiB(31.99%) tps: 83,964 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.5246 global_avg_mtp_loss: 13.9132 +[titan] 2025-07-10 11:03:25,016 - root - INFO - lr: 3.0400e-05 gnorm: 1.05 [21:28:20< 0:32:01] +[titan] 2025-07-10 11:03:28,949 - root - INFO - step: 97580 loss: 16.1504 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6900 +[titan] 2025-07-10 11:03:28,949 - root - INFO - lr: 3.0398e-05 gnorm: 1.09 [21:28:24< 0:31:57] +[titan] 2025-07-10 11:03:32,888 - root - INFO - step: 97585 loss: 16.1488 memory: 44.58GiB(31.99%) tps: 83,198 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4589 global_avg_mtp_loss: 13.6899 +[titan] 2025-07-10 11:03:32,888 - root - INFO - lr: 3.0396e-05 gnorm: 1.08 [21:28:28< 0:31:53] +[titan] 2025-07-10 11:03:36,856 - root - INFO - step: 97590 loss: 16.1933 memory: 44.58GiB(31.99%) tps: 82,579 tflops: 284.99 mfu: 28.82% global_avg_ntp_loss: 2.4656 global_avg_mtp_loss: 13.7277 +[titan] 2025-07-10 11:03:36,857 - root - INFO - lr: 3.0395e-05 gnorm: 1.04 [21:28:32< 0:31:49] +[titan] 2025-07-10 11:03:40,766 - root - INFO - step: 97595 loss: 15.9690 memory: 44.58GiB(31.99%) tps: 83,825 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.4212 global_avg_mtp_loss: 13.5478 +[titan] 2025-07-10 11:03:40,766 - root - INFO - lr: 3.0393e-05 gnorm: 1.05 [21:28:36< 0:31:45] +[titan] 2025-07-10 11:03:43,886 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:03:44,686 - root - INFO - step: 97600 loss: 16.1130 memory: 44.58GiB(31.99%) tps: 83,604 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4484 global_avg_mtp_loss: 13.6646 +[titan] 2025-07-10 11:03:44,686 - root - INFO - lr: 3.0391e-05 gnorm: 1.05 [21:28:40< 0:31:41] +[titan] 2025-07-10 11:03:48,656 - root - INFO - step: 97605 loss: 16.1746 memory: 44.58GiB(31.99%) tps: 82,550 tflops: 284.89 mfu: 28.81% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.7197 +[titan] 2025-07-10 11:03:48,656 - root - INFO - lr: 3.0390e-05 gnorm: 1.04 [21:28:44< 0:31:37] +[titan] 2025-07-10 11:03:52,588 - root - INFO - step: 97610 loss: 16.4535 memory: 44.58GiB(31.99%) tps: 83,343 tflops: 287.63 mfu: 29.08% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 11:03:52,588 - root - INFO - lr: 3.0388e-05 gnorm: 1.09 [21:28:48< 0:31:33] +[titan] 2025-07-10 11:03:56,510 - root - INFO - step: 97615 loss: 16.2421 memory: 44.58GiB(31.99%) tps: 83,547 tflops: 288.34 mfu: 29.15% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.7583 +[titan] 2025-07-10 11:03:56,510 - root - INFO - lr: 3.0386e-05 gnorm: 1.09 [21:28:52< 0:31:29] +[titan] 2025-07-10 11:04:00,406 - root - INFO - step: 97620 loss: 16.2794 memory: 44.58GiB(31.99%) tps: 84,114 tflops: 290.29 mfu: 29.35% global_avg_ntp_loss: 2.5037 global_avg_mtp_loss: 13.7757 +[titan] 2025-07-10 11:04:00,407 - root - INFO - lr: 3.0385e-05 gnorm: 1.06 [21:28:55< 0:31:25] +[titan] 2025-07-10 11:04:04,311 - root - INFO - step: 97625 loss: 16.2125 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4775 global_avg_mtp_loss: 13.7350 +[titan] 2025-07-10 11:04:04,312 - root - INFO - lr: 3.0383e-05 gnorm: 1.06 [21:28:59< 0:31:21] +[titan] 2025-07-10 11:04:08,242 - root - INFO - step: 97630 loss: 15.8780 memory: 44.58GiB(31.99%) tps: 83,374 tflops: 287.74 mfu: 29.09% global_avg_ntp_loss: 2.4162 global_avg_mtp_loss: 13.4618 +[titan] 2025-07-10 11:04:08,242 - root - INFO - lr: 3.0382e-05 gnorm: 1.05 [21:29:03< 0:31:17] +[titan] 2025-07-10 11:04:12,144 - root - INFO - step: 97635 loss: 16.0475 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4405 global_avg_mtp_loss: 13.6070 +[titan] 2025-07-10 11:04:12,145 - root - INFO - lr: 3.0380e-05 gnorm: 1.03 [21:29:07< 0:31:13] +[titan] 2025-07-10 11:04:16,067 - root - INFO - step: 97640 loss: 16.0510 memory: 44.58GiB(31.99%) tps: 83,545 tflops: 288.33 mfu: 29.15% global_avg_ntp_loss: 2.4404 global_avg_mtp_loss: 13.6106 +[titan] 2025-07-10 11:04:16,067 - root - INFO - lr: 3.0378e-05 gnorm: 1.06 [21:29:11< 0:31:09] +[titan] 2025-07-10 11:04:20,005 - root - INFO - step: 97645 loss: 15.9701 memory: 44.58GiB(31.99%) tps: 83,212 tflops: 287.18 mfu: 29.04% global_avg_ntp_loss: 2.4555 global_avg_mtp_loss: 13.5146 +[titan] 2025-07-10 11:04:20,005 - root - INFO - lr: 3.0377e-05 gnorm: 1.08 [21:29:15< 0:31:05] +[titan] 2025-07-10 11:04:23,154 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:04:23,942 - root - INFO - step: 97650 loss: 16.3124 memory: 44.58GiB(31.99%) tps: 83,239 tflops: 287.27 mfu: 29.05% global_avg_ntp_loss: 2.4877 global_avg_mtp_loss: 13.8246 +[titan] 2025-07-10 11:04:23,942 - root - INFO - lr: 3.0375e-05 gnorm: 1.05 [21:29:19< 0:31:01] +[titan] 2025-07-10 11:04:27,875 - root - INFO - step: 97655 loss: 16.2821 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.7920 +[titan] 2025-07-10 11:04:27,875 - root - INFO - lr: 3.0374e-05 gnorm: 1.03 [21:29:23< 0:30:57] +[titan] 2025-07-10 11:04:31,798 - root - INFO - step: 97660 loss: 15.9001 memory: 44.58GiB(31.99%) tps: 83,535 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4273 global_avg_mtp_loss: 13.4728 +[titan] 2025-07-10 11:04:31,798 - root - INFO - lr: 3.0372e-05 gnorm: 1.09 [21:29:27< 0:30:53] +[titan] 2025-07-10 11:04:35,747 - root - INFO - step: 97665 loss: 16.0201 memory: 44.58GiB(31.99%) tps: 82,979 tflops: 286.37 mfu: 28.96% global_avg_ntp_loss: 2.4441 global_avg_mtp_loss: 13.5760 +[titan] 2025-07-10 11:04:35,748 - root - INFO - lr: 3.0370e-05 gnorm: 1.05 [21:29:31< 0:30:49] +[titan] 2025-07-10 11:04:39,702 - root - INFO - step: 97670 loss: 16.1682 memory: 44.58GiB(31.99%) tps: 82,872 tflops: 286.00 mfu: 28.92% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.7049 +[titan] 2025-07-10 11:04:39,702 - root - INFO - lr: 3.0369e-05 gnorm: 1.01 [21:29:35< 0:30:45] +[titan] 2025-07-10 11:04:43,693 - root - INFO - step: 97675 loss: 16.3444 memory: 44.58GiB(31.99%) tps: 82,104 tflops: 283.35 mfu: 28.65% global_avg_ntp_loss: 2.5061 global_avg_mtp_loss: 13.8384 +[titan] 2025-07-10 11:04:43,694 - root - INFO - lr: 3.0367e-05 gnorm: 1.10 [21:29:39< 0:30:41] +[titan] 2025-07-10 11:04:47,594 - root - INFO - step: 97680 loss: 16.1344 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6743 +[titan] 2025-07-10 11:04:47,594 - root - INFO - lr: 3.0366e-05 gnorm: 1.08 [21:29:43< 0:30:37] +[titan] 2025-07-10 11:04:51,496 - root - INFO - step: 97685 loss: 16.4017 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.8908 +[titan] 2025-07-10 11:04:51,496 - root - INFO - lr: 3.0364e-05 gnorm: 1.08 [21:29:47< 0:30:33] +[titan] 2025-07-10 11:04:55,445 - root - INFO - step: 97690 loss: 16.0783 memory: 44.58GiB(31.99%) tps: 82,985 tflops: 286.39 mfu: 28.96% global_avg_ntp_loss: 2.4464 global_avg_mtp_loss: 13.6318 +[titan] 2025-07-10 11:04:55,445 - root - INFO - lr: 3.0363e-05 gnorm: 1.07 [21:29:50< 0:30:30] +[titan] 2025-07-10 11:04:59,359 - root - INFO - step: 97695 loss: 15.9388 memory: 44.58GiB(31.99%) tps: 83,734 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4264 global_avg_mtp_loss: 13.5124 +[titan] 2025-07-10 11:04:59,359 - root - INFO - lr: 3.0361e-05 gnorm: 1.16 [21:29:54< 0:30:26] +[titan] 2025-07-10 11:05:02,492 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:05:03,287 - root - INFO - step: 97700 loss: 16.3121 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.8077 +[titan] 2025-07-10 11:05:03,287 - root - INFO - lr: 3.0359e-05 gnorm: 1.04 [21:29:58< 0:30:22] +[titan] 2025-07-10 11:05:07,233 - root - INFO - step: 97705 loss: 16.1511 memory: 44.58GiB(31.99%) tps: 83,038 tflops: 286.58 mfu: 28.98% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.6804 +[titan] 2025-07-10 11:05:07,234 - root - INFO - lr: 3.0358e-05 gnorm: 1.03 [21:30:02< 0:30:18] +[titan] 2025-07-10 11:05:11,156 - root - INFO - step: 97710 loss: 16.1231 memory: 44.58GiB(31.99%) tps: 83,553 tflops: 288.35 mfu: 29.16% global_avg_ntp_loss: 2.4670 global_avg_mtp_loss: 13.6561 +[titan] 2025-07-10 11:05:11,156 - root - INFO - lr: 3.0356e-05 gnorm: 1.09 [21:30:06< 0:30:14] +[titan] 2025-07-10 11:05:15,080 - root - INFO - step: 97715 loss: 16.1452 memory: 44.58GiB(31.99%) tps: 83,500 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.4499 global_avg_mtp_loss: 13.6953 +[titan] 2025-07-10 11:05:15,081 - root - INFO - lr: 3.0355e-05 gnorm: 1.01 [21:30:10< 0:30:10] +[titan] 2025-07-10 11:05:18,983 - root - INFO - step: 97720 loss: 16.2112 memory: 44.58GiB(31.99%) tps: 83,965 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4836 global_avg_mtp_loss: 13.7276 +[titan] 2025-07-10 11:05:18,984 - root - INFO - lr: 3.0353e-05 gnorm: 1.07 [21:30:14< 0:30:06] +[titan] 2025-07-10 11:05:22,900 - root - INFO - step: 97725 loss: 16.0581 memory: 44.58GiB(31.99%) tps: 83,669 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.6206 +[titan] 2025-07-10 11:05:22,900 - root - INFO - lr: 3.0352e-05 gnorm: 1.08 [21:30:18< 0:30:02] +[titan] 2025-07-10 11:05:26,824 - root - INFO - step: 97730 loss: 16.0330 memory: 44.58GiB(31.99%) tps: 83,528 tflops: 288.27 mfu: 29.15% global_avg_ntp_loss: 2.4298 global_avg_mtp_loss: 13.6032 +[titan] 2025-07-10 11:05:26,824 - root - INFO - lr: 3.0350e-05 gnorm: 1.07 [21:30:22< 0:29:58] +[titan] 2025-07-10 11:05:30,728 - root - INFO - step: 97735 loss: 16.0272 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4376 global_avg_mtp_loss: 13.5896 +[titan] 2025-07-10 11:05:30,729 - root - INFO - lr: 3.0349e-05 gnorm: 1.10 [21:30:26< 0:29:54] +[titan] 2025-07-10 11:05:34,630 - root - INFO - step: 97740 loss: 16.2846 memory: 44.58GiB(31.99%) tps: 83,994 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.8026 +[titan] 2025-07-10 11:05:34,630 - root - INFO - lr: 3.0347e-05 gnorm: 1.07 [21:30:30< 0:29:50] +[titan] 2025-07-10 11:05:38,603 - root - INFO - step: 97745 loss: 16.1740 memory: 44.58GiB(31.99%) tps: 82,483 tflops: 284.66 mfu: 28.78% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.6942 +[titan] 2025-07-10 11:05:38,603 - root - INFO - lr: 3.0345e-05 gnorm: 1.03 [21:30:34< 0:29:46] +[titan] 2025-07-10 11:05:41,718 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:05:42,530 - root - INFO - step: 97750 loss: 15.8075 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.3982 global_avg_mtp_loss: 13.4092 +[titan] 2025-07-10 11:05:42,530 - root - INFO - lr: 3.0344e-05 gnorm: 1.08 [21:30:38< 0:29:42] +[titan] 2025-07-10 11:05:46,477 - root - INFO - step: 97755 loss: 16.2826 memory: 44.58GiB(31.99%) tps: 83,036 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4812 global_avg_mtp_loss: 13.8015 +[titan] 2025-07-10 11:05:46,477 - root - INFO - lr: 3.0342e-05 gnorm: 1.04 [21:30:42< 0:29:38] +[titan] 2025-07-10 11:05:50,474 - root - INFO - step: 97760 loss: 16.3246 memory: 44.58GiB(31.99%) tps: 81,988 tflops: 282.95 mfu: 28.61% global_avg_ntp_loss: 2.5005 global_avg_mtp_loss: 13.8241 +[titan] 2025-07-10 11:05:50,474 - root - INFO - lr: 3.0341e-05 gnorm: 1.08 [21:30:46< 0:29:34] +[titan] 2025-07-10 11:05:54,368 - root - INFO - step: 97765 loss: 16.0715 memory: 44.58GiB(31.99%) tps: 84,154 tflops: 290.43 mfu: 29.37% global_avg_ntp_loss: 2.4387 global_avg_mtp_loss: 13.6328 +[titan] 2025-07-10 11:05:54,368 - root - INFO - lr: 3.0339e-05 gnorm: 1.05 [21:30:49< 0:29:30] +[titan] 2025-07-10 11:05:58,294 - root - INFO - step: 97770 loss: 16.3411 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.5041 global_avg_mtp_loss: 13.8370 +[titan] 2025-07-10 11:05:58,295 - root - INFO - lr: 3.0338e-05 gnorm: 1.06 [21:30:53< 0:29:26] +[titan] 2025-07-10 11:06:02,240 - root - INFO - step: 97775 loss: 16.1663 memory: 44.58GiB(31.99%) tps: 83,053 tflops: 286.63 mfu: 28.98% global_avg_ntp_loss: 2.4479 global_avg_mtp_loss: 13.7184 +[titan] 2025-07-10 11:06:02,241 - root - INFO - lr: 3.0336e-05 gnorm: 1.15 [21:30:57< 0:29:22] +[titan] 2025-07-10 11:06:06,157 - root - INFO - step: 97780 loss: 16.0622 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4530 global_avg_mtp_loss: 13.6092 +[titan] 2025-07-10 11:06:06,157 - root - INFO - lr: 3.0335e-05 gnorm: 1.08 [21:31:01< 0:29:18] +[titan] 2025-07-10 11:06:10,069 - root - INFO - step: 97785 loss: 16.2881 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4917 global_avg_mtp_loss: 13.7964 +[titan] 2025-07-10 11:06:10,069 - root - INFO - lr: 3.0333e-05 gnorm: 1.05 [21:31:05< 0:29:14] +[titan] 2025-07-10 11:06:14,076 - root - INFO - step: 97790 loss: 16.1601 memory: 44.58GiB(31.99%) tps: 81,778 tflops: 282.23 mfu: 28.54% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6940 +[titan] 2025-07-10 11:06:14,076 - root - INFO - lr: 3.0332e-05 gnorm: 1.06 [21:31:09< 0:29:10] +[titan] 2025-07-10 11:06:15,794 - root - INFO - Dumping profiler traces at step 97792 +[titan] 2025-07-10 11:06:15,829 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 11:06:18,223 - root - INFO - step: 97795 loss: 16.2831 memory: 44.58GiB(31.99%) tps: 79,026 tflops: 272.73 mfu: 27.58% global_avg_ntp_loss: 2.5076 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 11:06:18,223 - root - INFO - lr: 3.0330e-05 gnorm: 1.06 [21:31:13< 0:29:06] +[titan] 2025-07-10 11:06:21,362 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:06:22,150 - root - INFO - step: 97800 loss: 15.9880 memory: 44.58GiB(31.99%) tps: 83,442 tflops: 287.97 mfu: 29.12% global_avg_ntp_loss: 2.4359 global_avg_mtp_loss: 13.5521 +[titan] 2025-07-10 11:06:22,151 - root - INFO - lr: 3.0329e-05 gnorm: 1.04 [21:31:17< 0:29:02] +[titan] 2025-07-10 11:06:26,060 - root - INFO - step: 97805 loss: 16.0873 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4507 global_avg_mtp_loss: 13.6366 +[titan] 2025-07-10 11:06:26,060 - root - INFO - lr: 3.0327e-05 gnorm: 1.06 [21:31:21< 0:28:58] +[titan] 2025-07-10 11:06:30,009 - root - INFO - step: 97810 loss: 16.3420 memory: 44.58GiB(31.99%) tps: 82,988 tflops: 286.41 mfu: 28.96% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8444 +[titan] 2025-07-10 11:06:30,009 - root - INFO - lr: 3.0326e-05 gnorm: 1.07 [21:31:25< 0:28:54] +[titan] 2025-07-10 11:06:33,939 - root - INFO - step: 97815 loss: 16.3749 memory: 44.58GiB(31.99%) tps: 83,387 tflops: 287.78 mfu: 29.10% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.8864 +[titan] 2025-07-10 11:06:33,939 - root - INFO - lr: 3.0324e-05 gnorm: 1.20 [21:31:29< 0:28:50] +[titan] 2025-07-10 11:06:37,902 - root - INFO - step: 97820 loss: 16.1193 memory: 44.58GiB(31.99%) tps: 82,704 tflops: 285.43 mfu: 28.86% global_avg_ntp_loss: 2.4672 global_avg_mtp_loss: 13.6521 +[titan] 2025-07-10 11:06:37,902 - root - INFO - lr: 3.0323e-05 gnorm: 1.15 [21:31:33< 0:28:47] +[titan] 2025-07-10 11:06:41,861 - root - INFO - step: 97825 loss: 16.0085 memory: 44.58GiB(31.99%) tps: 82,760 tflops: 285.62 mfu: 28.88% global_avg_ntp_loss: 2.4349 global_avg_mtp_loss: 13.5736 +[titan] 2025-07-10 11:06:41,862 - root - INFO - lr: 3.0321e-05 gnorm: 1.06 [21:31:37< 0:28:43] +[titan] 2025-07-10 11:06:45,772 - root - INFO - step: 97830 loss: 16.3127 memory: 44.58GiB(31.99%) tps: 83,798 tflops: 289.20 mfu: 29.24% global_avg_ntp_loss: 2.4957 global_avg_mtp_loss: 13.8170 +[titan] 2025-07-10 11:06:45,772 - root - INFO - lr: 3.0320e-05 gnorm: 1.05 [21:31:41< 0:28:39] +[titan] 2025-07-10 11:06:49,684 - root - INFO - step: 97835 loss: 16.1794 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4698 global_avg_mtp_loss: 13.7095 +[titan] 2025-07-10 11:06:49,684 - root - INFO - lr: 3.0318e-05 gnorm: 1.08 [21:31:45< 0:28:35] +[titan] 2025-07-10 11:06:53,612 - root - INFO - step: 97840 loss: 16.3966 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.8745 +[titan] 2025-07-10 11:06:53,612 - root - INFO - lr: 3.0317e-05 gnorm: 1.03 [21:31:49< 0:28:31] +[titan] 2025-07-10 11:06:57,532 - root - INFO - step: 97845 loss: 16.4179 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5114 global_avg_mtp_loss: 13.9064 +[titan] 2025-07-10 11:06:57,532 - root - INFO - lr: 3.0316e-05 gnorm: 1.05 [21:31:53< 0:28:27] +[titan] 2025-07-10 11:07:00,664 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:07:01,458 - root - INFO - step: 97850 loss: 16.3883 memory: 44.58GiB(31.99%) tps: 83,477 tflops: 288.09 mfu: 29.13% global_avg_ntp_loss: 2.4954 global_avg_mtp_loss: 13.8930 +[titan] 2025-07-10 11:07:01,458 - root - INFO - lr: 3.0314e-05 gnorm: 1.14 [21:31:56< 0:28:23] +[titan] 2025-07-10 11:07:05,381 - root - INFO - step: 97855 loss: 16.3171 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4842 global_avg_mtp_loss: 13.8329 +[titan] 2025-07-10 11:07:05,382 - root - INFO - lr: 3.0313e-05 gnorm: 1.05 [21:32:00< 0:28:19] +[titan] 2025-07-10 11:07:09,275 - root - INFO - step: 97860 loss: 16.2785 memory: 44.58GiB(31.99%) tps: 84,173 tflops: 290.49 mfu: 29.37% global_avg_ntp_loss: 2.4903 global_avg_mtp_loss: 13.7882 +[titan] 2025-07-10 11:07:09,275 - root - INFO - lr: 3.0311e-05 gnorm: 1.05 [21:32:04< 0:28:15] +[titan] 2025-07-10 11:07:13,189 - root - INFO - step: 97865 loss: 16.1935 memory: 44.58GiB(31.99%) tps: 83,711 tflops: 288.90 mfu: 29.21% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7109 +[titan] 2025-07-10 11:07:13,190 - root - INFO - lr: 3.0310e-05 gnorm: 1.24 [21:32:08< 0:28:11] +[titan] 2025-07-10 11:07:17,092 - root - INFO - step: 97870 loss: 16.0539 memory: 44.58GiB(31.99%) tps: 83,980 tflops: 289.83 mfu: 29.31% global_avg_ntp_loss: 2.4455 global_avg_mtp_loss: 13.6084 +[titan] 2025-07-10 11:07:17,092 - root - INFO - lr: 3.0308e-05 gnorm: 1.08 [21:32:12< 0:28:07] +[titan] 2025-07-10 11:07:20,987 - root - INFO - step: 97875 loss: 16.1339 memory: 44.58GiB(31.99%) tps: 84,127 tflops: 290.34 mfu: 29.36% global_avg_ntp_loss: 2.4551 global_avg_mtp_loss: 13.6789 +[titan] 2025-07-10 11:07:20,987 - root - INFO - lr: 3.0307e-05 gnorm: 1.05 [21:32:16< 0:28:03] +[titan] 2025-07-10 11:07:24,933 - root - INFO - step: 97880 loss: 16.5894 memory: 44.58GiB(31.99%) tps: 83,050 tflops: 286.62 mfu: 28.98% global_avg_ntp_loss: 2.5356 global_avg_mtp_loss: 14.0538 +[titan] 2025-07-10 11:07:24,933 - root - INFO - lr: 3.0305e-05 gnorm: 1.06 [21:32:20< 0:27:59] +[titan] 2025-07-10 11:07:28,840 - root - INFO - step: 97885 loss: 16.0313 memory: 44.58GiB(31.99%) tps: 83,882 tflops: 289.49 mfu: 29.27% global_avg_ntp_loss: 2.4441 global_avg_mtp_loss: 13.5872 +[titan] 2025-07-10 11:07:28,840 - root - INFO - lr: 3.0304e-05 gnorm: 1.07 [21:32:24< 0:27:55] +[titan] 2025-07-10 11:07:32,773 - root - INFO - step: 97890 loss: 16.2378 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7582 +[titan] 2025-07-10 11:07:32,773 - root - INFO - lr: 3.0303e-05 gnorm: 1.04 [21:32:28< 0:27:51] +[titan] 2025-07-10 11:07:36,686 - root - INFO - step: 97895 loss: 15.9641 memory: 44.58GiB(31.99%) tps: 83,751 tflops: 289.04 mfu: 29.23% global_avg_ntp_loss: 2.4301 global_avg_mtp_loss: 13.5340 +[titan] 2025-07-10 11:07:36,686 - root - INFO - lr: 3.0301e-05 gnorm: 1.47 [21:32:32< 0:27:47] +[titan] 2025-07-10 11:07:39,829 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:07:40,618 - root - INFO - step: 97900 loss: 16.4361 memory: 44.58GiB(31.99%) tps: 83,332 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.5221 global_avg_mtp_loss: 13.9140 +[titan] 2025-07-10 11:07:40,619 - root - INFO - lr: 3.0300e-05 gnorm: 1.05 [21:32:36< 0:27:43] +[titan] 2025-07-10 11:07:44,558 - root - INFO - step: 97905 loss: 16.0236 memory: 44.58GiB(31.99%) tps: 83,182 tflops: 287.08 mfu: 29.03% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.5814 +[titan] 2025-07-10 11:07:44,558 - root - INFO - lr: 3.0298e-05 gnorm: 1.07 [21:32:40< 0:27:39] +[titan] 2025-07-10 11:07:48,502 - root - INFO - step: 97910 loss: 16.3714 memory: 44.58GiB(31.99%) tps: 83,087 tflops: 286.75 mfu: 28.99% global_avg_ntp_loss: 2.4970 global_avg_mtp_loss: 13.8744 +[titan] 2025-07-10 11:07:48,503 - root - INFO - lr: 3.0297e-05 gnorm: 1.03 [21:32:44< 0:27:35] +[titan] 2025-07-10 11:07:52,419 - root - INFO - step: 97915 loss: 16.0520 memory: 44.58GiB(31.99%) tps: 83,670 tflops: 288.76 mfu: 29.20% global_avg_ntp_loss: 2.4355 global_avg_mtp_loss: 13.6165 +[titan] 2025-07-10 11:07:52,419 - root - INFO - lr: 3.0295e-05 gnorm: 1.03 [21:32:47< 0:27:31] +[titan] 2025-07-10 11:07:56,366 - root - INFO - step: 97920 loss: 16.2035 memory: 44.58GiB(31.99%) tps: 83,035 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4677 global_avg_mtp_loss: 13.7357 +[titan] 2025-07-10 11:07:56,366 - root - INFO - lr: 3.0294e-05 gnorm: 1.05 [21:32:51< 0:27:27] +[titan] 2025-07-10 11:08:00,280 - root - INFO - step: 97925 loss: 16.3739 memory: 44.58GiB(31.99%) tps: 83,721 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4980 global_avg_mtp_loss: 13.8759 +[titan] 2025-07-10 11:08:00,281 - root - INFO - lr: 3.0293e-05 gnorm: 1.03 [21:32:55< 0:27:23] +[titan] 2025-07-10 11:08:04,204 - root - INFO - step: 97930 loss: 15.9956 memory: 44.58GiB(31.99%) tps: 83,522 tflops: 288.25 mfu: 29.15% global_avg_ntp_loss: 2.4283 global_avg_mtp_loss: 13.5673 +[titan] 2025-07-10 11:08:04,204 - root - INFO - lr: 3.0291e-05 gnorm: 1.20 [21:32:59< 0:27:19] +[titan] 2025-07-10 11:08:08,129 - root - INFO - step: 97935 loss: 16.1168 memory: 44.58GiB(31.99%) tps: 83,485 tflops: 288.12 mfu: 29.13% global_avg_ntp_loss: 2.4647 global_avg_mtp_loss: 13.6521 +[titan] 2025-07-10 11:08:08,130 - root - INFO - lr: 3.0290e-05 gnorm: 1.09 [21:33:03< 0:27:15] +[titan] 2025-07-10 11:08:12,051 - root - INFO - step: 97940 loss: 16.1841 memory: 44.58GiB(31.99%) tps: 83,569 tflops: 288.41 mfu: 29.16% global_avg_ntp_loss: 2.4666 global_avg_mtp_loss: 13.7175 +[titan] 2025-07-10 11:08:12,051 - root - INFO - lr: 3.0288e-05 gnorm: 1.04 [21:33:07< 0:27:11] +[titan] 2025-07-10 11:08:15,976 - root - INFO - step: 97945 loss: 16.1289 memory: 44.58GiB(31.99%) tps: 83,482 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4660 global_avg_mtp_loss: 13.6629 +[titan] 2025-07-10 11:08:15,977 - root - INFO - lr: 3.0287e-05 gnorm: 1.07 [21:33:11< 0:27:07] +[titan] 2025-07-10 11:08:19,133 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:08:19,923 - root - INFO - step: 97950 loss: 16.2168 memory: 44.58GiB(31.99%) tps: 83,040 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7422 +[titan] 2025-07-10 11:08:19,923 - root - INFO - lr: 3.0286e-05 gnorm: 1.04 [21:33:15< 0:27:03] +[titan] 2025-07-10 11:08:23,823 - root - INFO - step: 97955 loss: 16.1227 memory: 44.58GiB(31.99%) tps: 84,033 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4455 global_avg_mtp_loss: 13.6772 +[titan] 2025-07-10 11:08:23,823 - root - INFO - lr: 3.0284e-05 gnorm: 1.08 [21:33:19< 0:27:00] +[titan] 2025-07-10 11:08:27,722 - root - INFO - step: 97960 loss: 16.2157 memory: 44.58GiB(31.99%) tps: 84,048 tflops: 290.06 mfu: 29.33% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7282 +[titan] 2025-07-10 11:08:27,722 - root - INFO - lr: 3.0283e-05 gnorm: 1.10 [21:33:23< 0:26:56] +[titan] 2025-07-10 11:08:31,668 - root - INFO - step: 97965 loss: 16.0129 memory: 44.58GiB(31.99%) tps: 83,042 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4293 global_avg_mtp_loss: 13.5836 +[titan] 2025-07-10 11:08:31,669 - root - INFO - lr: 3.0281e-05 gnorm: 1.09 [21:33:27< 0:26:52] +[titan] 2025-07-10 11:08:35,585 - root - INFO - step: 97970 loss: 16.2496 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4867 global_avg_mtp_loss: 13.7629 +[titan] 2025-07-10 11:08:35,586 - root - INFO - lr: 3.0280e-05 gnorm: 1.01 [21:33:31< 0:26:48] +[titan] 2025-07-10 11:08:39,486 - root - INFO - step: 97975 loss: 16.2117 memory: 44.58GiB(31.99%) tps: 84,011 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4740 global_avg_mtp_loss: 13.7378 +[titan] 2025-07-10 11:08:39,486 - root - INFO - lr: 3.0279e-05 gnorm: 1.11 [21:33:34< 0:26:44] +[titan] 2025-07-10 11:08:43,408 - root - INFO - step: 97980 loss: 16.0692 memory: 44.58GiB(31.99%) tps: 83,559 tflops: 288.37 mfu: 29.16% global_avg_ntp_loss: 2.4554 global_avg_mtp_loss: 13.6139 +[titan] 2025-07-10 11:08:43,408 - root - INFO - lr: 3.0277e-05 gnorm: 1.05 [21:33:38< 0:26:40] +[titan] 2025-07-10 11:08:47,334 - root - INFO - step: 97985 loss: 16.0299 memory: 44.58GiB(31.99%) tps: 83,481 tflops: 288.11 mfu: 29.13% global_avg_ntp_loss: 2.4392 global_avg_mtp_loss: 13.5907 +[titan] 2025-07-10 11:08:47,334 - root - INFO - lr: 3.0276e-05 gnorm: 1.07 [21:33:42< 0:26:36] +[titan] 2025-07-10 11:08:51,274 - root - INFO - step: 97990 loss: 16.4195 memory: 44.58GiB(31.99%) tps: 83,167 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.5045 global_avg_mtp_loss: 13.9150 +[titan] 2025-07-10 11:08:51,274 - root - INFO - lr: 3.0275e-05 gnorm: 1.03 [21:33:46< 0:26:32] +[titan] 2025-07-10 11:08:55,215 - root - INFO - step: 97995 loss: 15.9655 memory: 44.58GiB(31.99%) tps: 83,166 tflops: 287.02 mfu: 29.02% global_avg_ntp_loss: 2.4289 global_avg_mtp_loss: 13.5366 +[titan] 2025-07-10 11:08:55,215 - root - INFO - lr: 3.0273e-05 gnorm: 1.07 [21:33:50< 0:26:28] +[titan] 2025-07-10 11:08:58,352 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:08:59,141 - root - INFO - step: 98000 loss: 16.1103 memory: 44.58GiB(31.99%) tps: 83,475 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4592 global_avg_mtp_loss: 13.6511 +[titan] 2025-07-10 11:08:59,141 - root - INFO - lr: 3.0272e-05 gnorm: 1.04 [21:33:54< 0:26:24] +[titan] 2025-07-10 11:09:03,041 - root - INFO - step: 98005 loss: 16.3155 memory: 44.58GiB(31.99%) tps: 84,013 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8210 +[titan] 2025-07-10 11:09:03,042 - root - INFO - lr: 3.0270e-05 gnorm: 1.02 [21:33:58< 0:26:20] +[titan] 2025-07-10 11:09:06,951 - root - INFO - step: 98010 loss: 16.0705 memory: 44.58GiB(31.99%) tps: 83,817 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4508 global_avg_mtp_loss: 13.6197 +[titan] 2025-07-10 11:09:06,952 - root - INFO - lr: 3.0269e-05 gnorm: 1.06 [21:34:02< 0:26:16] +[titan] 2025-07-10 11:09:10,860 - root - INFO - step: 98015 loss: 16.3214 memory: 44.58GiB(31.99%) tps: 83,839 tflops: 289.34 mfu: 29.26% global_avg_ntp_loss: 2.4896 global_avg_mtp_loss: 13.8318 +[titan] 2025-07-10 11:09:10,860 - root - INFO - lr: 3.0268e-05 gnorm: 1.06 [21:34:06< 0:26:12] +[titan] 2025-07-10 11:09:14,787 - root - INFO - step: 98020 loss: 15.9532 memory: 44.58GiB(31.99%) tps: 83,445 tflops: 287.98 mfu: 29.12% global_avg_ntp_loss: 2.4395 global_avg_mtp_loss: 13.5137 +[titan] 2025-07-10 11:09:14,788 - root - INFO - lr: 3.0266e-05 gnorm: 1.02 [21:34:10< 0:26:08] +[titan] 2025-07-10 11:09:18,720 - root - INFO - step: 98025 loss: 16.3031 memory: 44.58GiB(31.99%) tps: 83,327 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4840 global_avg_mtp_loss: 13.8191 +[titan] 2025-07-10 11:09:18,721 - root - INFO - lr: 3.0265e-05 gnorm: 1.09 [21:34:14< 0:26:04] +[titan] 2025-07-10 11:09:22,628 - root - INFO - step: 98030 loss: 16.2755 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7938 +[titan] 2025-07-10 11:09:22,628 - root - INFO - lr: 3.0264e-05 gnorm: 1.08 [21:34:18< 0:26:00] +[titan] 2025-07-10 11:09:26,573 - root - INFO - step: 98035 loss: 16.3843 memory: 44.58GiB(31.99%) tps: 83,072 tflops: 286.69 mfu: 28.99% global_avg_ntp_loss: 2.5104 global_avg_mtp_loss: 13.8739 +[titan] 2025-07-10 11:09:26,573 - root - INFO - lr: 3.0262e-05 gnorm: 1.11 [21:34:22< 0:25:56] +[titan] 2025-07-10 11:09:30,477 - root - INFO - step: 98040 loss: 16.1236 memory: 44.58GiB(31.99%) tps: 83,929 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4519 global_avg_mtp_loss: 13.6717 +[titan] 2025-07-10 11:09:30,478 - root - INFO - lr: 3.0261e-05 gnorm: 1.00 [21:34:25< 0:25:52] +[titan] 2025-07-10 11:09:34,450 - root - INFO - step: 98045 loss: 16.1405 memory: 44.58GiB(31.99%) tps: 82,501 tflops: 284.72 mfu: 28.79% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.6958 +[titan] 2025-07-10 11:09:34,450 - root - INFO - lr: 3.0260e-05 gnorm: 1.05 [21:34:29< 0:25:48] +[titan] 2025-07-10 11:09:37,572 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:09:38,384 - root - INFO - step: 98050 loss: 16.2689 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7897 +[titan] 2025-07-10 11:09:38,384 - root - INFO - lr: 3.0258e-05 gnorm: 1.06 [21:34:33< 0:25:44] +[titan] 2025-07-10 11:09:42,296 - root - INFO - step: 98055 loss: 16.1686 memory: 44.58GiB(31.99%) tps: 83,757 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.4717 global_avg_mtp_loss: 13.6969 +[titan] 2025-07-10 11:09:42,297 - root - INFO - lr: 3.0257e-05 gnorm: 1.10 [21:34:37< 0:25:40] +[titan] 2025-07-10 11:09:46,201 - root - INFO - step: 98060 loss: 16.2886 memory: 44.58GiB(31.99%) tps: 83,935 tflops: 289.67 mfu: 29.29% global_avg_ntp_loss: 2.4838 global_avg_mtp_loss: 13.8048 +[titan] 2025-07-10 11:09:46,201 - root - INFO - lr: 3.0256e-05 gnorm: 1.06 [21:34:41< 0:25:36] +[titan] 2025-07-10 11:09:50,113 - root - INFO - step: 98065 loss: 16.4102 memory: 44.58GiB(31.99%) tps: 83,777 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4994 global_avg_mtp_loss: 13.9108 +[titan] 2025-07-10 11:09:50,113 - root - INFO - lr: 3.0254e-05 gnorm: 1.06 [21:34:45< 0:25:32] +[titan] 2025-07-10 11:09:54,017 - root - INFO - step: 98070 loss: 16.2228 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4829 global_avg_mtp_loss: 13.7399 +[titan] 2025-07-10 11:09:54,017 - root - INFO - lr: 3.0253e-05 gnorm: 1.01 [21:34:49< 0:25:28] +[titan] 2025-07-10 11:09:57,942 - root - INFO - step: 98075 loss: 16.1467 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4628 global_avg_mtp_loss: 13.6839 +[titan] 2025-07-10 11:09:57,942 - root - INFO - lr: 3.0252e-05 gnorm: 1.10 [21:34:53< 0:25:24] +[titan] 2025-07-10 11:10:01,854 - root - INFO - step: 98080 loss: 16.5048 memory: 44.58GiB(31.99%) tps: 83,756 tflops: 289.06 mfu: 29.23% global_avg_ntp_loss: 2.5410 global_avg_mtp_loss: 13.9638 +[titan] 2025-07-10 11:10:01,855 - root - INFO - lr: 3.0250e-05 gnorm: 1.06 [21:34:57< 0:25:20] +[titan] 2025-07-10 11:10:05,784 - root - INFO - step: 98085 loss: 16.1749 memory: 44.58GiB(31.99%) tps: 83,408 tflops: 287.85 mfu: 29.11% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.7088 +[titan] 2025-07-10 11:10:05,784 - root - INFO - lr: 3.0249e-05 gnorm: 1.08 [21:35:01< 0:25:17] +[titan] 2025-07-10 11:10:09,697 - root - INFO - step: 98090 loss: 16.1662 memory: 44.58GiB(31.99%) tps: 83,745 tflops: 289.02 mfu: 29.22% global_avg_ntp_loss: 2.4655 global_avg_mtp_loss: 13.7007 +[titan] 2025-07-10 11:10:09,697 - root - INFO - lr: 3.0248e-05 gnorm: 1.13 [21:35:05< 0:25:13] +[titan] 2025-07-10 11:10:13,597 - root - INFO - step: 98095 loss: 16.2752 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.4792 global_avg_mtp_loss: 13.7960 +[titan] 2025-07-10 11:10:13,597 - root - INFO - lr: 3.0247e-05 gnorm: 1.09 [21:35:09< 0:25:09] +[titan] 2025-07-10 11:10:16,731 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:10:17,520 - root - INFO - step: 98100 loss: 16.2077 memory: 44.58GiB(31.99%) tps: 83,524 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4728 global_avg_mtp_loss: 13.7349 +[titan] 2025-07-10 11:10:17,520 - root - INFO - lr: 3.0245e-05 gnorm: 1.03 [21:35:13< 0:25:05] +[titan] 2025-07-10 11:10:21,432 - root - INFO - step: 98105 loss: 15.9179 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4184 global_avg_mtp_loss: 13.4996 +[titan] 2025-07-10 11:10:21,432 - root - INFO - lr: 3.0244e-05 gnorm: 1.08 [21:35:16< 0:25:01] +[titan] 2025-07-10 11:10:25,349 - root - INFO - step: 98110 loss: 16.3378 memory: 44.58GiB(31.99%) tps: 83,657 tflops: 288.71 mfu: 29.19% global_avg_ntp_loss: 2.4950 global_avg_mtp_loss: 13.8427 +[titan] 2025-07-10 11:10:25,350 - root - INFO - lr: 3.0243e-05 gnorm: 1.04 [21:35:20< 0:24:57] +[titan] 2025-07-10 11:10:29,313 - root - INFO - step: 98115 loss: 16.2160 memory: 44.58GiB(31.99%) tps: 82,670 tflops: 285.31 mfu: 28.85% global_avg_ntp_loss: 2.4860 global_avg_mtp_loss: 13.7300 +[titan] 2025-07-10 11:10:29,314 - root - INFO - lr: 3.0241e-05 gnorm: 1.11 [21:35:24< 0:24:53] +[titan] 2025-07-10 11:10:33,214 - root - INFO - step: 98120 loss: 16.3415 memory: 44.58GiB(31.99%) tps: 84,007 tflops: 289.92 mfu: 29.31% global_avg_ntp_loss: 2.5021 global_avg_mtp_loss: 13.8394 +[titan] 2025-07-10 11:10:33,215 - root - INFO - lr: 3.0240e-05 gnorm: 1.06 [21:35:28< 0:24:49] +[titan] 2025-07-10 11:10:37,123 - root - INFO - step: 98125 loss: 16.0581 memory: 44.58GiB(31.99%) tps: 83,848 tflops: 289.37 mfu: 29.26% global_avg_ntp_loss: 2.4523 global_avg_mtp_loss: 13.6057 +[titan] 2025-07-10 11:10:37,123 - root - INFO - lr: 3.0239e-05 gnorm: 1.12 [21:35:32< 0:24:45] +[titan] 2025-07-10 11:10:41,023 - root - INFO - step: 98130 loss: 16.4379 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.5131 global_avg_mtp_loss: 13.9248 +[titan] 2025-07-10 11:10:41,024 - root - INFO - lr: 3.0238e-05 gnorm: 1.03 [21:35:36< 0:24:41] +[titan] 2025-07-10 11:10:44,951 - root - INFO - step: 98135 loss: 16.2936 memory: 44.58GiB(31.99%) tps: 83,450 tflops: 288.00 mfu: 29.12% global_avg_ntp_loss: 2.4818 global_avg_mtp_loss: 13.8118 +[titan] 2025-07-10 11:10:44,951 - root - INFO - lr: 3.0236e-05 gnorm: 1.07 [21:35:40< 0:24:37] +[titan] 2025-07-10 11:10:48,868 - root - INFO - step: 98140 loss: 16.2182 memory: 44.58GiB(31.99%) tps: 83,664 tflops: 288.74 mfu: 29.20% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7382 +[titan] 2025-07-10 11:10:48,868 - root - INFO - lr: 3.0235e-05 gnorm: 1.05 [21:35:44< 0:24:33] +[titan] 2025-07-10 11:10:52,778 - root - INFO - step: 98145 loss: 16.0558 memory: 44.58GiB(31.99%) tps: 83,813 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4697 global_avg_mtp_loss: 13.5861 +[titan] 2025-07-10 11:10:52,778 - root - INFO - lr: 3.0234e-05 gnorm: 1.05 [21:35:48< 0:24:29] +[titan] 2025-07-10 11:10:55,938 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:10:56,732 - root - INFO - step: 98150 loss: 16.2890 memory: 44.58GiB(31.99%) tps: 82,876 tflops: 286.02 mfu: 28.92% global_avg_ntp_loss: 2.4996 global_avg_mtp_loss: 13.7893 +[titan] 2025-07-10 11:10:56,732 - root - INFO - lr: 3.0233e-05 gnorm: 1.07 [21:35:52< 0:24:25] +[titan] 2025-07-10 11:11:00,708 - root - INFO - step: 98155 loss: 16.0568 memory: 44.58GiB(31.99%) tps: 82,419 tflops: 284.44 mfu: 28.76% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6064 +[titan] 2025-07-10 11:11:00,708 - root - INFO - lr: 3.0231e-05 gnorm: 1.11 [21:35:56< 0:24:21] +[titan] 2025-07-10 11:11:04,669 - root - INFO - step: 98160 loss: 16.1278 memory: 44.58GiB(31.99%) tps: 82,732 tflops: 285.52 mfu: 28.87% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6627 +[titan] 2025-07-10 11:11:04,670 - root - INFO - lr: 3.0230e-05 gnorm: 1.08 [21:36:00< 0:24:17] +[titan] 2025-07-10 11:11:08,576 - root - INFO - step: 98165 loss: 16.0638 memory: 44.58GiB(31.99%) tps: 83,879 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4458 global_avg_mtp_loss: 13.6180 +[titan] 2025-07-10 11:11:08,577 - root - INFO - lr: 3.0229e-05 gnorm: 1.06 [21:36:04< 0:24:13] +[titan] 2025-07-10 11:11:12,557 - root - INFO - step: 98170 loss: 15.9581 memory: 44.58GiB(31.99%) tps: 82,335 tflops: 284.15 mfu: 28.73% global_avg_ntp_loss: 2.4366 global_avg_mtp_loss: 13.5215 +[titan] 2025-07-10 11:11:12,557 - root - INFO - lr: 3.0228e-05 gnorm: 1.10 [21:36:08< 0:24:09] +[titan] 2025-07-10 11:11:16,494 - root - INFO - step: 98175 loss: 16.3499 memory: 44.58GiB(31.99%) tps: 83,237 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.5027 global_avg_mtp_loss: 13.8472 +[titan] 2025-07-10 11:11:16,494 - root - INFO - lr: 3.0226e-05 gnorm: 1.03 [21:36:11< 0:24:05] +[titan] 2025-07-10 11:11:20,408 - root - INFO - step: 98180 loss: 15.8516 memory: 44.58GiB(31.99%) tps: 83,726 tflops: 288.95 mfu: 29.22% global_avg_ntp_loss: 2.4147 global_avg_mtp_loss: 13.4368 +[titan] 2025-07-10 11:11:20,408 - root - INFO - lr: 3.0225e-05 gnorm: 1.03 [21:36:15< 0:24:01] +[titan] 2025-07-10 11:11:24,330 - root - INFO - step: 98185 loss: 16.0599 memory: 44.58GiB(31.99%) tps: 83,561 tflops: 288.38 mfu: 29.16% global_avg_ntp_loss: 2.4424 global_avg_mtp_loss: 13.6174 +[titan] 2025-07-10 11:11:24,330 - root - INFO - lr: 3.0224e-05 gnorm: 1.08 [21:36:19< 0:23:57] +[titan] 2025-07-10 11:11:28,290 - root - INFO - step: 98190 loss: 16.1925 memory: 44.58GiB(31.99%) tps: 82,751 tflops: 285.59 mfu: 28.88% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.7252 +[titan] 2025-07-10 11:11:28,290 - root - INFO - lr: 3.0223e-05 gnorm: 1.06 [21:36:23< 0:23:53] +[titan] 2025-07-10 11:11:32,198 - root - INFO - step: 98195 loss: 16.0414 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.40 mfu: 29.26% global_avg_ntp_loss: 2.4312 global_avg_mtp_loss: 13.6102 +[titan] 2025-07-10 11:11:32,198 - root - INFO - lr: 3.0221e-05 gnorm: 1.06 [21:36:27< 0:23:49] +[titan] 2025-07-10 11:11:35,318 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:11:36,108 - root - INFO - step: 98200 loss: 16.2984 memory: 44.58GiB(31.99%) tps: 83,810 tflops: 289.24 mfu: 29.25% global_avg_ntp_loss: 2.4883 global_avg_mtp_loss: 13.8101 +[titan] 2025-07-10 11:11:36,108 - root - INFO - lr: 3.0220e-05 gnorm: 1.09 [21:36:31< 0:23:45] +[titan] 2025-07-10 11:11:40,042 - root - INFO - step: 98205 loss: 16.0936 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6354 +[titan] 2025-07-10 11:11:40,042 - root - INFO - lr: 3.0219e-05 gnorm: 1.03 [21:36:35< 0:23:41] +[titan] 2025-07-10 11:11:43,944 - root - INFO - step: 98210 loss: 16.2897 memory: 44.58GiB(31.99%) tps: 83,984 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4873 global_avg_mtp_loss: 13.8025 +[titan] 2025-07-10 11:11:43,945 - root - INFO - lr: 3.0218e-05 gnorm: 1.08 [21:36:39< 0:23:37] +[titan] 2025-07-10 11:11:47,874 - root - INFO - step: 98215 loss: 16.1586 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4592 global_avg_mtp_loss: 13.6994 +[titan] 2025-07-10 11:11:47,874 - root - INFO - lr: 3.0217e-05 gnorm: 1.11 [21:36:43< 0:23:34] +[titan] 2025-07-10 11:11:51,790 - root - INFO - step: 98220 loss: 15.9947 memory: 44.58GiB(31.99%) tps: 83,689 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4297 global_avg_mtp_loss: 13.5650 +[titan] 2025-07-10 11:11:51,790 - root - INFO - lr: 3.0215e-05 gnorm: 1.05 [21:36:47< 0:23:30] +[titan] 2025-07-10 11:11:55,710 - root - INFO - step: 98225 loss: 15.9610 memory: 44.58GiB(31.99%) tps: 83,610 tflops: 288.55 mfu: 29.18% global_avg_ntp_loss: 2.4280 global_avg_mtp_loss: 13.5330 +[titan] 2025-07-10 11:11:55,710 - root - INFO - lr: 3.0214e-05 gnorm: 1.07 [21:36:51< 0:23:26] +[titan] 2025-07-10 11:11:59,649 - root - INFO - step: 98230 loss: 16.3548 memory: 44.58GiB(31.99%) tps: 83,196 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8509 +[titan] 2025-07-10 11:11:59,649 - root - INFO - lr: 3.0213e-05 gnorm: 1.07 [21:36:55< 0:23:22] +[titan] 2025-07-10 11:12:03,573 - root - INFO - step: 98235 loss: 16.2088 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7398 +[titan] 2025-07-10 11:12:03,573 - root - INFO - lr: 3.0212e-05 gnorm: 1.04 [21:36:59< 0:23:18] +[titan] 2025-07-10 11:12:07,499 - root - INFO - step: 98240 loss: 16.1364 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4642 global_avg_mtp_loss: 13.6723 +[titan] 2025-07-10 11:12:07,500 - root - INFO - lr: 3.0210e-05 gnorm: 1.13 [21:37:02< 0:23:14] +[titan] 2025-07-10 11:12:11,409 - root - INFO - step: 98245 loss: 16.1609 memory: 44.58GiB(31.99%) tps: 83,821 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.6885 +[titan] 2025-07-10 11:12:11,409 - root - INFO - lr: 3.0209e-05 gnorm: 1.07 [21:37:06< 0:23:10] +[titan] 2025-07-10 11:12:14,561 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:12:15,359 - root - INFO - step: 98250 loss: 15.9750 memory: 44.58GiB(31.99%) tps: 82,960 tflops: 286.31 mfu: 28.95% global_avg_ntp_loss: 2.4326 global_avg_mtp_loss: 13.5424 +[titan] 2025-07-10 11:12:15,360 - root - INFO - lr: 3.0208e-05 gnorm: 1.07 [21:37:10< 0:23:06] +[titan] 2025-07-10 11:12:19,276 - root - INFO - step: 98255 loss: 16.5486 memory: 44.58GiB(31.99%) tps: 83,684 tflops: 288.81 mfu: 29.20% global_avg_ntp_loss: 2.5367 global_avg_mtp_loss: 14.0119 +[titan] 2025-07-10 11:12:19,276 - root - INFO - lr: 3.0207e-05 gnorm: 1.07 [21:37:14< 0:23:02] +[titan] 2025-07-10 11:12:23,183 - root - INFO - step: 98260 loss: 16.0743 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4378 global_avg_mtp_loss: 13.6366 +[titan] 2025-07-10 11:12:23,184 - root - INFO - lr: 3.0206e-05 gnorm: 1.10 [21:37:18< 0:22:58] +[titan] 2025-07-10 11:12:27,154 - root - INFO - step: 98265 loss: 16.2563 memory: 44.58GiB(31.99%) tps: 82,533 tflops: 284.84 mfu: 28.80% global_avg_ntp_loss: 2.4801 global_avg_mtp_loss: 13.7762 +[titan] 2025-07-10 11:12:27,154 - root - INFO - lr: 3.0205e-05 gnorm: 1.03 [21:37:22< 0:22:54] +[titan] 2025-07-10 11:12:31,057 - root - INFO - step: 98270 loss: 16.3982 memory: 44.58GiB(31.99%) tps: 83,975 tflops: 289.81 mfu: 29.30% global_avg_ntp_loss: 2.4993 global_avg_mtp_loss: 13.8988 +[titan] 2025-07-10 11:12:31,057 - root - INFO - lr: 3.0203e-05 gnorm: 1.07 [21:37:26< 0:22:50] +[titan] 2025-07-10 11:12:34,975 - root - INFO - step: 98275 loss: 16.1137 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.6500 +[titan] 2025-07-10 11:12:34,975 - root - INFO - lr: 3.0202e-05 gnorm: 1.06 [21:37:30< 0:22:46] +[titan] 2025-07-10 11:12:38,895 - root - INFO - step: 98280 loss: 15.9742 memory: 44.58GiB(31.99%) tps: 83,595 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4406 global_avg_mtp_loss: 13.5336 +[titan] 2025-07-10 11:12:38,896 - root - INFO - lr: 3.0201e-05 gnorm: 1.03 [21:37:34< 0:22:42] +[titan] 2025-07-10 11:12:42,815 - root - INFO - step: 98285 loss: 16.2723 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4779 global_avg_mtp_loss: 13.7945 +[titan] 2025-07-10 11:12:42,815 - root - INFO - lr: 3.0200e-05 gnorm: 1.01 [21:37:38< 0:22:38] +[titan] 2025-07-10 11:12:46,736 - root - INFO - step: 98290 loss: 15.9942 memory: 44.58GiB(31.99%) tps: 83,589 tflops: 288.48 mfu: 29.17% global_avg_ntp_loss: 2.4358 global_avg_mtp_loss: 13.5583 +[titan] 2025-07-10 11:12:46,736 - root - INFO - lr: 3.0199e-05 gnorm: 1.06 [21:37:42< 0:22:34] +[titan] 2025-07-10 11:12:50,674 - root - INFO - step: 98295 loss: 16.1478 memory: 44.58GiB(31.99%) tps: 83,208 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4608 global_avg_mtp_loss: 13.6871 +[titan] 2025-07-10 11:12:50,674 - root - INFO - lr: 3.0198e-05 gnorm: 1.12 [21:37:46< 0:22:30] +[titan] 2025-07-10 11:12:53,796 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:12:54,586 - root - INFO - step: 98300 loss: 16.3551 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4992 global_avg_mtp_loss: 13.8559 +[titan] 2025-07-10 11:12:54,586 - root - INFO - lr: 3.0196e-05 gnorm: 1.08 [21:37:50< 0:22:26] +[titan] 2025-07-10 11:12:57,968 - root - INFO - Dumping profiler traces at step 98304 +[titan] 2025-07-10 11:12:58,002 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 11:12:58,795 - root - INFO - step: 98305 loss: 16.0107 memory: 44.58GiB(31.99%) tps: 77,849 tflops: 268.67 mfu: 27.17% global_avg_ntp_loss: 2.4416 global_avg_mtp_loss: 13.5690 +[titan] 2025-07-10 11:12:58,796 - root - INFO - lr: 3.0195e-05 gnorm: 1.07 [21:37:54< 0:22:22] +[titan] 2025-07-10 11:13:02,717 - root - INFO - step: 98310 loss: 16.0731 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4557 global_avg_mtp_loss: 13.6174 +[titan] 2025-07-10 11:13:02,717 - root - INFO - lr: 3.0194e-05 gnorm: 1.06 [21:37:58< 0:22:18] +[titan] 2025-07-10 11:13:06,626 - root - INFO - step: 98315 loss: 16.2511 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4820 global_avg_mtp_loss: 13.7692 +[titan] 2025-07-10 11:13:06,627 - root - INFO - lr: 3.0193e-05 gnorm: 1.09 [21:38:02< 0:22:14] +[titan] 2025-07-10 11:13:10,531 - root - INFO - step: 98320 loss: 16.3177 memory: 44.58GiB(31.99%) tps: 83,930 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4916 global_avg_mtp_loss: 13.8261 +[titan] 2025-07-10 11:13:10,531 - root - INFO - lr: 3.0192e-05 gnorm: 1.05 [21:38:06< 0:22:10] +[titan] 2025-07-10 11:13:14,465 - root - INFO - step: 98325 loss: 16.2754 memory: 44.58GiB(31.99%) tps: 83,302 tflops: 287.49 mfu: 29.07% global_avg_ntp_loss: 2.4875 global_avg_mtp_loss: 13.7878 +[titan] 2025-07-10 11:13:14,465 - root - INFO - lr: 3.0191e-05 gnorm: 1.07 [21:38:09< 0:22:06] +[titan] 2025-07-10 11:13:18,361 - root - INFO - step: 98330 loss: 16.4921 memory: 44.58GiB(31.99%) tps: 84,105 tflops: 290.26 mfu: 29.35% global_avg_ntp_loss: 2.5331 global_avg_mtp_loss: 13.9590 +[titan] 2025-07-10 11:13:18,362 - root - INFO - lr: 3.0190e-05 gnorm: 1.09 [21:38:13< 0:22:02] +[titan] 2025-07-10 11:13:22,265 - root - INFO - step: 98335 loss: 16.1498 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 11:13:22,265 - root - INFO - lr: 3.0188e-05 gnorm: 1.02 [21:38:17< 0:21:58] +[titan] 2025-07-10 11:13:26,160 - root - INFO - step: 98340 loss: 16.3340 memory: 44.58GiB(31.99%) tps: 84,124 tflops: 290.33 mfu: 29.36% global_avg_ntp_loss: 2.4976 global_avg_mtp_loss: 13.8364 +[titan] 2025-07-10 11:13:26,160 - root - INFO - lr: 3.0187e-05 gnorm: 1.05 [21:38:21< 0:21:54] +[titan] 2025-07-10 11:13:30,070 - root - INFO - step: 98345 loss: 16.1669 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4637 global_avg_mtp_loss: 13.7032 +[titan] 2025-07-10 11:13:30,070 - root - INFO - lr: 3.0186e-05 gnorm: 1.05 [21:38:25< 0:21:51] +[titan] 2025-07-10 11:13:33,205 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:13:34,006 - root - INFO - step: 98350 loss: 16.0773 memory: 44.58GiB(31.99%) tps: 83,262 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6169 +[titan] 2025-07-10 11:13:34,006 - root - INFO - lr: 3.0185e-05 gnorm: 1.08 [21:38:29< 0:21:47] +[titan] 2025-07-10 11:13:37,939 - root - INFO - step: 98355 loss: 15.9463 memory: 44.58GiB(31.99%) tps: 83,305 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4152 global_avg_mtp_loss: 13.5311 +[titan] 2025-07-10 11:13:37,940 - root - INFO - lr: 3.0184e-05 gnorm: 1.07 [21:38:33< 0:21:43] +[titan] 2025-07-10 11:13:41,858 - root - INFO - step: 98360 loss: 16.1411 memory: 44.58GiB(31.99%) tps: 83,634 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 11:13:41,858 - root - INFO - lr: 3.0183e-05 gnorm: 1.05 [21:38:37< 0:21:39] +[titan] 2025-07-10 11:13:45,794 - root - INFO - step: 98365 loss: 16.0423 memory: 44.58GiB(31.99%) tps: 83,263 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4357 global_avg_mtp_loss: 13.6066 +[titan] 2025-07-10 11:13:45,794 - root - INFO - lr: 3.0182e-05 gnorm: 1.01 [21:38:41< 0:21:35] +[titan] 2025-07-10 11:13:49,727 - root - INFO - step: 98370 loss: 16.3858 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.5082 global_avg_mtp_loss: 13.8776 +[titan] 2025-07-10 11:13:49,727 - root - INFO - lr: 3.0181e-05 gnorm: 1.06 [21:38:45< 0:21:31] +[titan] 2025-07-10 11:13:53,665 - root - INFO - step: 98375 loss: 16.1423 memory: 44.58GiB(31.99%) tps: 83,202 tflops: 287.15 mfu: 29.03% global_avg_ntp_loss: 2.4559 global_avg_mtp_loss: 13.6864 +[titan] 2025-07-10 11:13:53,666 - root - INFO - lr: 3.0179e-05 gnorm: 1.10 [21:38:49< 0:21:27] +[titan] 2025-07-10 11:13:57,586 - root - INFO - step: 98380 loss: 15.9584 memory: 44.58GiB(31.99%) tps: 83,593 tflops: 288.49 mfu: 29.17% global_avg_ntp_loss: 2.4325 global_avg_mtp_loss: 13.5259 +[titan] 2025-07-10 11:13:57,586 - root - INFO - lr: 3.0178e-05 gnorm: 1.08 [21:38:53< 0:21:23] +[titan] 2025-07-10 11:14:01,488 - root - INFO - step: 98385 loss: 16.2043 memory: 44.58GiB(31.99%) tps: 83,982 tflops: 289.84 mfu: 29.31% global_avg_ntp_loss: 2.4758 global_avg_mtp_loss: 13.7285 +[titan] 2025-07-10 11:14:01,488 - root - INFO - lr: 3.0177e-05 gnorm: 1.07 [21:38:56< 0:21:19] +[titan] 2025-07-10 11:14:05,390 - root - INFO - step: 98390 loss: 16.3750 memory: 44.58GiB(31.99%) tps: 83,993 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.5006 global_avg_mtp_loss: 13.8744 +[titan] 2025-07-10 11:14:05,390 - root - INFO - lr: 3.0176e-05 gnorm: 1.04 [21:39:00< 0:21:15] +[titan] 2025-07-10 11:14:09,336 - root - INFO - step: 98395 loss: 16.1505 memory: 44.58GiB(31.99%) tps: 83,046 tflops: 286.60 mfu: 28.98% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.6829 +[titan] 2025-07-10 11:14:09,336 - root - INFO - lr: 3.0175e-05 gnorm: 1.03 [21:39:04< 0:21:11] +[titan] 2025-07-10 11:14:12,460 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:14:13,256 - root - INFO - step: 98400 loss: 16.2094 memory: 44.58GiB(31.99%) tps: 83,603 tflops: 288.53 mfu: 29.17% global_avg_ntp_loss: 2.4738 global_avg_mtp_loss: 13.7356 +[titan] 2025-07-10 11:14:13,256 - root - INFO - lr: 3.0174e-05 gnorm: 1.01 [21:39:08< 0:21:07] +[titan] 2025-07-10 11:14:17,154 - root - INFO - step: 98405 loss: 16.0458 memory: 44.58GiB(31.99%) tps: 84,055 tflops: 290.09 mfu: 29.33% global_avg_ntp_loss: 2.4468 global_avg_mtp_loss: 13.5990 +[titan] 2025-07-10 11:14:17,155 - root - INFO - lr: 3.0173e-05 gnorm: 1.02 [21:39:12< 0:21:03] +[titan] 2025-07-10 11:14:21,063 - root - INFO - step: 98410 loss: 15.9702 memory: 44.58GiB(31.99%) tps: 83,854 tflops: 289.39 mfu: 29.26% global_avg_ntp_loss: 2.4408 global_avg_mtp_loss: 13.5294 +[titan] 2025-07-10 11:14:21,063 - root - INFO - lr: 3.0172e-05 gnorm: 1.04 [21:39:16< 0:20:59] +[titan] 2025-07-10 11:14:24,978 - root - INFO - step: 98415 loss: 15.8340 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4072 global_avg_mtp_loss: 13.4268 +[titan] 2025-07-10 11:14:24,979 - root - INFO - lr: 3.0171e-05 gnorm: 1.03 [21:39:20< 0:20:55] +[titan] 2025-07-10 11:14:28,883 - root - INFO - step: 98420 loss: 16.0542 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4358 global_avg_mtp_loss: 13.6184 +[titan] 2025-07-10 11:14:28,884 - root - INFO - lr: 3.0170e-05 gnorm: 1.04 [21:39:24< 0:20:51] +[titan] 2025-07-10 11:14:32,808 - root - INFO - step: 98425 loss: 15.9099 memory: 44.58GiB(31.99%) tps: 83,510 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4241 global_avg_mtp_loss: 13.4858 +[titan] 2025-07-10 11:14:32,808 - root - INFO - lr: 3.0169e-05 gnorm: 1.09 [21:39:28< 0:20:47] +[titan] 2025-07-10 11:14:36,744 - root - INFO - step: 98430 loss: 16.3496 memory: 44.58GiB(31.99%) tps: 83,257 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.5102 global_avg_mtp_loss: 13.8394 +[titan] 2025-07-10 11:14:36,744 - root - INFO - lr: 3.0168e-05 gnorm: 1.15 [21:39:32< 0:20:43] +[titan] 2025-07-10 11:14:40,643 - root - INFO - step: 98435 loss: 16.6113 memory: 44.58GiB(31.99%) tps: 84,042 tflops: 290.04 mfu: 29.33% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 14.0756 +[titan] 2025-07-10 11:14:40,644 - root - INFO - lr: 3.0166e-05 gnorm: 1.11 [21:39:36< 0:20:39] +[titan] 2025-07-10 11:14:44,575 - root - INFO - step: 98440 loss: 15.9801 memory: 44.58GiB(31.99%) tps: 83,340 tflops: 287.62 mfu: 29.08% global_avg_ntp_loss: 2.4463 global_avg_mtp_loss: 13.5337 +[titan] 2025-07-10 11:14:44,576 - root - INFO - lr: 3.0165e-05 gnorm: 1.18 [21:39:40< 0:20:35] +[titan] 2025-07-10 11:14:48,478 - root - INFO - step: 98445 loss: 16.4471 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.5316 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 11:14:48,478 - root - INFO - lr: 3.0164e-05 gnorm: 1.14 [21:39:43< 0:20:31] +[titan] 2025-07-10 11:14:51,635 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:14:52,433 - root - INFO - step: 98450 loss: 16.1768 memory: 44.58GiB(31.99%) tps: 82,860 tflops: 285.96 mfu: 28.91% global_avg_ntp_loss: 2.4711 global_avg_mtp_loss: 13.7057 +[titan] 2025-07-10 11:14:52,433 - root - INFO - lr: 3.0163e-05 gnorm: 1.07 [21:39:47< 0:20:27] +[titan] 2025-07-10 11:14:56,394 - root - INFO - step: 98455 loss: 16.1804 memory: 44.58GiB(31.99%) tps: 82,730 tflops: 285.51 mfu: 28.87% global_avg_ntp_loss: 2.4744 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 11:14:56,394 - root - INFO - lr: 3.0162e-05 gnorm: 1.11 [21:39:51< 0:20:23] +[titan] 2025-07-10 11:15:00,327 - root - INFO - step: 98460 loss: 16.3618 memory: 44.58GiB(31.99%) tps: 83,323 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.5168 global_avg_mtp_loss: 13.8450 +[titan] 2025-07-10 11:15:00,327 - root - INFO - lr: 3.0161e-05 gnorm: 1.02 [21:39:55< 0:20:19] +[titan] 2025-07-10 11:15:04,244 - root - INFO - step: 98465 loss: 16.0886 memory: 44.58GiB(31.99%) tps: 83,668 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4409 global_avg_mtp_loss: 13.6478 +[titan] 2025-07-10 11:15:04,244 - root - INFO - lr: 3.0160e-05 gnorm: 1.10 [21:39:59< 0:20:15] +[titan] 2025-07-10 11:15:08,195 - root - INFO - step: 98470 loss: 16.4084 memory: 44.58GiB(31.99%) tps: 82,935 tflops: 286.22 mfu: 28.94% global_avg_ntp_loss: 2.5140 global_avg_mtp_loss: 13.8944 +[titan] 2025-07-10 11:15:08,196 - root - INFO - lr: 3.0159e-05 gnorm: 1.00 [21:40:03< 0:20:11] +[titan] 2025-07-10 11:15:12,109 - root - INFO - step: 98475 loss: 16.0844 memory: 44.58GiB(31.99%) tps: 83,731 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4539 global_avg_mtp_loss: 13.6305 +[titan] 2025-07-10 11:15:12,110 - root - INFO - lr: 3.0158e-05 gnorm: 1.04 [21:40:07< 0:20:08] +[titan] 2025-07-10 11:15:16,063 - root - INFO - step: 98480 loss: 16.1384 memory: 44.58GiB(31.99%) tps: 82,885 tflops: 286.05 mfu: 28.92% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6734 +[titan] 2025-07-10 11:15:16,063 - root - INFO - lr: 3.0157e-05 gnorm: 1.01 [21:40:11< 0:20:04] +[titan] 2025-07-10 11:15:19,984 - root - INFO - step: 98485 loss: 16.2102 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.4724 global_avg_mtp_loss: 13.7378 +[titan] 2025-07-10 11:15:19,985 - root - INFO - lr: 3.0156e-05 gnorm: 1.46 [21:40:15< 0:20:00] +[titan] 2025-07-10 11:15:23,886 - root - INFO - step: 98490 loss: 16.0672 memory: 44.58GiB(31.99%) tps: 83,997 tflops: 289.89 mfu: 29.31% global_avg_ntp_loss: 2.4425 global_avg_mtp_loss: 13.6247 +[titan] 2025-07-10 11:15:23,886 - root - INFO - lr: 3.0155e-05 gnorm: 1.09 [21:40:19< 0:19:56] +[titan] 2025-07-10 11:15:27,805 - root - INFO - step: 98495 loss: 16.1084 memory: 44.58GiB(31.99%) tps: 83,611 tflops: 288.56 mfu: 29.18% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6564 +[titan] 2025-07-10 11:15:27,806 - root - INFO - lr: 3.0154e-05 gnorm: 1.08 [21:40:23< 0:19:52] +[titan] 2025-07-10 11:15:30,952 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:15:31,739 - root - INFO - step: 98500 loss: 16.2479 memory: 44.58GiB(31.99%) tps: 83,319 tflops: 287.55 mfu: 29.07% global_avg_ntp_loss: 2.4793 global_avg_mtp_loss: 13.7686 +[titan] 2025-07-10 11:15:31,739 - root - INFO - lr: 3.0153e-05 gnorm: 1.10 [21:40:27< 0:19:48] +[titan] 2025-07-10 11:15:35,660 - root - INFO - step: 98505 loss: 16.4013 memory: 44.58GiB(31.99%) tps: 83,577 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.5123 global_avg_mtp_loss: 13.8890 +[titan] 2025-07-10 11:15:35,660 - root - INFO - lr: 3.0152e-05 gnorm: 1.11 [21:40:31< 0:19:44] +[titan] 2025-07-10 11:15:39,599 - root - INFO - step: 98510 loss: 16.2195 memory: 44.58GiB(31.99%) tps: 83,199 tflops: 287.13 mfu: 29.03% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7397 +[titan] 2025-07-10 11:15:39,599 - root - INFO - lr: 3.0151e-05 gnorm: 1.09 [21:40:35< 0:19:40] +[titan] 2025-07-10 11:15:43,508 - root - INFO - step: 98515 loss: 16.1255 memory: 44.58GiB(31.99%) tps: 83,822 tflops: 289.28 mfu: 29.25% global_avg_ntp_loss: 2.4522 global_avg_mtp_loss: 13.6733 +[titan] 2025-07-10 11:15:43,509 - root - INFO - lr: 3.0150e-05 gnorm: 1.10 [21:40:38< 0:19:36] +[titan] 2025-07-10 11:15:47,438 - root - INFO - step: 98520 loss: 16.3024 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4902 global_avg_mtp_loss: 13.8123 +[titan] 2025-07-10 11:15:47,438 - root - INFO - lr: 3.0149e-05 gnorm: 1.08 [21:40:42< 0:19:32] +[titan] 2025-07-10 11:15:51,343 - root - INFO - step: 98525 loss: 16.0142 memory: 44.58GiB(31.99%) tps: 83,927 tflops: 289.65 mfu: 29.29% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.5582 +[titan] 2025-07-10 11:15:51,343 - root - INFO - lr: 3.0148e-05 gnorm: 1.22 [21:40:46< 0:19:28] +[titan] 2025-07-10 11:15:55,258 - root - INFO - step: 98530 loss: 15.8986 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4182 global_avg_mtp_loss: 13.4804 +[titan] 2025-07-10 11:15:55,258 - root - INFO - lr: 3.0147e-05 gnorm: 1.07 [21:40:50< 0:19:24] +[titan] 2025-07-10 11:15:59,174 - root - INFO - step: 98535 loss: 15.9775 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4365 global_avg_mtp_loss: 13.5410 +[titan] 2025-07-10 11:15:59,175 - root - INFO - lr: 3.0146e-05 gnorm: 1.09 [21:40:54< 0:19:20] +[titan] 2025-07-10 11:16:03,080 - root - INFO - step: 98540 loss: 16.1135 memory: 44.58GiB(31.99%) tps: 83,911 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4679 global_avg_mtp_loss: 13.6455 +[titan] 2025-07-10 11:16:03,080 - root - INFO - lr: 3.0145e-05 gnorm: 1.06 [21:40:58< 0:19:16] +[titan] 2025-07-10 11:16:06,998 - root - INFO - step: 98545 loss: 15.9876 memory: 44.58GiB(31.99%) tps: 83,636 tflops: 288.64 mfu: 29.19% global_avg_ntp_loss: 2.4509 global_avg_mtp_loss: 13.5366 +[titan] 2025-07-10 11:16:06,998 - root - INFO - lr: 3.0144e-05 gnorm: 1.11 [21:41:02< 0:19:12] +[titan] 2025-07-10 11:16:10,115 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:16:10,908 - root - INFO - step: 98550 loss: 16.1187 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6573 +[titan] 2025-07-10 11:16:10,909 - root - INFO - lr: 3.0143e-05 gnorm: 1.08 [21:41:06< 0:19:08] +[titan] 2025-07-10 11:16:14,880 - root - INFO - step: 98555 loss: 16.0965 memory: 44.58GiB(31.99%) tps: 82,503 tflops: 284.73 mfu: 28.79% global_avg_ntp_loss: 2.4556 global_avg_mtp_loss: 13.6409 +[titan] 2025-07-10 11:16:14,881 - root - INFO - lr: 3.0142e-05 gnorm: 1.08 [21:41:10< 0:19:04] +[titan] 2025-07-10 11:16:18,807 - root - INFO - step: 98560 loss: 16.0858 memory: 44.58GiB(31.99%) tps: 83,452 tflops: 288.01 mfu: 29.12% global_avg_ntp_loss: 2.4482 global_avg_mtp_loss: 13.6376 +[titan] 2025-07-10 11:16:18,808 - root - INFO - lr: 3.0141e-05 gnorm: 1.11 [21:41:14< 0:19:00] +[titan] 2025-07-10 11:16:22,712 - root - INFO - step: 98565 loss: 16.3251 memory: 44.58GiB(31.99%) tps: 83,926 tflops: 289.64 mfu: 29.29% global_avg_ntp_loss: 2.4870 global_avg_mtp_loss: 13.8380 +[titan] 2025-07-10 11:16:22,713 - root - INFO - lr: 3.0140e-05 gnorm: 1.05 [21:41:18< 0:18:56] +[titan] 2025-07-10 11:16:26,647 - root - INFO - step: 98570 loss: 15.9896 memory: 44.58GiB(31.99%) tps: 83,280 tflops: 287.41 mfu: 29.06% global_avg_ntp_loss: 2.4318 global_avg_mtp_loss: 13.5578 +[titan] 2025-07-10 11:16:26,648 - root - INFO - lr: 3.0139e-05 gnorm: 1.09 [21:41:22< 0:18:52] +[titan] 2025-07-10 11:16:30,566 - root - INFO - step: 98575 loss: 16.0429 memory: 44.58GiB(31.99%) tps: 83,630 tflops: 288.62 mfu: 29.18% global_avg_ntp_loss: 2.4492 global_avg_mtp_loss: 13.5937 +[titan] 2025-07-10 11:16:30,566 - root - INFO - lr: 3.0138e-05 gnorm: 1.13 [21:41:26< 0:18:48] +[titan] 2025-07-10 11:16:34,491 - root - INFO - step: 98580 loss: 16.1818 memory: 44.58GiB(31.99%) tps: 83,498 tflops: 288.16 mfu: 29.14% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7041 +[titan] 2025-07-10 11:16:34,491 - root - INFO - lr: 3.0137e-05 gnorm: 1.03 [21:41:29< 0:18:44] +[titan] 2025-07-10 11:16:38,394 - root - INFO - step: 98585 loss: 16.0742 memory: 44.58GiB(31.99%) tps: 83,962 tflops: 289.77 mfu: 29.30% global_avg_ntp_loss: 2.4450 global_avg_mtp_loss: 13.6292 +[titan] 2025-07-10 11:16:38,394 - root - INFO - lr: 3.0136e-05 gnorm: 1.05 [21:41:33< 0:18:40] +[titan] 2025-07-10 11:16:42,308 - root - INFO - step: 98590 loss: 16.1629 memory: 44.58GiB(31.99%) tps: 83,720 tflops: 288.93 mfu: 29.21% global_avg_ntp_loss: 2.4661 global_avg_mtp_loss: 13.6968 +[titan] 2025-07-10 11:16:42,309 - root - INFO - lr: 3.0135e-05 gnorm: 1.02 [21:41:37< 0:18:36] +[titan] 2025-07-10 11:16:46,253 - root - INFO - step: 98595 loss: 15.9337 memory: 44.58GiB(31.99%) tps: 83,067 tflops: 286.68 mfu: 28.99% global_avg_ntp_loss: 2.4237 global_avg_mtp_loss: 13.5100 +[titan] 2025-07-10 11:16:46,254 - root - INFO - lr: 3.0134e-05 gnorm: 1.06 [21:41:41< 0:18:32] +[titan] 2025-07-10 11:16:49,398 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:16:50,222 - root - INFO - step: 98600 loss: 16.0679 memory: 44.58GiB(31.99%) tps: 82,570 tflops: 284.96 mfu: 28.81% global_avg_ntp_loss: 2.4479 global_avg_mtp_loss: 13.6200 +[titan] 2025-07-10 11:16:50,223 - root - INFO - lr: 3.0133e-05 gnorm: 1.04 [21:41:45< 0:18:29] +[titan] 2025-07-10 11:16:54,174 - root - INFO - step: 98605 loss: 16.1870 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.4621 global_avg_mtp_loss: 13.7249 +[titan] 2025-07-10 11:16:54,175 - root - INFO - lr: 3.0132e-05 gnorm: 1.08 [21:41:49< 0:18:25] +[titan] 2025-07-10 11:16:58,080 - root - INFO - step: 98610 loss: 15.8970 memory: 44.58GiB(31.99%) tps: 83,920 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4223 global_avg_mtp_loss: 13.4747 +[titan] 2025-07-10 11:16:58,080 - root - INFO - lr: 3.0131e-05 gnorm: 1.10 [21:41:53< 0:18:21] +[titan] 2025-07-10 11:17:01,994 - root - INFO - step: 98615 loss: 16.3988 memory: 44.58GiB(31.99%) tps: 83,709 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4978 global_avg_mtp_loss: 13.9010 +[titan] 2025-07-10 11:17:01,995 - root - INFO - lr: 3.0130e-05 gnorm: 1.06 [21:41:57< 0:18:17] +[titan] 2025-07-10 11:17:05,945 - root - INFO - step: 98620 loss: 15.9722 memory: 44.58GiB(31.99%) tps: 82,946 tflops: 286.26 mfu: 28.94% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.5393 +[titan] 2025-07-10 11:17:05,946 - root - INFO - lr: 3.0129e-05 gnorm: 1.13 [21:42:01< 0:18:13] +[titan] 2025-07-10 11:17:09,858 - root - INFO - step: 98625 loss: 15.9952 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4268 global_avg_mtp_loss: 13.5684 +[titan] 2025-07-10 11:17:09,858 - root - INFO - lr: 3.0128e-05 gnorm: 1.06 [21:42:05< 0:18:09] +[titan] 2025-07-10 11:17:13,788 - root - INFO - step: 98630 loss: 16.2383 memory: 44.58GiB(31.99%) tps: 83,389 tflops: 287.79 mfu: 29.10% global_avg_ntp_loss: 2.4852 global_avg_mtp_loss: 13.7531 +[titan] 2025-07-10 11:17:13,788 - root - INFO - lr: 3.0128e-05 gnorm: 1.09 [21:42:09< 0:18:05] +[titan] 2025-07-10 11:17:17,725 - root - INFO - step: 98635 loss: 16.2786 memory: 44.58GiB(31.99%) tps: 83,242 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4863 global_avg_mtp_loss: 13.7923 +[titan] 2025-07-10 11:17:17,725 - root - INFO - lr: 3.0127e-05 gnorm: 1.06 [21:42:13< 0:18:01] +[titan] 2025-07-10 11:17:21,637 - root - INFO - step: 98640 loss: 16.2387 memory: 44.58GiB(31.99%) tps: 83,774 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7678 +[titan] 2025-07-10 11:17:21,637 - root - INFO - lr: 3.0126e-05 gnorm: 1.05 [21:42:17< 0:17:57] +[titan] 2025-07-10 11:17:25,561 - root - INFO - step: 98645 loss: 16.2386 memory: 44.58GiB(31.99%) tps: 83,517 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4907 global_avg_mtp_loss: 13.7479 +[titan] 2025-07-10 11:17:25,561 - root - INFO - lr: 3.0125e-05 gnorm: 1.08 [21:42:21< 0:17:53] +[titan] 2025-07-10 11:17:28,690 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:17:29,479 - root - INFO - step: 98650 loss: 16.0023 memory: 44.58GiB(31.99%) tps: 83,649 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4282 global_avg_mtp_loss: 13.5741 +[titan] 2025-07-10 11:17:29,479 - root - INFO - lr: 3.0124e-05 gnorm: 1.11 [21:42:24< 0:17:49] +[titan] 2025-07-10 11:17:33,429 - root - INFO - step: 98655 loss: 16.4441 memory: 44.58GiB(31.99%) tps: 82,965 tflops: 286.33 mfu: 28.95% global_avg_ntp_loss: 2.5301 global_avg_mtp_loss: 13.9141 +[titan] 2025-07-10 11:17:33,429 - root - INFO - lr: 3.0123e-05 gnorm: 1.05 [21:42:28< 0:17:45] +[titan] 2025-07-10 11:17:37,446 - root - INFO - step: 98660 loss: 15.9272 memory: 44.58GiB(31.99%) tps: 81,578 tflops: 281.54 mfu: 28.47% global_avg_ntp_loss: 2.4219 global_avg_mtp_loss: 13.5052 +[titan] 2025-07-10 11:17:37,446 - root - INFO - lr: 3.0122e-05 gnorm: 1.08 [21:42:32< 0:17:41] +[titan] 2025-07-10 11:17:41,383 - root - INFO - step: 98665 loss: 16.3074 memory: 44.58GiB(31.99%) tps: 83,232 tflops: 287.25 mfu: 29.04% global_avg_ntp_loss: 2.4940 global_avg_mtp_loss: 13.8134 +[titan] 2025-07-10 11:17:41,383 - root - INFO - lr: 3.0121e-05 gnorm: 1.12 [21:42:36< 0:17:37] +[titan] 2025-07-10 11:17:45,284 - root - INFO - step: 98670 loss: 16.2483 memory: 44.58GiB(31.99%) tps: 84,009 tflops: 289.93 mfu: 29.32% global_avg_ntp_loss: 2.4707 global_avg_mtp_loss: 13.7776 +[titan] 2025-07-10 11:17:45,284 - root - INFO - lr: 3.0120e-05 gnorm: 1.05 [21:42:40< 0:17:33] +[titan] 2025-07-10 11:17:49,192 - root - INFO - step: 98675 loss: 15.8190 memory: 44.58GiB(31.99%) tps: 83,860 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.3992 global_avg_mtp_loss: 13.4198 +[titan] 2025-07-10 11:17:49,192 - root - INFO - lr: 3.0119e-05 gnorm: 1.13 [21:42:44< 0:17:29] +[titan] 2025-07-10 11:17:53,098 - root - INFO - step: 98680 loss: 16.3617 memory: 44.58GiB(31.99%) tps: 83,894 tflops: 289.53 mfu: 29.28% global_avg_ntp_loss: 2.5081 global_avg_mtp_loss: 13.8536 +[titan] 2025-07-10 11:17:53,098 - root - INFO - lr: 3.0118e-05 gnorm: 1.07 [21:42:48< 0:17:25] +[titan] 2025-07-10 11:17:57,066 - root - INFO - step: 98685 loss: 15.8494 memory: 44.58GiB(31.99%) tps: 82,595 tflops: 285.05 mfu: 28.82% global_avg_ntp_loss: 2.4344 global_avg_mtp_loss: 13.4150 +[titan] 2025-07-10 11:17:57,066 - root - INFO - lr: 3.0118e-05 gnorm: 1.06 [21:42:52< 0:17:21] +[titan] 2025-07-10 11:18:01,009 - root - INFO - step: 98690 loss: 16.0535 memory: 44.58GiB(31.99%) tps: 83,112 tflops: 286.83 mfu: 29.00% global_avg_ntp_loss: 2.4457 global_avg_mtp_loss: 13.6078 +[titan] 2025-07-10 11:18:01,009 - root - INFO - lr: 3.0117e-05 gnorm: 1.04 [21:42:56< 0:17:17] +[titan] 2025-07-10 11:18:04,918 - root - INFO - step: 98695 loss: 16.1847 memory: 44.58GiB(31.99%) tps: 83,826 tflops: 289.30 mfu: 29.25% global_avg_ntp_loss: 2.4669 global_avg_mtp_loss: 13.7178 +[titan] 2025-07-10 11:18:04,919 - root - INFO - lr: 3.0116e-05 gnorm: 1.05 [21:43:00< 0:17:13] +[titan] 2025-07-10 11:18:08,112 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:18:08,934 - root - INFO - step: 98700 loss: 16.0921 memory: 44.58GiB(31.99%) tps: 81,609 tflops: 281.65 mfu: 28.48% global_avg_ntp_loss: 2.4507 global_avg_mtp_loss: 13.6413 +[titan] 2025-07-10 11:18:08,934 - root - INFO - lr: 3.0115e-05 gnorm: 1.06 [21:43:04< 0:17:09] +[titan] 2025-07-10 11:18:12,833 - root - INFO - step: 98705 loss: 16.1810 memory: 44.58GiB(31.99%) tps: 84,053 tflops: 290.08 mfu: 29.33% global_avg_ntp_loss: 2.4688 global_avg_mtp_loss: 13.7122 +[titan] 2025-07-10 11:18:12,833 - root - INFO - lr: 3.0114e-05 gnorm: 1.04 [21:43:08< 0:17:05] +[titan] 2025-07-10 11:18:16,738 - root - INFO - step: 98710 loss: 15.9834 memory: 44.58GiB(31.99%) tps: 83,923 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4380 global_avg_mtp_loss: 13.5454 +[titan] 2025-07-10 11:18:16,738 - root - INFO - lr: 3.0113e-05 gnorm: 1.10 [21:43:12< 0:17:01] +[titan] 2025-07-10 11:18:20,712 - root - INFO - step: 98715 loss: 15.9946 memory: 44.58GiB(31.99%) tps: 82,452 tflops: 284.55 mfu: 28.77% global_avg_ntp_loss: 2.4374 global_avg_mtp_loss: 13.5572 +[titan] 2025-07-10 11:18:20,713 - root - INFO - lr: 3.0112e-05 gnorm: 1.03 [21:43:16< 0:16:57] +[titan] 2025-07-10 11:18:24,684 - root - INFO - step: 98720 loss: 16.2291 memory: 44.58GiB(31.99%) tps: 82,524 tflops: 284.80 mfu: 28.80% global_avg_ntp_loss: 2.4759 global_avg_mtp_loss: 13.7533 +[titan] 2025-07-10 11:18:24,684 - root - INFO - lr: 3.0111e-05 gnorm: 1.05 [21:43:20< 0:16:53] +[titan] 2025-07-10 11:18:28,619 - root - INFO - step: 98725 loss: 16.3847 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4971 global_avg_mtp_loss: 13.8875 +[titan] 2025-07-10 11:18:28,619 - root - INFO - lr: 3.0110e-05 gnorm: 1.04 [21:43:24< 0:16:49] +[titan] 2025-07-10 11:18:32,535 - root - INFO - step: 98730 loss: 16.0017 memory: 44.58GiB(31.99%) tps: 83,678 tflops: 288.79 mfu: 29.20% global_avg_ntp_loss: 2.4338 global_avg_mtp_loss: 13.5679 +[titan] 2025-07-10 11:18:32,536 - root - INFO - lr: 3.0110e-05 gnorm: 1.15 [21:43:27< 0:16:46] +[titan] 2025-07-10 11:18:36,438 - root - INFO - step: 98735 loss: 16.2805 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.7849 +[titan] 2025-07-10 11:18:36,438 - root - INFO - lr: 3.0109e-05 gnorm: 1.08 [21:43:31< 0:16:42] +[titan] 2025-07-10 11:18:40,344 - root - INFO - step: 98740 loss: 16.0528 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.5991 +[titan] 2025-07-10 11:18:40,344 - root - INFO - lr: 3.0108e-05 gnorm: 1.09 [21:43:35< 0:16:38] +[titan] 2025-07-10 11:18:44,257 - root - INFO - step: 98745 loss: 15.8404 memory: 44.58GiB(31.99%) tps: 83,741 tflops: 289.01 mfu: 29.22% global_avg_ntp_loss: 2.4106 global_avg_mtp_loss: 13.4298 +[titan] 2025-07-10 11:18:44,258 - root - INFO - lr: 3.0107e-05 gnorm: 1.07 [21:43:39< 0:16:34] +[titan] 2025-07-10 11:18:47,358 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:18:48,152 - root - INFO - step: 98750 loss: 16.2455 memory: 44.58GiB(31.99%) tps: 84,146 tflops: 290.40 mfu: 29.36% global_avg_ntp_loss: 2.4692 global_avg_mtp_loss: 13.7763 +[titan] 2025-07-10 11:18:48,152 - root - INFO - lr: 3.0106e-05 gnorm: 1.05 [21:43:43< 0:16:30] +[titan] 2025-07-10 11:18:52,073 - root - INFO - step: 98755 loss: 16.1597 memory: 44.58GiB(31.99%) tps: 83,584 tflops: 288.46 mfu: 29.17% global_avg_ntp_loss: 2.4627 global_avg_mtp_loss: 13.6970 +[titan] 2025-07-10 11:18:52,073 - root - INFO - lr: 3.0105e-05 gnorm: 1.11 [21:43:47< 0:16:26] +[titan] 2025-07-10 11:18:55,983 - root - INFO - step: 98760 loss: 16.1147 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4540 global_avg_mtp_loss: 13.6607 +[titan] 2025-07-10 11:18:55,983 - root - INFO - lr: 3.0104e-05 gnorm: 1.07 [21:43:51< 0:16:22] +[titan] 2025-07-10 11:18:59,956 - root - INFO - step: 98765 loss: 16.1657 memory: 44.58GiB(31.99%) tps: 82,475 tflops: 284.63 mfu: 28.78% global_avg_ntp_loss: 2.4797 global_avg_mtp_loss: 13.6861 +[titan] 2025-07-10 11:18:59,956 - root - INFO - lr: 3.0104e-05 gnorm: 1.07 [21:43:55< 0:16:18] +[titan] 2025-07-10 11:19:03,907 - root - INFO - step: 98770 loss: 16.2237 memory: 44.58GiB(31.99%) tps: 82,951 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.4727 global_avg_mtp_loss: 13.7510 +[titan] 2025-07-10 11:19:03,907 - root - INFO - lr: 3.0103e-05 gnorm: 1.05 [21:43:59< 0:16:14] +[titan] 2025-07-10 11:19:07,857 - root - INFO - step: 98775 loss: 16.1765 memory: 44.58GiB(31.99%) tps: 82,950 tflops: 286.28 mfu: 28.95% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.7075 +[titan] 2025-07-10 11:19:07,858 - root - INFO - lr: 3.0102e-05 gnorm: 1.09 [21:44:03< 0:16:10] +[titan] 2025-07-10 11:19:11,753 - root - INFO - step: 98780 loss: 16.2820 memory: 44.58GiB(31.99%) tps: 84,137 tflops: 290.37 mfu: 29.36% global_avg_ntp_loss: 2.4854 global_avg_mtp_loss: 13.7966 +[titan] 2025-07-10 11:19:11,753 - root - INFO - lr: 3.0101e-05 gnorm: 1.07 [21:44:07< 0:16:06] +[titan] 2025-07-10 11:19:15,687 - root - INFO - step: 98785 loss: 15.7182 memory: 44.58GiB(31.99%) tps: 83,292 tflops: 287.45 mfu: 29.07% global_avg_ntp_loss: 2.3908 global_avg_mtp_loss: 13.3274 +[titan] 2025-07-10 11:19:15,687 - root - INFO - lr: 3.0100e-05 gnorm: 1.12 [21:44:11< 0:16:02] +[titan] 2025-07-10 11:19:19,599 - root - INFO - step: 98790 loss: 16.3458 memory: 44.58GiB(31.99%) tps: 83,781 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4999 global_avg_mtp_loss: 13.8459 +[titan] 2025-07-10 11:19:19,599 - root - INFO - lr: 3.0100e-05 gnorm: 1.09 [21:44:15< 0:15:58] +[titan] 2025-07-10 11:19:23,525 - root - INFO - step: 98795 loss: 16.2158 memory: 44.58GiB(31.99%) tps: 83,473 tflops: 288.08 mfu: 29.13% global_avg_ntp_loss: 2.4757 global_avg_mtp_loss: 13.7401 +[titan] 2025-07-10 11:19:23,525 - root - INFO - lr: 3.0099e-05 gnorm: 1.04 [21:44:18< 0:15:54] +[titan] 2025-07-10 11:19:26,681 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:19:27,537 - root - INFO - step: 98800 loss: 15.9053 memory: 44.58GiB(31.99%) tps: 81,676 tflops: 281.88 mfu: 28.50% global_avg_ntp_loss: 2.4131 global_avg_mtp_loss: 13.4922 +[titan] 2025-07-10 11:19:27,537 - root - INFO - lr: 3.0098e-05 gnorm: 1.07 [21:44:22< 0:15:50] +[titan] 2025-07-10 11:19:31,466 - root - INFO - step: 98805 loss: 16.0722 memory: 44.58GiB(31.99%) tps: 83,397 tflops: 287.82 mfu: 29.10% global_avg_ntp_loss: 2.4357 global_avg_mtp_loss: 13.6365 +[titan] 2025-07-10 11:19:31,467 - root - INFO - lr: 3.0097e-05 gnorm: 1.06 [21:44:26< 0:15:46] +[titan] 2025-07-10 11:19:35,372 - root - INFO - step: 98810 loss: 16.2404 memory: 44.58GiB(31.99%) tps: 83,915 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7649 +[titan] 2025-07-10 11:19:35,372 - root - INFO - lr: 3.0096e-05 gnorm: 1.07 [21:44:30< 0:15:42] +[titan] 2025-07-10 11:19:39,362 - root - INFO - step: 98815 loss: 16.1540 memory: 44.58GiB(31.99%) tps: 82,130 tflops: 283.44 mfu: 28.66% global_avg_ntp_loss: 2.4690 global_avg_mtp_loss: 13.6849 +[titan] 2025-07-10 11:19:39,362 - root - INFO - lr: 3.0095e-05 gnorm: 1.08 [21:44:34< 0:15:38] +[titan] 2025-07-10 11:19:40,300 - root - INFO - Dumping profiler traces at step 98816 +[titan] 2025-07-10 11:19:40,334 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 11:19:43,497 - root - INFO - step: 98820 loss: 16.1234 memory: 44.58GiB(31.99%) tps: 79,260 tflops: 273.54 mfu: 27.66% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.6576 +[titan] 2025-07-10 11:19:43,497 - root - INFO - lr: 3.0095e-05 gnorm: 1.03 [21:44:38< 0:15:34] +[titan] 2025-07-10 11:19:47,404 - root - INFO - step: 98825 loss: 16.2254 memory: 44.58GiB(31.99%) tps: 83,866 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7550 +[titan] 2025-07-10 11:19:47,404 - root - INFO - lr: 3.0094e-05 gnorm: 1.07 [21:44:42< 0:15:30] +[titan] 2025-07-10 11:19:51,340 - root - INFO - step: 98830 loss: 16.2429 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.7633 +[titan] 2025-07-10 11:19:51,340 - root - INFO - lr: 3.0093e-05 gnorm: 1.05 [21:44:46< 0:15:26] +[titan] 2025-07-10 11:19:55,256 - root - INFO - step: 98835 loss: 16.2941 memory: 44.58GiB(31.99%) tps: 83,677 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.5013 global_avg_mtp_loss: 13.7928 +[titan] 2025-07-10 11:19:55,257 - root - INFO - lr: 3.0092e-05 gnorm: 1.06 [21:44:50< 0:15:22] +[titan] 2025-07-10 11:19:59,173 - root - INFO - step: 98840 loss: 16.0984 memory: 44.58GiB(31.99%) tps: 83,676 tflops: 288.78 mfu: 29.20% global_avg_ntp_loss: 2.4372 global_avg_mtp_loss: 13.6612 +[titan] 2025-07-10 11:19:59,173 - root - INFO - lr: 3.0091e-05 gnorm: 1.06 [21:44:54< 0:15:18] +[titan] 2025-07-10 11:20:03,109 - root - INFO - step: 98845 loss: 16.2443 memory: 44.58GiB(31.99%) tps: 83,255 tflops: 287.33 mfu: 29.05% global_avg_ntp_loss: 2.4799 global_avg_mtp_loss: 13.7644 +[titan] 2025-07-10 11:20:03,110 - root - INFO - lr: 3.0091e-05 gnorm: 1.04 [21:44:58< 0:15:14] +[titan] 2025-07-10 11:20:06,228 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:20:07,017 - root - INFO - step: 98850 loss: 16.2643 memory: 44.58GiB(31.99%) tps: 83,859 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7843 +[titan] 2025-07-10 11:20:07,017 - root - INFO - lr: 3.0090e-05 gnorm: 1.08 [21:45:02< 0:15:10] +[titan] 2025-07-10 11:20:10,910 - root - INFO - step: 98855 loss: 16.2389 memory: 44.58GiB(31.99%) tps: 84,193 tflops: 290.56 mfu: 29.38% global_avg_ntp_loss: 2.4766 global_avg_mtp_loss: 13.7623 +[titan] 2025-07-10 11:20:10,910 - root - INFO - lr: 3.0089e-05 gnorm: 1.08 [21:45:06< 0:15:06] +[titan] 2025-07-10 11:20:14,829 - root - INFO - step: 98860 loss: 16.1031 memory: 44.58GiB(31.99%) tps: 83,605 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.4582 global_avg_mtp_loss: 13.6449 +[titan] 2025-07-10 11:20:14,830 - root - INFO - lr: 3.0088e-05 gnorm: 1.05 [21:45:10< 0:15:03] +[titan] 2025-07-10 11:20:18,741 - root - INFO - step: 98865 loss: 16.0461 memory: 44.58GiB(31.99%) tps: 83,778 tflops: 289.13 mfu: 29.23% global_avg_ntp_loss: 2.4422 global_avg_mtp_loss: 13.6039 +[titan] 2025-07-10 11:20:18,741 - root - INFO - lr: 3.0088e-05 gnorm: 1.11 [21:45:14< 0:14:59] +[titan] 2025-07-10 11:20:22,679 - root - INFO - step: 98870 loss: 15.9933 memory: 44.58GiB(31.99%) tps: 83,214 tflops: 287.19 mfu: 29.04% global_avg_ntp_loss: 2.4379 global_avg_mtp_loss: 13.5555 +[titan] 2025-07-10 11:20:22,680 - root - INFO - lr: 3.0087e-05 gnorm: 1.15 [21:45:18< 0:14:55] +[titan] 2025-07-10 11:20:26,608 - root - INFO - step: 98875 loss: 16.5440 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.5357 global_avg_mtp_loss: 14.0083 +[titan] 2025-07-10 11:20:26,608 - root - INFO - lr: 3.0086e-05 gnorm: 1.05 [21:45:22< 0:14:51] +[titan] 2025-07-10 11:20:30,511 - root - INFO - step: 98880 loss: 16.0655 memory: 44.58GiB(31.99%) tps: 83,970 tflops: 289.80 mfu: 29.30% global_avg_ntp_loss: 2.4528 global_avg_mtp_loss: 13.6127 +[titan] 2025-07-10 11:20:30,511 - root - INFO - lr: 3.0085e-05 gnorm: 1.08 [21:45:25< 0:14:47] +[titan] 2025-07-10 11:20:34,420 - root - INFO - step: 98885 loss: 16.1400 memory: 44.58GiB(31.99%) tps: 83,837 tflops: 289.33 mfu: 29.26% global_avg_ntp_loss: 2.4591 global_avg_mtp_loss: 13.6809 +[titan] 2025-07-10 11:20:34,420 - root - INFO - lr: 3.0084e-05 gnorm: 1.08 [21:45:29< 0:14:43] +[titan] 2025-07-10 11:20:38,347 - root - INFO - step: 98890 loss: 15.9689 memory: 44.58GiB(31.99%) tps: 83,435 tflops: 287.95 mfu: 29.11% global_avg_ntp_loss: 2.4309 global_avg_mtp_loss: 13.5380 +[titan] 2025-07-10 11:20:38,348 - root - INFO - lr: 3.0084e-05 gnorm: 1.09 [21:45:33< 0:14:39] +[titan] 2025-07-10 11:20:42,315 - root - INFO - step: 98895 loss: 16.1933 memory: 44.58GiB(31.99%) tps: 82,588 tflops: 285.03 mfu: 28.82% global_avg_ntp_loss: 2.4742 global_avg_mtp_loss: 13.7190 +[titan] 2025-07-10 11:20:42,316 - root - INFO - lr: 3.0083e-05 gnorm: 1.02 [21:45:37< 0:14:35] +[titan] 2025-07-10 11:20:45,429 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:20:46,222 - root - INFO - step: 98900 loss: 16.0277 memory: 44.58GiB(31.99%) tps: 83,896 tflops: 289.54 mfu: 29.28% global_avg_ntp_loss: 2.4356 global_avg_mtp_loss: 13.5921 +[titan] 2025-07-10 11:20:46,222 - root - INFO - lr: 3.0082e-05 gnorm: 1.18 [21:45:41< 0:14:31] +[titan] 2025-07-10 11:20:50,127 - root - INFO - step: 98905 loss: 16.1020 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6425 +[titan] 2025-07-10 11:20:50,127 - root - INFO - lr: 3.0081e-05 gnorm: 1.07 [21:45:45< 0:14:27] +[titan] 2025-07-10 11:20:54,061 - root - INFO - step: 98910 loss: 15.9201 memory: 44.58GiB(31.99%) tps: 83,306 tflops: 287.50 mfu: 29.07% global_avg_ntp_loss: 2.4266 global_avg_mtp_loss: 13.4936 +[titan] 2025-07-10 11:20:54,061 - root - INFO - lr: 3.0081e-05 gnorm: 1.03 [21:45:49< 0:14:23] +[titan] 2025-07-10 11:20:57,967 - root - INFO - step: 98915 loss: 16.4771 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.5233 global_avg_mtp_loss: 13.9537 +[titan] 2025-07-10 11:20:57,967 - root - INFO - lr: 3.0080e-05 gnorm: 1.13 [21:45:53< 0:14:19] +[titan] 2025-07-10 11:21:01,881 - root - INFO - step: 98920 loss: 16.1259 memory: 44.58GiB(31.99%) tps: 83,735 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4554 global_avg_mtp_loss: 13.6705 +[titan] 2025-07-10 11:21:01,881 - root - INFO - lr: 3.0079e-05 gnorm: 1.02 [21:45:57< 0:14:15] +[titan] 2025-07-10 11:21:05,799 - root - INFO - step: 98925 loss: 16.1538 memory: 44.58GiB(31.99%) tps: 83,645 tflops: 288.67 mfu: 29.19% global_avg_ntp_loss: 2.4549 global_avg_mtp_loss: 13.6989 +[titan] 2025-07-10 11:21:05,799 - root - INFO - lr: 3.0079e-05 gnorm: 1.05 [21:46:01< 0:14:11] +[titan] 2025-07-10 11:21:09,730 - root - INFO - step: 98930 loss: 16.2296 memory: 44.58GiB(31.99%) tps: 83,348 tflops: 287.65 mfu: 29.08% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.7599 +[titan] 2025-07-10 11:21:09,731 - root - INFO - lr: 3.0078e-05 gnorm: 1.09 [21:46:05< 0:14:07] +[titan] 2025-07-10 11:21:13,677 - root - INFO - step: 98935 loss: 16.1139 memory: 44.58GiB(31.99%) tps: 83,031 tflops: 286.55 mfu: 28.97% global_avg_ntp_loss: 2.4596 global_avg_mtp_loss: 13.6543 +[titan] 2025-07-10 11:21:13,678 - root - INFO - lr: 3.0077e-05 gnorm: 1.03 [21:46:09< 0:14:03] +[titan] 2025-07-10 11:21:17,595 - root - INFO - step: 98940 loss: 16.2331 memory: 44.58GiB(31.99%) tps: 83,651 tflops: 288.69 mfu: 29.19% global_avg_ntp_loss: 2.4834 global_avg_mtp_loss: 13.7497 +[titan] 2025-07-10 11:21:17,595 - root - INFO - lr: 3.0076e-05 gnorm: 1.07 [21:46:13< 0:13:59] +[titan] 2025-07-10 11:21:21,511 - root - INFO - step: 98945 loss: 16.1537 memory: 44.58GiB(31.99%) tps: 83,690 tflops: 288.83 mfu: 29.20% global_avg_ntp_loss: 2.4665 global_avg_mtp_loss: 13.6871 +[titan] 2025-07-10 11:21:21,511 - root - INFO - lr: 3.0076e-05 gnorm: 1.03 [21:46:16< 0:13:55] +[titan] 2025-07-10 11:21:24,633 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:21:25,420 - root - INFO - step: 98950 loss: 16.2142 memory: 44.58GiB(31.99%) tps: 83,828 tflops: 289.31 mfu: 29.25% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7362 +[titan] 2025-07-10 11:21:25,420 - root - INFO - lr: 3.0075e-05 gnorm: 1.06 [21:46:20< 0:13:51] +[titan] 2025-07-10 11:21:29,328 - root - INFO - step: 98955 loss: 15.9319 memory: 44.58GiB(31.99%) tps: 83,858 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4226 global_avg_mtp_loss: 13.5093 +[titan] 2025-07-10 11:21:29,328 - root - INFO - lr: 3.0074e-05 gnorm: 1.14 [21:46:24< 0:13:47] +[titan] 2025-07-10 11:21:33,240 - root - INFO - step: 98960 loss: 16.2468 memory: 44.58GiB(31.99%) tps: 83,765 tflops: 289.09 mfu: 29.23% global_avg_ntp_loss: 2.4633 global_avg_mtp_loss: 13.7835 +[titan] 2025-07-10 11:21:33,241 - root - INFO - lr: 3.0074e-05 gnorm: 1.04 [21:46:28< 0:13:43] +[titan] 2025-07-10 11:21:37,176 - root - INFO - step: 98965 loss: 16.3298 memory: 44.58GiB(31.99%) tps: 83,265 tflops: 287.36 mfu: 29.06% global_avg_ntp_loss: 2.4898 global_avg_mtp_loss: 13.8401 +[titan] 2025-07-10 11:21:37,177 - root - INFO - lr: 3.0073e-05 gnorm: 1.07 [21:46:32< 0:13:39] +[titan] 2025-07-10 11:21:41,127 - root - INFO - step: 98970 loss: 16.0364 memory: 44.58GiB(31.99%) tps: 82,956 tflops: 286.30 mfu: 28.95% global_avg_ntp_loss: 2.4447 global_avg_mtp_loss: 13.5917 +[titan] 2025-07-10 11:21:41,127 - root - INFO - lr: 3.0072e-05 gnorm: 1.08 [21:46:36< 0:13:35] +[titan] 2025-07-10 11:21:45,038 - root - INFO - step: 98975 loss: 16.4158 memory: 44.58GiB(31.99%) tps: 83,789 tflops: 289.17 mfu: 29.24% global_avg_ntp_loss: 2.4966 global_avg_mtp_loss: 13.9192 +[titan] 2025-07-10 11:21:45,038 - root - INFO - lr: 3.0071e-05 gnorm: 1.10 [21:46:40< 0:13:31] +[titan] 2025-07-10 11:21:48,948 - root - INFO - step: 98980 loss: 16.2630 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4826 global_avg_mtp_loss: 13.7804 +[titan] 2025-07-10 11:21:48,949 - root - INFO - lr: 3.0071e-05 gnorm: 1.07 [21:46:44< 0:13:27] +[titan] 2025-07-10 11:21:52,852 - root - INFO - step: 98985 loss: 16.4076 memory: 44.58GiB(31.99%) tps: 83,959 tflops: 289.75 mfu: 29.30% global_avg_ntp_loss: 2.5030 global_avg_mtp_loss: 13.9046 +[titan] 2025-07-10 11:21:52,852 - root - INFO - lr: 3.0070e-05 gnorm: 1.08 [21:46:48< 0:13:24] +[titan] 2025-07-10 11:21:56,762 - root - INFO - step: 98990 loss: 16.2050 memory: 44.58GiB(31.99%) tps: 83,804 tflops: 289.22 mfu: 29.24% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7314 +[titan] 2025-07-10 11:21:56,762 - root - INFO - lr: 3.0069e-05 gnorm: 1.10 [21:46:52< 0:13:20] +[titan] 2025-07-10 11:22:00,684 - root - INFO - step: 98995 loss: 16.2671 memory: 44.58GiB(31.99%) tps: 83,565 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4729 global_avg_mtp_loss: 13.7942 +[titan] 2025-07-10 11:22:00,684 - root - INFO - lr: 3.0069e-05 gnorm: 1.12 [21:46:56< 0:13:16] +[titan] 2025-07-10 11:22:03,835 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:22:04,629 - root - INFO - step: 99000 loss: 16.1548 memory: 44.58GiB(31.99%) tps: 83,078 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4618 global_avg_mtp_loss: 13.6930 +[titan] 2025-07-10 11:22:04,629 - root - INFO - lr: 3.0068e-05 gnorm: 1.10 [21:47:00< 0:13:12] +[titan] 2025-07-10 11:22:08,559 - root - INFO - step: 99005 loss: 16.1453 memory: 44.58GiB(31.99%) tps: 83,380 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4604 global_avg_mtp_loss: 13.6849 +[titan] 2025-07-10 11:22:08,559 - root - INFO - lr: 3.0067e-05 gnorm: 1.10 [21:47:03< 0:13:08] +[titan] 2025-07-10 11:22:12,550 - root - INFO - step: 99010 loss: 16.1156 memory: 44.58GiB(31.99%) tps: 82,118 tflops: 283.40 mfu: 28.66% global_avg_ntp_loss: 2.4689 global_avg_mtp_loss: 13.6467 +[titan] 2025-07-10 11:22:12,550 - root - INFO - lr: 3.0067e-05 gnorm: 1.09 [21:47:07< 0:13:04] +[titan] 2025-07-10 11:22:16,479 - root - INFO - step: 99015 loss: 16.2000 memory: 44.58GiB(31.99%) tps: 83,402 tflops: 287.83 mfu: 29.10% global_avg_ntp_loss: 2.4683 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 11:22:16,479 - root - INFO - lr: 3.0066e-05 gnorm: 1.10 [21:47:11< 0:13:00] +[titan] 2025-07-10 11:22:20,406 - root - INFO - step: 99020 loss: 16.2206 memory: 44.58GiB(31.99%) tps: 83,447 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4805 global_avg_mtp_loss: 13.7400 +[titan] 2025-07-10 11:22:20,406 - root - INFO - lr: 3.0065e-05 gnorm: 1.09 [21:47:15< 0:12:56] +[titan] 2025-07-10 11:22:24,352 - root - INFO - step: 99025 loss: 16.1781 memory: 44.58GiB(31.99%) tps: 83,057 tflops: 286.64 mfu: 28.98% global_avg_ntp_loss: 2.4731 global_avg_mtp_loss: 13.7051 +[titan] 2025-07-10 11:22:24,352 - root - INFO - lr: 3.0065e-05 gnorm: 1.06 [21:47:19< 0:12:52] +[titan] 2025-07-10 11:22:28,272 - root - INFO - step: 99030 loss: 16.0963 memory: 44.58GiB(31.99%) tps: 83,594 tflops: 288.50 mfu: 29.17% global_avg_ntp_loss: 2.4416 global_avg_mtp_loss: 13.6548 +[titan] 2025-07-10 11:22:28,272 - root - INFO - lr: 3.0064e-05 gnorm: 1.08 [21:47:23< 0:12:48] +[titan] 2025-07-10 11:22:32,200 - root - INFO - step: 99035 loss: 16.2458 memory: 44.58GiB(31.99%) tps: 83,426 tflops: 287.92 mfu: 29.11% global_avg_ntp_loss: 2.4755 global_avg_mtp_loss: 13.7703 +[titan] 2025-07-10 11:22:32,201 - root - INFO - lr: 3.0063e-05 gnorm: 1.07 [21:47:27< 0:12:44] +[titan] 2025-07-10 11:22:36,107 - root - INFO - step: 99040 loss: 15.9501 memory: 44.58GiB(31.99%) tps: 83,892 tflops: 289.53 mfu: 29.27% global_avg_ntp_loss: 2.4308 global_avg_mtp_loss: 13.5193 +[titan] 2025-07-10 11:22:36,107 - root - INFO - lr: 3.0063e-05 gnorm: 1.05 [21:47:31< 0:12:40] +[titan] 2025-07-10 11:22:40,033 - root - INFO - step: 99045 loss: 16.1539 memory: 44.58GiB(31.99%) tps: 83,461 tflops: 288.04 mfu: 29.12% global_avg_ntp_loss: 2.4623 global_avg_mtp_loss: 13.6917 +[titan] 2025-07-10 11:22:40,034 - root - INFO - lr: 3.0062e-05 gnorm: 1.06 [21:47:35< 0:12:36] +[titan] 2025-07-10 11:22:43,145 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:22:43,939 - root - INFO - step: 99050 loss: 16.1463 memory: 44.58GiB(31.99%) tps: 83,907 tflops: 289.58 mfu: 29.28% global_avg_ntp_loss: 2.4550 global_avg_mtp_loss: 13.6913 +[titan] 2025-07-10 11:22:43,939 - root - INFO - lr: 3.0061e-05 gnorm: 1.06 [21:47:39< 0:12:32] +[titan] 2025-07-10 11:22:47,850 - root - INFO - step: 99055 loss: 16.2853 memory: 44.58GiB(31.99%) tps: 83,793 tflops: 289.18 mfu: 29.24% global_avg_ntp_loss: 2.4796 global_avg_mtp_loss: 13.8057 +[titan] 2025-07-10 11:22:47,850 - root - INFO - lr: 3.0061e-05 gnorm: 1.05 [21:47:43< 0:12:28] +[titan] 2025-07-10 11:22:51,775 - root - INFO - step: 99060 loss: 16.2376 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4876 global_avg_mtp_loss: 13.7500 +[titan] 2025-07-10 11:22:51,775 - root - INFO - lr: 3.0060e-05 gnorm: 1.06 [21:47:47< 0:12:24] +[titan] 2025-07-10 11:22:55,685 - root - INFO - step: 99065 loss: 16.1207 memory: 44.58GiB(31.99%) tps: 83,818 tflops: 289.27 mfu: 29.25% global_avg_ntp_loss: 2.4533 global_avg_mtp_loss: 13.6674 +[titan] 2025-07-10 11:22:55,685 - root - INFO - lr: 3.0059e-05 gnorm: 1.07 [21:47:51< 0:12:20] +[titan] 2025-07-10 11:22:59,641 - root - INFO - step: 99070 loss: 16.1678 memory: 44.58GiB(31.99%) tps: 82,847 tflops: 285.92 mfu: 28.91% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.7112 +[titan] 2025-07-10 11:22:59,641 - root - INFO - lr: 3.0059e-05 gnorm: 1.06 [21:47:55< 0:12:16] +[titan] 2025-07-10 11:23:03,566 - root - INFO - step: 99075 loss: 16.1738 memory: 44.58GiB(31.99%) tps: 83,490 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4653 global_avg_mtp_loss: 13.7085 +[titan] 2025-07-10 11:23:03,566 - root - INFO - lr: 3.0058e-05 gnorm: 1.09 [21:47:58< 0:12:12] +[titan] 2025-07-10 11:23:07,455 - root - INFO - step: 99080 loss: 16.1558 memory: 44.58GiB(31.99%) tps: 84,270 tflops: 290.83 mfu: 29.41% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6956 +[titan] 2025-07-10 11:23:07,455 - root - INFO - lr: 3.0058e-05 gnorm: 1.05 [21:48:02< 0:12:08] +[titan] 2025-07-10 11:23:11,358 - root - INFO - step: 99085 loss: 16.1952 memory: 44.58GiB(31.99%) tps: 83,953 tflops: 289.74 mfu: 29.30% global_avg_ntp_loss: 2.4705 global_avg_mtp_loss: 13.7247 +[titan] 2025-07-10 11:23:11,358 - root - INFO - lr: 3.0057e-05 gnorm: 1.29 [21:48:06< 0:12:04] +[titan] 2025-07-10 11:23:15,301 - root - INFO - step: 99090 loss: 16.0270 memory: 44.58GiB(31.99%) tps: 83,114 tflops: 286.84 mfu: 29.00% global_avg_ntp_loss: 2.4346 global_avg_mtp_loss: 13.5924 +[titan] 2025-07-10 11:23:15,301 - root - INFO - lr: 3.0056e-05 gnorm: 1.03 [21:48:10< 0:12:00] +[titan] 2025-07-10 11:23:19,217 - root - INFO - step: 99095 loss: 15.9943 memory: 44.58GiB(31.99%) tps: 83,688 tflops: 288.82 mfu: 29.20% global_avg_ntp_loss: 2.4378 global_avg_mtp_loss: 13.5565 +[titan] 2025-07-10 11:23:19,217 - root - INFO - lr: 3.0056e-05 gnorm: 1.13 [21:48:14< 0:11:56] +[titan] 2025-07-10 11:23:22,363 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:23:23,157 - root - INFO - step: 99100 loss: 15.9927 memory: 44.58GiB(31.99%) tps: 83,173 tflops: 287.04 mfu: 29.02% global_avg_ntp_loss: 2.4296 global_avg_mtp_loss: 13.5631 +[titan] 2025-07-10 11:23:23,157 - root - INFO - lr: 3.0055e-05 gnorm: 1.15 [21:48:18< 0:11:52] +[titan] 2025-07-10 11:23:27,072 - root - INFO - step: 99105 loss: 16.4290 memory: 44.58GiB(31.99%) tps: 83,702 tflops: 288.87 mfu: 29.21% global_avg_ntp_loss: 2.5110 global_avg_mtp_loss: 13.9179 +[titan] 2025-07-10 11:23:27,073 - root - INFO - lr: 3.0054e-05 gnorm: 1.09 [21:48:22< 0:11:48] +[titan] 2025-07-10 11:23:30,999 - root - INFO - step: 99110 loss: 15.9387 memory: 44.58GiB(31.99%) tps: 83,448 tflops: 287.99 mfu: 29.12% global_avg_ntp_loss: 2.4301 global_avg_mtp_loss: 13.5086 +[titan] 2025-07-10 11:23:31,000 - root - INFO - lr: 3.0054e-05 gnorm: 1.12 [21:48:26< 0:11:44] +[titan] 2025-07-10 11:23:34,910 - root - INFO - step: 99115 loss: 16.2074 memory: 44.58GiB(31.99%) tps: 83,806 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4756 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 11:23:34,910 - root - INFO - lr: 3.0053e-05 gnorm: 1.07 [21:48:30< 0:11:41] +[titan] 2025-07-10 11:23:38,829 - root - INFO - step: 99120 loss: 16.0577 memory: 44.58GiB(31.99%) tps: 83,614 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.6156 +[titan] 2025-07-10 11:23:38,829 - root - INFO - lr: 3.0053e-05 gnorm: 1.04 [21:48:34< 0:11:37] +[titan] 2025-07-10 11:23:42,728 - root - INFO - step: 99125 loss: 16.3307 memory: 44.58GiB(31.99%) tps: 84,058 tflops: 290.10 mfu: 29.33% global_avg_ntp_loss: 2.4862 global_avg_mtp_loss: 13.8445 +[titan] 2025-07-10 11:23:42,728 - root - INFO - lr: 3.0052e-05 gnorm: 1.07 [21:48:38< 0:11:33] +[titan] 2025-07-10 11:23:46,665 - root - INFO - step: 99130 loss: 16.0080 memory: 44.58GiB(31.99%) tps: 83,235 tflops: 287.26 mfu: 29.05% global_avg_ntp_loss: 2.4313 global_avg_mtp_loss: 13.5767 +[titan] 2025-07-10 11:23:46,665 - root - INFO - lr: 3.0051e-05 gnorm: 1.08 [21:48:42< 0:11:29] +[titan] 2025-07-10 11:23:50,603 - root - INFO - step: 99135 loss: 16.3630 memory: 44.58GiB(31.99%) tps: 83,222 tflops: 287.21 mfu: 29.04% global_avg_ntp_loss: 2.4977 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 11:23:50,603 - root - INFO - lr: 3.0051e-05 gnorm: 1.05 [21:48:46< 0:11:25] +[titan] 2025-07-10 11:23:54,529 - root - INFO - step: 99140 loss: 16.3317 memory: 44.58GiB(31.99%) tps: 83,467 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4921 global_avg_mtp_loss: 13.8396 +[titan] 2025-07-10 11:23:54,529 - root - INFO - lr: 3.0050e-05 gnorm: 1.06 [21:48:49< 0:11:21] +[titan] 2025-07-10 11:23:58,444 - root - INFO - step: 99145 loss: 16.0641 memory: 44.58GiB(31.99%) tps: 83,715 tflops: 288.92 mfu: 29.21% global_avg_ntp_loss: 2.4427 global_avg_mtp_loss: 13.6214 +[titan] 2025-07-10 11:23:58,444 - root - INFO - lr: 3.0050e-05 gnorm: 1.05 [21:48:53< 0:11:17] +[titan] 2025-07-10 11:24:01,630 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:24:02,439 - root - INFO - step: 99150 loss: 16.3213 memory: 44.58GiB(31.99%) tps: 82,026 tflops: 283.08 mfu: 28.62% global_avg_ntp_loss: 2.4951 global_avg_mtp_loss: 13.8263 +[titan] 2025-07-10 11:24:02,439 - root - INFO - lr: 3.0049e-05 gnorm: 1.09 [21:48:57< 0:11:13] +[titan] 2025-07-10 11:24:06,367 - root - INFO - step: 99155 loss: 16.4341 memory: 44.58GiB(31.99%) tps: 83,422 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.5156 global_avg_mtp_loss: 13.9185 +[titan] 2025-07-10 11:24:06,368 - root - INFO - lr: 3.0049e-05 gnorm: 1.05 [21:49:01< 0:11:09] +[titan] 2025-07-10 11:24:10,304 - root - INFO - step: 99160 loss: 16.0584 memory: 44.58GiB(31.99%) tps: 83,240 tflops: 287.28 mfu: 29.05% global_avg_ntp_loss: 2.4401 global_avg_mtp_loss: 13.6183 +[titan] 2025-07-10 11:24:10,305 - root - INFO - lr: 3.0048e-05 gnorm: 1.09 [21:49:05< 0:11:05] +[titan] 2025-07-10 11:24:14,241 - root - INFO - step: 99165 loss: 16.0607 memory: 44.58GiB(31.99%) tps: 83,251 tflops: 287.31 mfu: 29.05% global_avg_ntp_loss: 2.4434 global_avg_mtp_loss: 13.6172 +[titan] 2025-07-10 11:24:14,241 - root - INFO - lr: 3.0047e-05 gnorm: 1.09 [21:49:09< 0:11:01] +[titan] 2025-07-10 11:24:18,153 - root - INFO - step: 99170 loss: 16.1638 memory: 44.58GiB(31.99%) tps: 83,769 tflops: 289.10 mfu: 29.23% global_avg_ntp_loss: 2.4807 global_avg_mtp_loss: 13.6830 +[titan] 2025-07-10 11:24:18,153 - root - INFO - lr: 3.0047e-05 gnorm: 1.13 [21:49:13< 0:10:57] +[titan] 2025-07-10 11:24:22,049 - root - INFO - step: 99175 loss: 15.9756 memory: 44.58GiB(31.99%) tps: 84,120 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4260 global_avg_mtp_loss: 13.5495 +[titan] 2025-07-10 11:24:22,049 - root - INFO - lr: 3.0046e-05 gnorm: 1.04 [21:49:17< 0:10:53] +[titan] 2025-07-10 11:24:25,949 - root - INFO - step: 99180 loss: 16.2166 memory: 44.58GiB(31.99%) tps: 84,014 tflops: 289.95 mfu: 29.32% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7429 +[titan] 2025-07-10 11:24:25,950 - root - INFO - lr: 3.0046e-05 gnorm: 1.05 [21:49:21< 0:10:49] +[titan] 2025-07-10 11:24:29,859 - root - INFO - step: 99185 loss: 16.4161 memory: 44.58GiB(31.99%) tps: 83,812 tflops: 289.25 mfu: 29.25% global_avg_ntp_loss: 2.5049 global_avg_mtp_loss: 13.9112 +[titan] 2025-07-10 11:24:29,860 - root - INFO - lr: 3.0045e-05 gnorm: 1.14 [21:49:25< 0:10:45] +[titan] 2025-07-10 11:24:33,761 - root - INFO - step: 99190 loss: 15.8854 memory: 44.58GiB(31.99%) tps: 83,991 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4054 global_avg_mtp_loss: 13.4800 +[titan] 2025-07-10 11:24:33,761 - root - INFO - lr: 3.0045e-05 gnorm: 1.04 [21:49:29< 0:10:41] +[titan] 2025-07-10 11:24:37,673 - root - INFO - step: 99195 loss: 16.0572 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4384 global_avg_mtp_loss: 13.6188 +[titan] 2025-07-10 11:24:37,673 - root - INFO - lr: 3.0044e-05 gnorm: 1.03 [21:49:33< 0:10:37] +[titan] 2025-07-10 11:24:40,791 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:24:41,583 - root - INFO - step: 99200 loss: 16.1884 memory: 44.58GiB(31.99%) tps: 83,815 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4675 global_avg_mtp_loss: 13.7210 +[titan] 2025-07-10 11:24:41,583 - root - INFO - lr: 3.0043e-05 gnorm: 1.04 [21:49:36< 0:10:33] +[titan] 2025-07-10 11:24:45,507 - root - INFO - step: 99205 loss: 16.0873 memory: 44.58GiB(31.99%) tps: 83,503 tflops: 288.18 mfu: 29.14% global_avg_ntp_loss: 2.4548 global_avg_mtp_loss: 13.6325 +[titan] 2025-07-10 11:24:45,507 - root - INFO - lr: 3.0043e-05 gnorm: 1.04 [21:49:40< 0:10:29] +[titan] 2025-07-10 11:24:49,452 - root - INFO - step: 99210 loss: 15.8772 memory: 44.58GiB(31.99%) tps: 83,079 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4131 global_avg_mtp_loss: 13.4642 +[titan] 2025-07-10 11:24:49,452 - root - INFO - lr: 3.0042e-05 gnorm: 1.09 [21:49:44< 0:10:25] +[titan] 2025-07-10 11:24:53,341 - root - INFO - step: 99215 loss: 16.1845 memory: 44.58GiB(31.99%) tps: 84,268 tflops: 290.82 mfu: 29.41% global_avg_ntp_loss: 2.4785 global_avg_mtp_loss: 13.7060 +[titan] 2025-07-10 11:24:53,341 - root - INFO - lr: 3.0042e-05 gnorm: 1.10 [21:49:48< 0:10:21] +[titan] 2025-07-10 11:24:57,269 - root - INFO - step: 99220 loss: 16.0901 memory: 44.58GiB(31.99%) tps: 83,417 tflops: 287.89 mfu: 29.11% global_avg_ntp_loss: 2.4566 global_avg_mtp_loss: 13.6335 +[titan] 2025-07-10 11:24:57,270 - root - INFO - lr: 3.0041e-05 gnorm: 1.08 [21:49:52< 0:10:17] +[titan] 2025-07-10 11:25:01,230 - root - INFO - step: 99225 loss: 16.1999 memory: 44.58GiB(31.99%) tps: 82,748 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.4726 global_avg_mtp_loss: 13.7273 +[titan] 2025-07-10 11:25:01,230 - root - INFO - lr: 3.0041e-05 gnorm: 1.12 [21:49:56< 0:10:13] +[titan] 2025-07-10 11:25:05,143 - root - INFO - step: 99230 loss: 16.0948 memory: 44.58GiB(31.99%) tps: 83,754 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6411 +[titan] 2025-07-10 11:25:05,143 - root - INFO - lr: 3.0040e-05 gnorm: 1.12 [21:50:00< 0:10:09] +[titan] 2025-07-10 11:25:09,051 - root - INFO - step: 99235 loss: 15.9571 memory: 44.58GiB(31.99%) tps: 83,845 tflops: 289.36 mfu: 29.26% global_avg_ntp_loss: 2.4274 global_avg_mtp_loss: 13.5297 +[titan] 2025-07-10 11:25:09,051 - root - INFO - lr: 3.0040e-05 gnorm: 1.14 [21:50:04< 0:10:05] +[titan] 2025-07-10 11:25:12,996 - root - INFO - step: 99240 loss: 16.3809 memory: 44.58GiB(31.99%) tps: 83,064 tflops: 286.67 mfu: 28.99% global_avg_ntp_loss: 2.5039 global_avg_mtp_loss: 13.8770 +[titan] 2025-07-10 11:25:12,997 - root - INFO - lr: 3.0039e-05 gnorm: 1.07 [21:50:08< 0:10:01] +[titan] 2025-07-10 11:25:16,901 - root - INFO - step: 99245 loss: 16.1643 memory: 44.58GiB(31.99%) tps: 83,922 tflops: 289.63 mfu: 29.29% global_avg_ntp_loss: 2.4768 global_avg_mtp_loss: 13.6874 +[titan] 2025-07-10 11:25:16,902 - root - INFO - lr: 3.0039e-05 gnorm: 1.09 [21:50:12< 0:09:58] +[titan] 2025-07-10 11:25:20,074 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:25:20,859 - root - INFO - step: 99250 loss: 16.3274 memory: 44.58GiB(31.99%) tps: 82,814 tflops: 285.81 mfu: 28.90% global_avg_ntp_loss: 2.5002 global_avg_mtp_loss: 13.8273 +[titan] 2025-07-10 11:25:20,859 - root - INFO - lr: 3.0038e-05 gnorm: 1.15 [21:50:16< 0:09:54] +[titan] 2025-07-10 11:25:24,763 - root - INFO - step: 99255 loss: 16.2042 memory: 44.58GiB(31.99%) tps: 83,940 tflops: 289.69 mfu: 29.29% global_avg_ntp_loss: 2.4800 global_avg_mtp_loss: 13.7242 +[titan] 2025-07-10 11:25:24,763 - root - INFO - lr: 3.0038e-05 gnorm: 1.05 [21:50:20< 0:09:50] +[titan] 2025-07-10 11:25:28,682 - root - INFO - step: 99260 loss: 16.1498 memory: 44.58GiB(31.99%) tps: 83,624 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4646 global_avg_mtp_loss: 13.6852 +[titan] 2025-07-10 11:25:28,682 - root - INFO - lr: 3.0037e-05 gnorm: 1.06 [21:50:24< 0:09:46] +[titan] 2025-07-10 11:25:32,642 - root - INFO - step: 99265 loss: 16.2023 memory: 44.58GiB(31.99%) tps: 82,749 tflops: 285.58 mfu: 28.88% global_avg_ntp_loss: 2.4790 global_avg_mtp_loss: 13.7233 +[titan] 2025-07-10 11:25:32,642 - root - INFO - lr: 3.0037e-05 gnorm: 1.09 [21:50:28< 0:09:42] +[titan] 2025-07-10 11:25:36,554 - root - INFO - step: 99270 loss: 15.9272 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4242 global_avg_mtp_loss: 13.5030 +[titan] 2025-07-10 11:25:36,554 - root - INFO - lr: 3.0036e-05 gnorm: 1.05 [21:50:31< 0:09:38] +[titan] 2025-07-10 11:25:40,460 - root - INFO - step: 99275 loss: 15.8695 memory: 44.58GiB(31.99%) tps: 83,900 tflops: 289.55 mfu: 29.28% global_avg_ntp_loss: 2.4160 global_avg_mtp_loss: 13.4535 +[titan] 2025-07-10 11:25:40,460 - root - INFO - lr: 3.0036e-05 gnorm: 1.08 [21:50:35< 0:09:34] +[titan] 2025-07-10 11:25:44,380 - root - INFO - step: 99280 loss: 16.2651 memory: 44.58GiB(31.99%) tps: 83,606 tflops: 288.54 mfu: 29.17% global_avg_ntp_loss: 2.5044 global_avg_mtp_loss: 13.7607 +[titan] 2025-07-10 11:25:44,380 - root - INFO - lr: 3.0035e-05 gnorm: 1.29 [21:50:39< 0:09:30] +[titan] 2025-07-10 11:25:48,291 - root - INFO - step: 99285 loss: 16.2432 memory: 44.58GiB(31.99%) tps: 83,780 tflops: 289.14 mfu: 29.24% global_avg_ntp_loss: 2.4746 global_avg_mtp_loss: 13.7687 +[titan] 2025-07-10 11:25:48,292 - root - INFO - lr: 3.0035e-05 gnorm: 1.07 [21:50:43< 0:09:26] +[titan] 2025-07-10 11:25:52,225 - root - INFO - step: 99290 loss: 15.9603 memory: 44.58GiB(31.99%) tps: 83,299 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4356 global_avg_mtp_loss: 13.5247 +[titan] 2025-07-10 11:25:52,226 - root - INFO - lr: 3.0034e-05 gnorm: 1.09 [21:50:47< 0:09:22] +[titan] 2025-07-10 11:25:56,149 - root - INFO - step: 99295 loss: 16.1280 memory: 44.58GiB(31.99%) tps: 83,533 tflops: 288.29 mfu: 29.15% global_avg_ntp_loss: 2.4641 global_avg_mtp_loss: 13.6639 +[titan] 2025-07-10 11:25:56,149 - root - INFO - lr: 3.0034e-05 gnorm: 1.11 [21:50:51< 0:09:18] +[titan] 2025-07-10 11:25:59,289 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:26:00,082 - root - INFO - step: 99300 loss: 16.2013 memory: 44.58GiB(31.99%) tps: 83,322 tflops: 287.56 mfu: 29.08% global_avg_ntp_loss: 2.4695 global_avg_mtp_loss: 13.7318 +[titan] 2025-07-10 11:26:00,082 - root - INFO - lr: 3.0033e-05 gnorm: 1.10 [21:50:55< 0:09:14] +[titan] 2025-07-10 11:26:04,001 - root - INFO - step: 99305 loss: 16.2409 memory: 44.58GiB(31.99%) tps: 83,616 tflops: 288.57 mfu: 29.18% global_avg_ntp_loss: 2.4658 global_avg_mtp_loss: 13.7751 +[titan] 2025-07-10 11:26:04,001 - root - INFO - lr: 3.0033e-05 gnorm: 1.04 [21:50:59< 0:09:10] +[titan] 2025-07-10 11:26:07,920 - root - INFO - step: 99310 loss: 15.8738 memory: 44.58GiB(31.99%) tps: 83,622 tflops: 288.59 mfu: 29.18% global_avg_ntp_loss: 2.4059 global_avg_mtp_loss: 13.4679 +[titan] 2025-07-10 11:26:07,920 - root - INFO - lr: 3.0032e-05 gnorm: 1.12 [21:51:03< 0:09:06] +[titan] 2025-07-10 11:26:11,853 - root - INFO - step: 99315 loss: 16.1176 memory: 44.58GiB(31.99%) tps: 83,331 tflops: 287.59 mfu: 29.08% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.6624 +[titan] 2025-07-10 11:26:11,853 - root - INFO - lr: 3.0032e-05 gnorm: 1.13 [21:51:07< 0:09:02] +[titan] 2025-07-10 11:26:15,783 - root - INFO - step: 99320 loss: 16.2381 memory: 44.58GiB(31.99%) tps: 83,383 tflops: 287.77 mfu: 29.10% global_avg_ntp_loss: 2.4817 global_avg_mtp_loss: 13.7564 +[titan] 2025-07-10 11:26:15,783 - root - INFO - lr: 3.0031e-05 gnorm: 1.07 [21:51:11< 0:08:58] +[titan] 2025-07-10 11:26:19,778 - root - INFO - step: 99325 loss: 15.9660 memory: 44.58GiB(31.99%) tps: 82,037 tflops: 283.12 mfu: 28.63% global_avg_ntp_loss: 2.4256 global_avg_mtp_loss: 13.5403 +[titan] 2025-07-10 11:26:19,778 - root - INFO - lr: 3.0031e-05 gnorm: 1.05 [21:51:15< 0:08:54] +[titan] 2025-07-10 11:26:22,271 - root - INFO - Dumping profiler traces at step 99328 +[titan] 2025-07-10 11:26:22,306 - root - INFO - Finished dumping profiler traces in 0.03 seconds +[titan] 2025-07-10 11:26:23,879 - root - INFO - step: 99330 loss: 16.1594 memory: 44.58GiB(31.99%) tps: 79,894 tflops: 275.73 mfu: 27.88% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.6981 +[titan] 2025-07-10 11:26:23,880 - root - INFO - lr: 3.0031e-05 gnorm: 1.11 [21:51:19< 0:08:50] +[titan] 2025-07-10 11:26:27,843 - root - INFO - step: 99335 loss: 16.1962 memory: 44.58GiB(31.99%) tps: 82,679 tflops: 285.34 mfu: 28.85% global_avg_ntp_loss: 2.4615 global_avg_mtp_loss: 13.7347 +[titan] 2025-07-10 11:26:27,843 - root - INFO - lr: 3.0030e-05 gnorm: 1.07 [21:51:23< 0:08:46] +[titan] 2025-07-10 11:26:31,771 - root - INFO - step: 99340 loss: 16.2541 memory: 44.58GiB(31.99%) tps: 83,438 tflops: 287.96 mfu: 29.12% global_avg_ntp_loss: 2.4815 global_avg_mtp_loss: 13.7725 +[titan] 2025-07-10 11:26:31,771 - root - INFO - lr: 3.0030e-05 gnorm: 1.08 [21:51:27< 0:08:42] +[titan] 2025-07-10 11:26:35,696 - root - INFO - step: 99345 loss: 16.1289 memory: 44.58GiB(31.99%) tps: 83,492 tflops: 288.15 mfu: 29.14% global_avg_ntp_loss: 2.4649 global_avg_mtp_loss: 13.6640 +[titan] 2025-07-10 11:26:35,696 - root - INFO - lr: 3.0029e-05 gnorm: 1.07 [21:51:31< 0:08:38] +[titan] 2025-07-10 11:26:38,810 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:26:39,601 - root - INFO - step: 99350 loss: 16.1405 memory: 44.58GiB(31.99%) tps: 83,917 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4650 global_avg_mtp_loss: 13.6754 +[titan] 2025-07-10 11:26:39,601 - root - INFO - lr: 3.0029e-05 gnorm: 1.07 [21:51:34< 0:08:34] +[titan] 2025-07-10 11:26:43,540 - root - INFO - step: 99355 loss: 15.9197 memory: 44.58GiB(31.99%) tps: 83,194 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4215 global_avg_mtp_loss: 13.4981 +[titan] 2025-07-10 11:26:43,540 - root - INFO - lr: 3.0028e-05 gnorm: 1.11 [21:51:38< 0:08:30] +[titan] 2025-07-10 11:26:47,434 - root - INFO - step: 99360 loss: 16.1145 memory: 44.58GiB(31.99%) tps: 84,157 tflops: 290.44 mfu: 29.37% global_avg_ntp_loss: 2.4527 global_avg_mtp_loss: 13.6618 +[titan] 2025-07-10 11:26:47,435 - root - INFO - lr: 3.0028e-05 gnorm: 1.06 [21:51:42< 0:08:26] +[titan] 2025-07-10 11:26:51,355 - root - INFO - step: 99365 loss: 16.1176 memory: 44.58GiB(31.99%) tps: 83,578 tflops: 288.44 mfu: 29.16% global_avg_ntp_loss: 2.4601 global_avg_mtp_loss: 13.6576 +[titan] 2025-07-10 11:26:51,356 - root - INFO - lr: 3.0027e-05 gnorm: 1.09 [21:51:46< 0:08:22] +[titan] 2025-07-10 11:26:55,302 - root - INFO - step: 99370 loss: 16.1877 memory: 44.58GiB(31.99%) tps: 83,042 tflops: 286.59 mfu: 28.98% global_avg_ntp_loss: 2.4616 global_avg_mtp_loss: 13.7261 +[titan] 2025-07-10 11:26:55,302 - root - INFO - lr: 3.0027e-05 gnorm: 1.08 [21:51:50< 0:08:19] +[titan] 2025-07-10 11:26:59,226 - root - INFO - step: 99375 loss: 16.0250 memory: 44.58GiB(31.99%) tps: 83,516 tflops: 288.23 mfu: 29.14% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5854 +[titan] 2025-07-10 11:26:59,226 - root - INFO - lr: 3.0027e-05 gnorm: 1.05 [21:51:54< 0:08:15] +[titan] 2025-07-10 11:27:03,135 - root - INFO - step: 99380 loss: 16.4422 memory: 44.58GiB(31.99%) tps: 83,823 tflops: 289.29 mfu: 29.25% global_avg_ntp_loss: 2.5232 global_avg_mtp_loss: 13.9190 +[titan] 2025-07-10 11:27:03,136 - root - INFO - lr: 3.0026e-05 gnorm: 1.09 [21:51:58< 0:08:11] +[titan] 2025-07-10 11:27:07,103 - root - INFO - step: 99385 loss: 16.2967 memory: 44.58GiB(31.99%) tps: 82,593 tflops: 285.04 mfu: 28.82% global_avg_ntp_loss: 2.4909 global_avg_mtp_loss: 13.8058 +[titan] 2025-07-10 11:27:07,103 - root - INFO - lr: 3.0026e-05 gnorm: 1.09 [21:52:02< 0:08:07] +[titan] 2025-07-10 11:27:10,995 - root - INFO - step: 99390 loss: 16.4221 memory: 44.58GiB(31.99%) tps: 84,202 tflops: 290.59 mfu: 29.38% global_avg_ntp_loss: 2.5112 global_avg_mtp_loss: 13.9109 +[titan] 2025-07-10 11:27:10,996 - root - INFO - lr: 3.0025e-05 gnorm: 1.07 [21:52:06< 0:08:03] +[titan] 2025-07-10 11:27:14,906 - root - INFO - step: 99395 loss: 16.2319 memory: 44.58GiB(31.99%) tps: 83,801 tflops: 289.21 mfu: 29.24% global_avg_ntp_loss: 2.4736 global_avg_mtp_loss: 13.7584 +[titan] 2025-07-10 11:27:14,906 - root - INFO - lr: 3.0025e-05 gnorm: 1.04 [21:52:10< 0:07:59] +[titan] 2025-07-10 11:27:18,045 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:27:18,836 - root - INFO - step: 99400 loss: 16.0319 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.75 mfu: 29.10% global_avg_ntp_loss: 2.4402 global_avg_mtp_loss: 13.5917 +[titan] 2025-07-10 11:27:18,837 - root - INFO - lr: 3.0024e-05 gnorm: 1.11 [21:52:14< 0:07:55] +[titan] 2025-07-10 11:27:22,753 - root - INFO - step: 99405 loss: 15.9937 memory: 44.58GiB(31.99%) tps: 83,666 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4428 global_avg_mtp_loss: 13.5509 +[titan] 2025-07-10 11:27:22,754 - root - INFO - lr: 3.0024e-05 gnorm: 1.09 [21:52:18< 0:07:51] +[titan] 2025-07-10 11:27:26,651 - root - INFO - step: 99410 loss: 16.1368 memory: 44.58GiB(31.99%) tps: 84,089 tflops: 290.20 mfu: 29.34% global_avg_ntp_loss: 2.4645 global_avg_mtp_loss: 13.6723 +[titan] 2025-07-10 11:27:26,651 - root - INFO - lr: 3.0024e-05 gnorm: 1.13 [21:52:22< 0:07:47] +[titan] 2025-07-10 11:27:30,558 - root - INFO - step: 99415 loss: 16.1997 memory: 44.58GiB(31.99%) tps: 83,870 tflops: 289.45 mfu: 29.27% global_avg_ntp_loss: 2.4882 global_avg_mtp_loss: 13.7115 +[titan] 2025-07-10 11:27:30,558 - root - INFO - lr: 3.0023e-05 gnorm: 1.15 [21:52:25< 0:07:43] +[titan] 2025-07-10 11:27:34,466 - root - INFO - step: 99420 loss: 16.2545 memory: 44.58GiB(31.99%) tps: 83,849 tflops: 289.38 mfu: 29.26% global_avg_ntp_loss: 2.4776 global_avg_mtp_loss: 13.7770 +[titan] 2025-07-10 11:27:34,467 - root - INFO - lr: 3.0023e-05 gnorm: 1.12 [21:52:29< 0:07:39] +[titan] 2025-07-10 11:27:38,380 - root - INFO - step: 99425 loss: 16.0429 memory: 44.58GiB(31.99%) tps: 83,748 tflops: 289.03 mfu: 29.22% global_avg_ntp_loss: 2.4612 global_avg_mtp_loss: 13.5818 +[titan] 2025-07-10 11:27:38,380 - root - INFO - lr: 3.0022e-05 gnorm: 1.44 [21:52:33< 0:07:35] +[titan] 2025-07-10 11:27:42,281 - root - INFO - step: 99430 loss: 15.9336 memory: 44.58GiB(31.99%) tps: 83,995 tflops: 289.88 mfu: 29.31% global_avg_ntp_loss: 2.4070 global_avg_mtp_loss: 13.5265 +[titan] 2025-07-10 11:27:42,281 - root - INFO - lr: 3.0022e-05 gnorm: 1.11 [21:52:37< 0:07:31] +[titan] 2025-07-10 11:27:46,177 - root - INFO - step: 99435 loss: 15.8959 memory: 44.58GiB(31.99%) tps: 84,119 tflops: 290.31 mfu: 29.35% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.4538 +[titan] 2025-07-10 11:27:46,177 - root - INFO - lr: 3.0022e-05 gnorm: 1.10 [21:52:41< 0:07:27] +[titan] 2025-07-10 11:27:50,088 - root - INFO - step: 99440 loss: 15.6181 memory: 44.58GiB(31.99%) tps: 83,787 tflops: 289.16 mfu: 29.24% global_avg_ntp_loss: 2.3772 global_avg_mtp_loss: 13.2409 +[titan] 2025-07-10 11:27:50,089 - root - INFO - lr: 3.0021e-05 gnorm: 1.12 [21:52:45< 0:07:23] +[titan] 2025-07-10 11:27:53,990 - root - INFO - step: 99445 loss: 16.1493 memory: 44.58GiB(31.99%) tps: 84,004 tflops: 289.91 mfu: 29.31% global_avg_ntp_loss: 2.4696 global_avg_mtp_loss: 13.6797 +[titan] 2025-07-10 11:27:53,990 - root - INFO - lr: 3.0021e-05 gnorm: 1.07 [21:52:49< 0:07:19] +[titan] 2025-07-10 11:27:57,098 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:27:57,890 - root - INFO - step: 99450 loss: 16.2243 memory: 44.58GiB(31.99%) tps: 84,012 tflops: 289.94 mfu: 29.32% global_avg_ntp_loss: 2.4781 global_avg_mtp_loss: 13.7462 +[titan] 2025-07-10 11:27:57,891 - root - INFO - lr: 3.0021e-05 gnorm: 1.11 [21:52:53< 0:07:15] +[titan] 2025-07-10 11:28:01,790 - root - INFO - step: 99455 loss: 16.3676 memory: 44.58GiB(31.99%) tps: 84,032 tflops: 290.01 mfu: 29.32% global_avg_ntp_loss: 2.4985 global_avg_mtp_loss: 13.8691 +[titan] 2025-07-10 11:28:01,791 - root - INFO - lr: 3.0020e-05 gnorm: 1.13 [21:52:57< 0:07:11] +[titan] 2025-07-10 11:28:05,693 - root - INFO - step: 99460 loss: 16.2529 memory: 44.58GiB(31.99%) tps: 83,976 tflops: 289.82 mfu: 29.30% global_avg_ntp_loss: 2.4894 global_avg_mtp_loss: 13.7635 +[titan] 2025-07-10 11:28:05,693 - root - INFO - lr: 3.0020e-05 gnorm: 1.08 [21:53:01< 0:07:07] +[titan] 2025-07-10 11:28:09,606 - root - INFO - step: 99465 loss: 16.1595 memory: 44.58GiB(31.99%) tps: 83,737 tflops: 288.99 mfu: 29.22% global_avg_ntp_loss: 2.4561 global_avg_mtp_loss: 13.7033 +[titan] 2025-07-10 11:28:09,607 - root - INFO - lr: 3.0019e-05 gnorm: 1.04 [21:53:04< 0:07:03] +[titan] 2025-07-10 11:28:13,531 - root - INFO - step: 99470 loss: 16.1486 memory: 44.58GiB(31.99%) tps: 83,508 tflops: 288.20 mfu: 29.14% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.6808 +[titan] 2025-07-10 11:28:13,531 - root - INFO - lr: 3.0019e-05 gnorm: 1.08 [21:53:08< 0:06:59] +[titan] 2025-07-10 11:28:17,466 - root - INFO - step: 99475 loss: 16.0085 memory: 44.58GiB(31.99%) tps: 83,274 tflops: 287.39 mfu: 29.06% global_avg_ntp_loss: 2.4310 global_avg_mtp_loss: 13.5776 +[titan] 2025-07-10 11:28:17,466 - root - INFO - lr: 3.0019e-05 gnorm: 1.10 [21:53:12< 0:06:55] +[titan] 2025-07-10 11:28:21,391 - root - INFO - step: 99480 loss: 15.9872 memory: 44.58GiB(31.99%) tps: 83,504 tflops: 288.19 mfu: 29.14% global_avg_ntp_loss: 2.4334 global_avg_mtp_loss: 13.5538 +[titan] 2025-07-10 11:28:21,391 - root - INFO - lr: 3.0018e-05 gnorm: 1.13 [21:53:16< 0:06:51] +[titan] 2025-07-10 11:28:25,317 - root - INFO - step: 99485 loss: 16.1435 memory: 44.58GiB(31.99%) tps: 83,468 tflops: 288.06 mfu: 29.13% global_avg_ntp_loss: 2.4569 global_avg_mtp_loss: 13.6865 +[titan] 2025-07-10 11:28:25,317 - root - INFO - lr: 3.0018e-05 gnorm: 1.07 [21:53:20< 0:06:47] +[titan] 2025-07-10 11:28:29,259 - root - INFO - step: 99490 loss: 16.0815 memory: 44.58GiB(31.99%) tps: 83,124 tflops: 286.87 mfu: 29.01% global_avg_ntp_loss: 2.4577 global_avg_mtp_loss: 13.6238 +[titan] 2025-07-10 11:28:29,260 - root - INFO - lr: 3.0018e-05 gnorm: 1.10 [21:53:24< 0:06:43] +[titan] 2025-07-10 11:28:33,201 - root - INFO - step: 99495 loss: 16.0119 memory: 44.58GiB(31.99%) tps: 83,150 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4276 global_avg_mtp_loss: 13.5843 +[titan] 2025-07-10 11:28:33,201 - root - INFO - lr: 3.0017e-05 gnorm: 1.05 [21:53:28< 0:06:40] +[titan] 2025-07-10 11:28:36,335 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:28:37,120 - root - INFO - step: 99500 loss: 16.3260 memory: 44.58GiB(31.99%) tps: 83,607 tflops: 288.54 mfu: 29.18% global_avg_ntp_loss: 2.4935 global_avg_mtp_loss: 13.8325 +[titan] 2025-07-10 11:28:37,121 - root - INFO - lr: 3.0017e-05 gnorm: 1.02 [21:53:32< 0:06:36] +[titan] 2025-07-10 11:28:41,049 - root - INFO - step: 99505 loss: 15.9688 memory: 44.58GiB(31.99%) tps: 83,409 tflops: 287.86 mfu: 29.11% global_avg_ntp_loss: 2.4253 global_avg_mtp_loss: 13.5435 +[titan] 2025-07-10 11:28:41,050 - root - INFO - lr: 3.0017e-05 gnorm: 1.11 [21:53:36< 0:06:32] +[titan] 2025-07-10 11:28:44,973 - root - INFO - step: 99510 loss: 16.2535 memory: 44.58GiB(31.99%) tps: 83,531 tflops: 288.28 mfu: 29.15% global_avg_ntp_loss: 2.4780 global_avg_mtp_loss: 13.7755 +[titan] 2025-07-10 11:28:44,973 - root - INFO - lr: 3.0016e-05 gnorm: 1.05 [21:53:40< 0:06:28] +[titan] 2025-07-10 11:28:48,901 - root - INFO - step: 99515 loss: 16.1979 memory: 44.58GiB(31.99%) tps: 83,421 tflops: 287.90 mfu: 29.11% global_avg_ntp_loss: 2.4676 global_avg_mtp_loss: 13.7302 +[titan] 2025-07-10 11:28:48,901 - root - INFO - lr: 3.0016e-05 gnorm: 1.09 [21:53:44< 0:06:24] +[titan] 2025-07-10 11:28:52,811 - root - INFO - step: 99520 loss: 16.1343 memory: 44.58GiB(31.99%) tps: 83,807 tflops: 289.23 mfu: 29.24% global_avg_ntp_loss: 2.4617 global_avg_mtp_loss: 13.6726 +[titan] 2025-07-10 11:28:52,812 - root - INFO - lr: 3.0016e-05 gnorm: 1.10 [21:53:48< 0:06:20] +[titan] 2025-07-10 11:28:56,713 - root - INFO - step: 99525 loss: 16.2041 memory: 44.58GiB(31.99%) tps: 83,992 tflops: 289.87 mfu: 29.31% global_avg_ntp_loss: 2.4734 global_avg_mtp_loss: 13.7307 +[titan] 2025-07-10 11:28:56,713 - root - INFO - lr: 3.0015e-05 gnorm: 1.07 [21:53:52< 0:06:16] +[titan] 2025-07-10 11:29:00,618 - root - INFO - step: 99530 loss: 16.1404 memory: 44.58GiB(31.99%) tps: 83,919 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4514 global_avg_mtp_loss: 13.6890 +[titan] 2025-07-10 11:29:00,619 - root - INFO - lr: 3.0015e-05 gnorm: 1.07 [21:53:55< 0:06:12] +[titan] 2025-07-10 11:29:04,546 - root - INFO - step: 99535 loss: 16.2108 memory: 44.58GiB(31.99%) tps: 83,434 tflops: 287.94 mfu: 29.11% global_avg_ntp_loss: 2.4631 global_avg_mtp_loss: 13.7478 +[titan] 2025-07-10 11:29:04,546 - root - INFO - lr: 3.0015e-05 gnorm: 1.03 [21:53:59< 0:06:08] +[titan] 2025-07-10 11:29:08,479 - root - INFO - step: 99540 loss: 16.0036 memory: 44.58GiB(31.99%) tps: 83,325 tflops: 287.57 mfu: 29.08% global_avg_ntp_loss: 2.4258 global_avg_mtp_loss: 13.5779 +[titan] 2025-07-10 11:29:08,479 - root - INFO - lr: 3.0014e-05 gnorm: 1.08 [21:54:03< 0:06:04] +[titan] 2025-07-10 11:29:12,452 - root - INFO - step: 99545 loss: 16.2009 memory: 44.58GiB(31.99%) tps: 82,479 tflops: 284.65 mfu: 28.78% global_avg_ntp_loss: 2.4599 global_avg_mtp_loss: 13.7410 +[titan] 2025-07-10 11:29:12,453 - root - INFO - lr: 3.0014e-05 gnorm: 1.06 [21:54:07< 0:06:00] +[titan] 2025-07-10 11:29:15,600 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:29:16,388 - root - INFO - step: 99550 loss: 16.1319 memory: 44.58GiB(31.99%) tps: 83,269 tflops: 287.37 mfu: 29.06% global_avg_ntp_loss: 2.4503 global_avg_mtp_loss: 13.6816 +[titan] 2025-07-10 11:29:16,388 - root - INFO - lr: 3.0014e-05 gnorm: 1.08 [21:54:11< 0:05:56] +[titan] 2025-07-10 11:29:20,306 - root - INFO - step: 99555 loss: 16.3198 memory: 44.58GiB(31.99%) tps: 83,638 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4912 global_avg_mtp_loss: 13.8287 +[titan] 2025-07-10 11:29:20,307 - root - INFO - lr: 3.0013e-05 gnorm: 1.07 [21:54:15< 0:05:52] +[titan] 2025-07-10 11:29:24,253 - root - INFO - step: 99560 loss: 16.1071 memory: 44.58GiB(31.99%) tps: 83,037 tflops: 286.57 mfu: 28.98% global_avg_ntp_loss: 2.4526 global_avg_mtp_loss: 13.6545 +[titan] 2025-07-10 11:29:24,253 - root - INFO - lr: 3.0013e-05 gnorm: 1.11 [21:54:19< 0:05:48] +[titan] 2025-07-10 11:29:28,197 - root - INFO - step: 99565 loss: 16.1871 memory: 44.58GiB(31.99%) tps: 83,080 tflops: 286.72 mfu: 28.99% global_avg_ntp_loss: 2.4704 global_avg_mtp_loss: 13.7167 +[titan] 2025-07-10 11:29:28,198 - root - INFO - lr: 3.0013e-05 gnorm: 1.05 [21:54:23< 0:05:44] +[titan] 2025-07-10 11:29:32,092 - root - INFO - step: 99570 loss: 16.4673 memory: 44.58GiB(31.99%) tps: 84,150 tflops: 290.42 mfu: 29.36% global_avg_ntp_loss: 2.5155 global_avg_mtp_loss: 13.9518 +[titan] 2025-07-10 11:29:32,092 - root - INFO - lr: 3.0013e-05 gnorm: 1.06 [21:54:27< 0:05:40] +[titan] 2025-07-10 11:29:36,013 - root - INFO - step: 99575 loss: 16.0844 memory: 44.58GiB(31.99%) tps: 83,580 tflops: 288.45 mfu: 29.17% global_avg_ntp_loss: 2.4651 global_avg_mtp_loss: 13.6194 +[titan] 2025-07-10 11:29:36,013 - root - INFO - lr: 3.0012e-05 gnorm: 1.07 [21:54:31< 0:05:36] +[titan] 2025-07-10 11:29:39,916 - root - INFO - step: 99580 loss: 16.1240 memory: 44.58GiB(31.99%) tps: 83,951 tflops: 289.73 mfu: 29.30% global_avg_ntp_loss: 2.4662 global_avg_mtp_loss: 13.6579 +[titan] 2025-07-10 11:29:39,917 - root - INFO - lr: 3.0012e-05 gnorm: 1.07 [21:54:35< 0:05:32] +[titan] 2025-07-10 11:29:43,823 - root - INFO - step: 99585 loss: 16.4455 memory: 44.58GiB(31.99%) tps: 83,887 tflops: 289.51 mfu: 29.27% global_avg_ntp_loss: 2.5334 global_avg_mtp_loss: 13.9121 +[titan] 2025-07-10 11:29:43,823 - root - INFO - lr: 3.0012e-05 gnorm: 1.12 [21:54:39< 0:05:28] +[titan] 2025-07-10 11:29:47,722 - root - INFO - step: 99590 loss: 16.2575 memory: 44.58GiB(31.99%) tps: 84,060 tflops: 290.11 mfu: 29.33% global_avg_ntp_loss: 2.4859 global_avg_mtp_loss: 13.7716 +[titan] 2025-07-10 11:29:47,722 - root - INFO - lr: 3.0011e-05 gnorm: 1.09 [21:54:43< 0:05:24] +[titan] 2025-07-10 11:29:51,640 - root - INFO - step: 99595 loss: 16.1203 memory: 44.58GiB(31.99%) tps: 83,625 tflops: 288.60 mfu: 29.18% global_avg_ntp_loss: 2.4614 global_avg_mtp_loss: 13.6589 +[titan] 2025-07-10 11:29:51,641 - root - INFO - lr: 3.0011e-05 gnorm: 1.06 [21:54:46< 0:05:20] +[titan] 2025-07-10 11:29:54,771 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:29:55,559 - root - INFO - step: 99600 loss: 15.9796 memory: 44.58GiB(31.99%) tps: 83,635 tflops: 288.64 mfu: 29.18% global_avg_ntp_loss: 2.4436 global_avg_mtp_loss: 13.5360 +[titan] 2025-07-10 11:29:55,559 - root - INFO - lr: 3.0011e-05 gnorm: 1.12 [21:54:50< 0:05:16] +[titan] 2025-07-10 11:29:59,466 - root - INFO - step: 99605 loss: 16.5037 memory: 44.58GiB(31.99%) tps: 83,869 tflops: 289.44 mfu: 29.27% global_avg_ntp_loss: 2.5237 global_avg_mtp_loss: 13.9800 +[titan] 2025-07-10 11:29:59,467 - root - INFO - lr: 3.0011e-05 gnorm: 1.11 [21:54:54< 0:05:12] +[titan] 2025-07-10 11:30:03,359 - root - INFO - step: 99610 loss: 16.2809 memory: 44.58GiB(31.99%) tps: 84,197 tflops: 290.58 mfu: 29.38% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.8039 +[titan] 2025-07-10 11:30:03,359 - root - INFO - lr: 3.0010e-05 gnorm: 1.02 [21:54:58< 0:05:08] +[titan] 2025-07-10 11:30:07,297 - root - INFO - step: 99615 loss: 16.2209 memory: 44.58GiB(31.99%) tps: 83,206 tflops: 287.16 mfu: 29.04% global_avg_ntp_loss: 2.4737 global_avg_mtp_loss: 13.7472 +[titan] 2025-07-10 11:30:07,297 - root - INFO - lr: 3.0010e-05 gnorm: 1.09 [21:55:02< 0:05:04] +[titan] 2025-07-10 11:30:11,228 - root - INFO - step: 99620 loss: 16.2539 memory: 44.58GiB(31.99%) tps: 83,379 tflops: 287.76 mfu: 29.10% global_avg_ntp_loss: 2.4730 global_avg_mtp_loss: 13.7809 +[titan] 2025-07-10 11:30:11,228 - root - INFO - lr: 3.0010e-05 gnorm: 1.12 [21:55:06< 0:05:00] +[titan] 2025-07-10 11:30:15,157 - root - INFO - step: 99625 loss: 16.2486 memory: 44.58GiB(31.99%) tps: 83,391 tflops: 287.80 mfu: 29.10% global_avg_ntp_loss: 2.4885 global_avg_mtp_loss: 13.7601 +[titan] 2025-07-10 11:30:15,158 - root - INFO - lr: 3.0010e-05 gnorm: 1.08 [21:55:10< 0:04:57] +[titan] 2025-07-10 11:30:19,063 - root - INFO - step: 99630 loss: 15.9985 memory: 44.58GiB(31.99%) tps: 83,916 tflops: 289.61 mfu: 29.28% global_avg_ntp_loss: 2.4326 global_avg_mtp_loss: 13.5659 +[titan] 2025-07-10 11:30:19,063 - root - INFO - lr: 3.0009e-05 gnorm: 1.11 [21:55:14< 0:04:53] +[titan] 2025-07-10 11:30:22,988 - root - INFO - step: 99635 loss: 16.1887 memory: 44.58GiB(31.99%) tps: 83,491 tflops: 288.14 mfu: 29.13% global_avg_ntp_loss: 2.4798 global_avg_mtp_loss: 13.7089 +[titan] 2025-07-10 11:30:22,988 - root - INFO - lr: 3.0009e-05 gnorm: 1.08 [21:55:18< 0:04:49] +[titan] 2025-07-10 11:30:26,903 - root - INFO - step: 99640 loss: 16.1549 memory: 44.58GiB(31.99%) tps: 83,707 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4635 global_avg_mtp_loss: 13.6915 +[titan] 2025-07-10 11:30:26,903 - root - INFO - lr: 3.0009e-05 gnorm: 1.07 [21:55:22< 0:04:45] +[titan] 2025-07-10 11:30:30,813 - root - INFO - step: 99645 loss: 16.2931 memory: 44.58GiB(31.99%) tps: 83,814 tflops: 289.26 mfu: 29.25% global_avg_ntp_loss: 2.4923 global_avg_mtp_loss: 13.8008 +[titan] 2025-07-10 11:30:30,813 - root - INFO - lr: 3.0009e-05 gnorm: 1.09 [21:55:26< 0:04:41] +[titan] 2025-07-10 11:30:33,930 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:30:34,720 - root - INFO - step: 99650 loss: 16.1395 memory: 44.58GiB(31.99%) tps: 83,878 tflops: 289.48 mfu: 29.27% global_avg_ntp_loss: 2.4524 global_avg_mtp_loss: 13.6871 +[titan] 2025-07-10 11:30:34,720 - root - INFO - lr: 3.0008e-05 gnorm: 1.07 [21:55:30< 0:04:37] +[titan] 2025-07-10 11:30:38,632 - root - INFO - step: 99655 loss: 16.3210 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4910 global_avg_mtp_loss: 13.8300 +[titan] 2025-07-10 11:30:38,632 - root - INFO - lr: 3.0008e-05 gnorm: 1.05 [21:55:33< 0:04:33] +[titan] 2025-07-10 11:30:42,574 - root - INFO - step: 99660 loss: 16.3095 memory: 44.58GiB(31.99%) tps: 83,127 tflops: 286.88 mfu: 29.01% global_avg_ntp_loss: 2.4845 global_avg_mtp_loss: 13.8251 +[titan] 2025-07-10 11:30:42,574 - root - INFO - lr: 3.0008e-05 gnorm: 1.05 [21:55:37< 0:04:29] +[titan] 2025-07-10 11:30:46,500 - root - INFO - step: 99665 loss: 16.0952 memory: 44.58GiB(31.99%) tps: 83,480 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.4564 global_avg_mtp_loss: 13.6388 +[titan] 2025-07-10 11:30:46,500 - root - INFO - lr: 3.0008e-05 gnorm: 1.14 [21:55:41< 0:04:25] +[titan] 2025-07-10 11:30:50,418 - root - INFO - step: 99670 loss: 16.1578 memory: 44.58GiB(31.99%) tps: 83,632 tflops: 288.63 mfu: 29.18% global_avg_ntp_loss: 2.4723 global_avg_mtp_loss: 13.6855 +[titan] 2025-07-10 11:30:50,419 - root - INFO - lr: 3.0007e-05 gnorm: 1.11 [21:55:45< 0:04:21] +[titan] 2025-07-10 11:30:54,331 - root - INFO - step: 99675 loss: 16.1346 memory: 44.58GiB(31.99%) tps: 83,755 tflops: 289.05 mfu: 29.23% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6665 +[titan] 2025-07-10 11:30:54,331 - root - INFO - lr: 3.0007e-05 gnorm: 1.15 [21:55:49< 0:04:17] +[titan] 2025-07-10 11:30:58,267 - root - INFO - step: 99680 loss: 15.9364 memory: 44.58GiB(31.99%) tps: 83,261 tflops: 287.35 mfu: 29.05% global_avg_ntp_loss: 2.4206 global_avg_mtp_loss: 13.5158 +[titan] 2025-07-10 11:30:58,267 - root - INFO - lr: 3.0007e-05 gnorm: 1.14 [21:55:53< 0:04:13] +[titan] 2025-07-10 11:31:02,170 - root - INFO - step: 99685 loss: 16.2467 memory: 44.58GiB(31.99%) tps: 83,968 tflops: 289.79 mfu: 29.30% global_avg_ntp_loss: 2.4694 global_avg_mtp_loss: 13.7774 +[titan] 2025-07-10 11:31:02,170 - root - INFO - lr: 3.0007e-05 gnorm: 1.07 [21:55:57< 0:04:09] +[titan] 2025-07-10 11:31:06,078 - root - INFO - step: 99690 loss: 16.2925 memory: 44.58GiB(31.99%) tps: 83,857 tflops: 289.41 mfu: 29.26% global_avg_ntp_loss: 2.4958 global_avg_mtp_loss: 13.7967 +[titan] 2025-07-10 11:31:06,078 - root - INFO - lr: 3.0007e-05 gnorm: 1.09 [21:56:01< 0:04:05] +[titan] 2025-07-10 11:31:09,999 - root - INFO - step: 99695 loss: 16.4198 memory: 44.58GiB(31.99%) tps: 83,573 tflops: 288.42 mfu: 29.16% global_avg_ntp_loss: 2.5163 global_avg_mtp_loss: 13.9035 +[titan] 2025-07-10 11:31:10,000 - root - INFO - lr: 3.0006e-05 gnorm: 1.08 [21:56:05< 0:04:01] +[titan] 2025-07-10 11:31:13,208 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:31:13,997 - root - INFO - step: 99700 loss: 16.1099 memory: 44.58GiB(31.99%) tps: 81,980 tflops: 282.93 mfu: 28.61% global_avg_ntp_loss: 2.4520 global_avg_mtp_loss: 13.6579 +[titan] 2025-07-10 11:31:13,997 - root - INFO - lr: 3.0006e-05 gnorm: 1.03 [21:56:09< 0:03:57] +[titan] 2025-07-10 11:31:17,897 - root - INFO - step: 99705 loss: 16.4659 memory: 44.58GiB(31.99%) tps: 84,034 tflops: 290.02 mfu: 29.32% global_avg_ntp_loss: 2.5231 global_avg_mtp_loss: 13.9428 +[titan] 2025-07-10 11:31:17,897 - root - INFO - lr: 3.0006e-05 gnorm: 1.11 [21:56:13< 0:03:53] +[titan] 2025-07-10 11:31:21,845 - root - INFO - step: 99710 loss: 16.2094 memory: 44.58GiB(31.99%) tps: 82,992 tflops: 286.42 mfu: 28.96% global_avg_ntp_loss: 2.4673 global_avg_mtp_loss: 13.7421 +[titan] 2025-07-10 11:31:21,846 - root - INFO - lr: 3.0006e-05 gnorm: 1.04 [21:56:17< 0:03:49] +[titan] 2025-07-10 11:31:25,761 - root - INFO - step: 99715 loss: 16.0351 memory: 44.58GiB(31.99%) tps: 83,696 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4489 global_avg_mtp_loss: 13.5862 +[titan] 2025-07-10 11:31:25,761 - root - INFO - lr: 3.0006e-05 gnorm: 1.12 [21:56:21< 0:03:45] +[titan] 2025-07-10 11:31:29,670 - root - INFO - step: 99720 loss: 16.1326 memory: 44.58GiB(31.99%) tps: 83,834 tflops: 289.33 mfu: 29.25% global_avg_ntp_loss: 2.4570 global_avg_mtp_loss: 13.6756 +[titan] 2025-07-10 11:31:29,670 - root - INFO - lr: 3.0005e-05 gnorm: 1.07 [21:56:25< 0:03:41] +[titan] 2025-07-10 11:31:33,600 - root - INFO - step: 99725 loss: 16.2206 memory: 44.58GiB(31.99%) tps: 83,394 tflops: 287.81 mfu: 29.10% global_avg_ntp_loss: 2.4956 global_avg_mtp_loss: 13.7250 +[titan] 2025-07-10 11:31:33,600 - root - INFO - lr: 3.0005e-05 gnorm: 1.18 [21:56:28< 0:03:37] +[titan] 2025-07-10 11:31:37,502 - root - INFO - step: 99730 loss: 15.7769 memory: 44.58GiB(31.99%) tps: 83,986 tflops: 289.85 mfu: 29.31% global_avg_ntp_loss: 2.3957 global_avg_mtp_loss: 13.3812 +[titan] 2025-07-10 11:31:37,502 - root - INFO - lr: 3.0005e-05 gnorm: 1.13 [21:56:32< 0:03:33] +[titan] 2025-07-10 11:31:41,437 - root - INFO - step: 99735 loss: 16.2166 memory: 44.58GiB(31.99%) tps: 83,271 tflops: 287.38 mfu: 29.06% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.7415 +[titan] 2025-07-10 11:31:41,437 - root - INFO - lr: 3.0005e-05 gnorm: 1.02 [21:56:36< 0:03:29] +[titan] 2025-07-10 11:31:45,359 - root - INFO - step: 99740 loss: 16.2442 memory: 44.58GiB(31.99%) tps: 83,563 tflops: 288.39 mfu: 29.16% global_avg_ntp_loss: 2.4879 global_avg_mtp_loss: 13.7563 +[titan] 2025-07-10 11:31:45,359 - root - INFO - lr: 3.0005e-05 gnorm: 1.09 [21:56:40< 0:03:25] +[titan] 2025-07-10 11:31:49,310 - root - INFO - step: 99745 loss: 16.4273 memory: 44.58GiB(31.99%) tps: 82,942 tflops: 286.25 mfu: 28.94% global_avg_ntp_loss: 2.5119 global_avg_mtp_loss: 13.9154 +[titan] 2025-07-10 11:31:49,310 - root - INFO - lr: 3.0004e-05 gnorm: 1.16 [21:56:44< 0:03:21] +[titan] 2025-07-10 11:31:52,422 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:31:53,205 - root - INFO - step: 99750 loss: 16.0285 memory: 44.58GiB(31.99%) tps: 84,131 tflops: 290.35 mfu: 29.36% global_avg_ntp_loss: 2.4319 global_avg_mtp_loss: 13.5966 +[titan] 2025-07-10 11:31:53,206 - root - INFO - lr: 3.0004e-05 gnorm: 1.05 [21:56:48< 0:03:18] +[titan] 2025-07-10 11:31:57,117 - root - INFO - step: 99755 loss: 16.0519 memory: 44.58GiB(31.99%) tps: 83,775 tflops: 289.12 mfu: 29.23% global_avg_ntp_loss: 2.4497 global_avg_mtp_loss: 13.6023 +[titan] 2025-07-10 11:31:57,117 - root - INFO - lr: 3.0004e-05 gnorm: 1.09 [21:56:52< 0:03:14] +[titan] 2025-07-10 11:32:01,058 - root - INFO - step: 99760 loss: 16.0256 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4329 global_avg_mtp_loss: 13.5927 +[titan] 2025-07-10 11:32:01,058 - root - INFO - lr: 3.0004e-05 gnorm: 1.07 [21:56:56< 0:03:10] +[titan] 2025-07-10 11:32:04,983 - root - INFO - step: 99765 loss: 15.8093 memory: 44.58GiB(31.99%) tps: 83,478 tflops: 288.10 mfu: 29.13% global_avg_ntp_loss: 2.3998 global_avg_mtp_loss: 13.4095 +[titan] 2025-07-10 11:32:04,984 - root - INFO - lr: 3.0004e-05 gnorm: 1.04 [21:57:00< 0:03:06] +[titan] 2025-07-10 11:32:08,941 - root - INFO - step: 99770 loss: 16.3034 memory: 44.58GiB(31.99%) tps: 82,793 tflops: 285.73 mfu: 28.89% global_avg_ntp_loss: 2.4946 global_avg_mtp_loss: 13.8088 +[titan] 2025-07-10 11:32:08,942 - root - INFO - lr: 3.0004e-05 gnorm: 1.05 [21:57:04< 0:03:02] +[titan] 2025-07-10 11:32:12,865 - root - INFO - step: 99775 loss: 16.0260 memory: 44.58GiB(31.99%) tps: 83,525 tflops: 288.26 mfu: 29.15% global_avg_ntp_loss: 2.4396 global_avg_mtp_loss: 13.5864 +[titan] 2025-07-10 11:32:12,865 - root - INFO - lr: 3.0003e-05 gnorm: 1.15 [21:57:08< 0:02:58] +[titan] 2025-07-10 11:32:16,779 - root - INFO - step: 99780 loss: 16.2941 memory: 44.58GiB(31.99%) tps: 83,733 tflops: 288.98 mfu: 29.22% global_avg_ntp_loss: 2.4931 global_avg_mtp_loss: 13.8010 +[titan] 2025-07-10 11:32:16,779 - root - INFO - lr: 3.0003e-05 gnorm: 1.08 [21:57:12< 0:02:54] +[titan] 2025-07-10 11:32:20,712 - root - INFO - step: 99785 loss: 16.0549 memory: 44.58GiB(31.99%) tps: 83,315 tflops: 287.53 mfu: 29.07% global_avg_ntp_loss: 2.4504 global_avg_mtp_loss: 13.6045 +[titan] 2025-07-10 11:32:20,713 - root - INFO - lr: 3.0003e-05 gnorm: 1.11 [21:57:16< 0:02:50] +[titan] 2025-07-10 11:32:24,666 - root - INFO - step: 99790 loss: 16.0210 memory: 44.58GiB(31.99%) tps: 82,889 tflops: 286.06 mfu: 28.92% global_avg_ntp_loss: 2.4435 global_avg_mtp_loss: 13.5775 +[titan] 2025-07-10 11:32:24,666 - root - INFO - lr: 3.0003e-05 gnorm: 1.08 [21:57:20< 0:02:46] +[titan] 2025-07-10 11:32:28,581 - root - INFO - step: 99795 loss: 16.2402 memory: 44.58GiB(31.99%) tps: 83,694 tflops: 288.84 mfu: 29.21% global_avg_ntp_loss: 2.4678 global_avg_mtp_loss: 13.7724 +[titan] 2025-07-10 11:32:28,582 - root - INFO - lr: 3.0003e-05 gnorm: 1.05 [21:57:23< 0:02:42] +[titan] 2025-07-10 11:32:31,732 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:32:32,522 - root - INFO - step: 99800 loss: 15.9145 memory: 44.58GiB(31.99%) tps: 83,169 tflops: 287.03 mfu: 29.02% global_avg_ntp_loss: 2.4252 global_avg_mtp_loss: 13.4893 +[titan] 2025-07-10 11:32:32,522 - root - INFO - lr: 3.0003e-05 gnorm: 1.05 [21:57:27< 0:02:38] +[titan] 2025-07-10 11:32:36,418 - root - INFO - step: 99805 loss: 16.2895 memory: 44.58GiB(31.99%) tps: 84,117 tflops: 290.30 mfu: 29.35% global_avg_ntp_loss: 2.4811 global_avg_mtp_loss: 13.8084 +[titan] 2025-07-10 11:32:36,418 - root - INFO - lr: 3.0003e-05 gnorm: 1.08 [21:57:31< 0:02:34] +[titan] 2025-07-10 11:32:40,309 - root - INFO - step: 99810 loss: 16.2615 memory: 44.58GiB(31.99%) tps: 84,223 tflops: 290.67 mfu: 29.39% global_avg_ntp_loss: 2.4865 global_avg_mtp_loss: 13.7750 +[titan] 2025-07-10 11:32:40,309 - root - INFO - lr: 3.0002e-05 gnorm: 1.08 [21:57:35< 0:02:30] +[titan] 2025-07-10 11:32:44,231 - root - INFO - step: 99815 loss: 16.1669 memory: 44.58GiB(31.99%) tps: 83,544 tflops: 288.32 mfu: 29.15% global_avg_ntp_loss: 2.4575 global_avg_mtp_loss: 13.7094 +[titan] 2025-07-10 11:32:44,232 - root - INFO - lr: 3.0002e-05 gnorm: 1.05 [21:57:39< 0:02:26] +[titan] 2025-07-10 11:32:48,155 - root - INFO - step: 99820 loss: 16.2782 memory: 44.58GiB(31.99%) tps: 83,519 tflops: 288.24 mfu: 29.14% global_avg_ntp_loss: 2.4837 global_avg_mtp_loss: 13.7944 +[titan] 2025-07-10 11:32:48,156 - root - INFO - lr: 3.0002e-05 gnorm: 1.04 [21:57:43< 0:02:22] +[titan] 2025-07-10 11:32:52,080 - root - INFO - step: 99825 loss: 16.3717 memory: 44.58GiB(31.99%) tps: 83,501 tflops: 288.17 mfu: 29.14% global_avg_ntp_loss: 2.5064 global_avg_mtp_loss: 13.8653 +[titan] 2025-07-10 11:32:52,080 - root - INFO - lr: 3.0002e-05 gnorm: 1.14 [21:57:47< 0:02:18] +[titan] 2025-07-10 11:32:56,019 - root - INFO - step: 99830 loss: 16.0075 memory: 44.58GiB(31.99%) tps: 83,195 tflops: 287.12 mfu: 29.03% global_avg_ntp_loss: 2.4305 global_avg_mtp_loss: 13.5770 +[titan] 2025-07-10 11:32:56,019 - root - INFO - lr: 3.0002e-05 gnorm: 1.10 [21:57:51< 0:02:14] +[titan] 2025-07-10 11:32:59,937 - root - INFO - step: 99835 loss: 16.1131 memory: 44.58GiB(31.99%) tps: 83,637 tflops: 288.65 mfu: 29.19% global_avg_ntp_loss: 2.4681 global_avg_mtp_loss: 13.6450 +[titan] 2025-07-10 11:32:59,938 - root - INFO - lr: 3.0002e-05 gnorm: 1.06 [21:57:55< 0:02:10] +[titan] 2025-07-10 11:33:03,980 - root - INFO - step: 99840 loss: 16.3633 memory: 44.58GiB(31.99%) tps: 81,062 tflops: 279.76 mfu: 28.29% global_avg_ntp_loss: 2.5050 global_avg_mtp_loss: 13.8582 +[titan] 2025-07-10 11:33:03,981 - root - INFO - lr: 3.0002e-05 gnorm: 1.06 [21:57:59< 0:02:06] +[titan] 2025-07-10 11:33:04,139 - root - INFO - Dumping profiler traces at step 99840 +[titan] 2025-07-10 11:33:04,174 - root - INFO - Finished dumping profiler traces in 0.04 seconds +[titan] 2025-07-10 11:33:08,117 - root - INFO - step: 99845 loss: 15.9118 memory: 44.58GiB(31.99%) tps: 79,217 tflops: 273.39 mfu: 27.64% global_avg_ntp_loss: 2.4216 global_avg_mtp_loss: 13.4902 +[titan] 2025-07-10 11:33:08,117 - root - INFO - lr: 3.0002e-05 gnorm: 1.11 [21:58:03< 0:02:02] +[titan] 2025-07-10 11:33:11,224 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:33:12,041 - root - INFO - step: 99850 loss: 16.2778 memory: 44.58GiB(31.99%) tps: 83,513 tflops: 288.22 mfu: 29.14% global_avg_ntp_loss: 2.4915 global_avg_mtp_loss: 13.7863 +[titan] 2025-07-10 11:33:12,042 - root - INFO - lr: 3.0002e-05 gnorm: 1.09 [21:58:07< 0:01:58] +[titan] 2025-07-10 11:33:15,983 - root - INFO - step: 99855 loss: 16.1622 memory: 44.58GiB(31.99%) tps: 83,133 tflops: 286.91 mfu: 29.01% global_avg_ntp_loss: 2.4751 global_avg_mtp_loss: 13.6872 +[titan] 2025-07-10 11:33:15,984 - root - INFO - lr: 3.0001e-05 gnorm: 1.07 [21:58:11< 0:01:54] +[titan] 2025-07-10 11:33:19,923 - root - INFO - step: 99860 loss: 16.2725 memory: 44.58GiB(31.99%) tps: 83,176 tflops: 287.06 mfu: 29.02% global_avg_ntp_loss: 2.4821 global_avg_mtp_loss: 13.7903 +[titan] 2025-07-10 11:33:19,924 - root - INFO - lr: 3.0001e-05 gnorm: 1.06 [21:58:15< 0:01:50] +[titan] 2025-07-10 11:33:23,837 - root - INFO - step: 99865 loss: 16.3166 memory: 44.58GiB(31.99%) tps: 83,730 tflops: 288.97 mfu: 29.22% global_avg_ntp_loss: 2.4979 global_avg_mtp_loss: 13.8187 +[titan] 2025-07-10 11:33:23,838 - root - INFO - lr: 3.0001e-05 gnorm: 1.05 [21:58:19< 0:01:46] +[titan] 2025-07-10 11:33:27,740 - root - INFO - step: 99870 loss: 16.2244 memory: 44.58GiB(31.99%) tps: 83,960 tflops: 289.76 mfu: 29.30% global_avg_ntp_loss: 2.4687 global_avg_mtp_loss: 13.7558 +[titan] 2025-07-10 11:33:27,741 - root - INFO - lr: 3.0001e-05 gnorm: 1.04 [21:58:23< 0:01:42] +[titan] 2025-07-10 11:33:31,684 - root - INFO - step: 99875 loss: 16.0756 memory: 44.58GiB(31.99%) tps: 83,097 tflops: 286.78 mfu: 29.00% global_avg_ntp_loss: 2.4537 global_avg_mtp_loss: 13.6219 +[titan] 2025-07-10 11:33:31,685 - root - INFO - lr: 3.0001e-05 gnorm: 1.09 [21:58:27< 0:01:39] +[titan] 2025-07-10 11:33:35,587 - root - INFO - step: 99880 loss: 16.0232 memory: 44.58GiB(31.99%) tps: 83,966 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4478 global_avg_mtp_loss: 13.5754 +[titan] 2025-07-10 11:33:35,588 - root - INFO - lr: 3.0001e-05 gnorm: 1.10 [21:58:30< 0:01:35] +[titan] 2025-07-10 11:33:39,499 - root - INFO - step: 99885 loss: 16.2165 memory: 44.58GiB(31.99%) tps: 83,782 tflops: 289.15 mfu: 29.24% global_avg_ntp_loss: 2.4750 global_avg_mtp_loss: 13.7415 +[titan] 2025-07-10 11:33:39,499 - root - INFO - lr: 3.0001e-05 gnorm: 1.17 [21:58:34< 0:01:31] +[titan] 2025-07-10 11:33:43,404 - root - INFO - step: 99890 loss: 16.1587 memory: 44.58GiB(31.99%) tps: 83,918 tflops: 289.62 mfu: 29.28% global_avg_ntp_loss: 2.4609 global_avg_mtp_loss: 13.6979 +[titan] 2025-07-10 11:33:43,404 - root - INFO - lr: 3.0001e-05 gnorm: 1.09 [21:58:38< 0:01:27] +[titan] 2025-07-10 11:33:47,319 - root - INFO - step: 99895 loss: 16.2310 memory: 44.58GiB(31.99%) tps: 83,704 tflops: 288.88 mfu: 29.21% global_avg_ntp_loss: 2.4760 global_avg_mtp_loss: 13.7550 +[titan] 2025-07-10 11:33:47,319 - root - INFO - lr: 3.0001e-05 gnorm: 1.05 [21:58:42< 0:01:23] +[titan] 2025-07-10 11:33:50,485 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:33:51,285 - root - INFO - step: 99900 loss: 16.0222 memory: 44.58GiB(31.99%) tps: 82,639 tflops: 285.20 mfu: 28.84% global_avg_ntp_loss: 2.4358 global_avg_mtp_loss: 13.5864 +[titan] 2025-07-10 11:33:51,285 - root - INFO - lr: 3.0001e-05 gnorm: 1.09 [21:58:46< 0:01:19] +[titan] 2025-07-10 11:33:55,237 - root - INFO - step: 99905 loss: 15.9778 memory: 44.58GiB(31.99%) tps: 82,922 tflops: 286.18 mfu: 28.94% global_avg_ntp_loss: 2.4421 global_avg_mtp_loss: 13.5357 +[titan] 2025-07-10 11:33:55,237 - root - INFO - lr: 3.0001e-05 gnorm: 1.10 [21:58:50< 0:01:15] +[titan] 2025-07-10 11:33:59,158 - root - INFO - step: 99910 loss: 15.8846 memory: 44.58GiB(31.99%) tps: 83,576 tflops: 288.43 mfu: 29.16% global_avg_ntp_loss: 2.4221 global_avg_mtp_loss: 13.4625 +[titan] 2025-07-10 11:33:59,158 - root - INFO - lr: 3.0001e-05 gnorm: 1.06 [21:58:54< 0:01:11] +[titan] 2025-07-10 11:34:03,075 - root - INFO - step: 99915 loss: 16.1565 memory: 44.58GiB(31.99%) tps: 83,667 tflops: 288.75 mfu: 29.20% global_avg_ntp_loss: 2.4611 global_avg_mtp_loss: 13.6953 +[titan] 2025-07-10 11:34:03,075 - root - INFO - lr: 3.0000e-05 gnorm: 1.07 [21:58:58< 0:01:07] +[titan] 2025-07-10 11:34:07,009 - root - INFO - step: 99920 loss: 15.9907 memory: 44.58GiB(31.99%) tps: 83,300 tflops: 287.48 mfu: 29.07% global_avg_ntp_loss: 2.4317 global_avg_mtp_loss: 13.5590 +[titan] 2025-07-10 11:34:07,009 - root - INFO - lr: 3.0000e-05 gnorm: 1.07 [21:59:02< 0:01:03] +[titan] 2025-07-10 11:34:10,918 - root - INFO - step: 99925 loss: 16.1222 memory: 44.58GiB(31.99%) tps: 83,841 tflops: 289.35 mfu: 29.26% global_avg_ntp_loss: 2.4576 global_avg_mtp_loss: 13.6646 +[titan] 2025-07-10 11:34:10,918 - root - INFO - lr: 3.0000e-05 gnorm: 1.08 [21:59:06< 0:00:59] +[titan] 2025-07-10 11:34:14,842 - root - INFO - step: 99930 loss: 16.1895 memory: 44.58GiB(31.99%) tps: 83,511 tflops: 288.21 mfu: 29.14% global_avg_ntp_loss: 2.4777 global_avg_mtp_loss: 13.7117 +[titan] 2025-07-10 11:34:14,842 - root - INFO - lr: 3.0000e-05 gnorm: 1.10 [21:59:10< 0:00:55] +[titan] 2025-07-10 11:34:18,757 - root - INFO - step: 99935 loss: 15.9273 memory: 44.58GiB(31.99%) tps: 83,708 tflops: 288.89 mfu: 29.21% global_avg_ntp_loss: 2.4130 global_avg_mtp_loss: 13.5143 +[titan] 2025-07-10 11:34:18,757 - root - INFO - lr: 3.0000e-05 gnorm: 1.08 [21:59:14< 0:00:51] +[titan] 2025-07-10 11:34:22,698 - root - INFO - step: 99940 loss: 16.3015 memory: 44.58GiB(31.99%) tps: 83,151 tflops: 286.97 mfu: 29.02% global_avg_ntp_loss: 2.4899 global_avg_mtp_loss: 13.8116 +[titan] 2025-07-10 11:34:22,698 - root - INFO - lr: 3.0000e-05 gnorm: 1.03 [21:59:18< 0:00:47] +[titan] 2025-07-10 11:34:26,601 - root - INFO - step: 99945 loss: 15.8452 memory: 44.58GiB(31.99%) tps: 83,967 tflops: 289.78 mfu: 29.30% global_avg_ntp_loss: 2.4121 global_avg_mtp_loss: 13.4331 +[titan] 2025-07-10 11:34:26,601 - root - INFO - lr: 3.0000e-05 gnorm: 1.11 [21:59:21< 0:00:43] +[titan] 2025-07-10 11:34:29,726 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:34:30,516 - root - INFO - step: 99950 loss: 16.1475 memory: 44.58GiB(31.99%) tps: 83,695 tflops: 288.85 mfu: 29.21% global_avg_ntp_loss: 2.4554 global_avg_mtp_loss: 13.6920 +[titan] 2025-07-10 11:34:30,517 - root - INFO - lr: 3.0000e-05 gnorm: 1.12 [21:59:25< 0:00:39] +[titan] 2025-07-10 11:34:34,417 - root - INFO - step: 99955 loss: 16.2684 memory: 44.58GiB(31.99%) tps: 84,018 tflops: 289.96 mfu: 29.32% global_avg_ntp_loss: 2.4770 global_avg_mtp_loss: 13.7914 +[titan] 2025-07-10 11:34:34,417 - root - INFO - lr: 3.0000e-05 gnorm: 1.10 [21:59:29< 0:00:35] +[titan] 2025-07-10 11:34:38,322 - root - INFO - step: 99960 loss: 16.2774 memory: 44.58GiB(31.99%) tps: 83,909 tflops: 289.59 mfu: 29.28% global_avg_ntp_loss: 2.4929 global_avg_mtp_loss: 13.7844 +[titan] 2025-07-10 11:34:38,323 - root - INFO - lr: 3.0000e-05 gnorm: 1.08 [21:59:33< 0:00:31] +[titan] 2025-07-10 11:34:42,228 - root - INFO - step: 99965 loss: 15.9990 memory: 44.58GiB(31.99%) tps: 83,903 tflops: 289.56 mfu: 29.28% global_avg_ntp_loss: 2.4304 global_avg_mtp_loss: 13.5686 +[titan] 2025-07-10 11:34:42,229 - root - INFO - lr: 3.0000e-05 gnorm: 1.08 [21:59:37< 0:00:27] +[titan] 2025-07-10 11:34:46,126 - root - INFO - step: 99970 loss: 15.9743 memory: 44.58GiB(31.99%) tps: 84,074 tflops: 290.15 mfu: 29.34% global_avg_ntp_loss: 2.4581 global_avg_mtp_loss: 13.5162 +[titan] 2025-07-10 11:34:46,127 - root - INFO - lr: 3.0000e-05 gnorm: 1.13 [21:59:41< 0:00:23] +[titan] 2025-07-10 11:34:50,039 - root - INFO - step: 99975 loss: 16.3188 memory: 44.58GiB(31.99%) tps: 83,761 tflops: 289.07 mfu: 29.23% global_avg_ntp_loss: 2.4848 global_avg_mtp_loss: 13.8340 +[titan] 2025-07-10 11:34:50,039 - root - INFO - lr: 3.0000e-05 gnorm: 1.07 [21:59:45< 0:00:19] +[titan] 2025-07-10 11:34:53,956 - root - INFO - step: 99980 loss: 16.2791 memory: 44.58GiB(31.99%) tps: 83,663 tflops: 288.73 mfu: 29.19% global_avg_ntp_loss: 2.4767 global_avg_mtp_loss: 13.8023 +[titan] 2025-07-10 11:34:53,956 - root - INFO - lr: 3.0000e-05 gnorm: 1.08 [21:59:49< 0:00:15] +[titan] 2025-07-10 11:34:57,884 - root - INFO - step: 99985 loss: 15.9571 memory: 44.58GiB(31.99%) tps: 83,430 tflops: 287.93 mfu: 29.11% global_avg_ntp_loss: 2.4383 global_avg_mtp_loss: 13.5188 +[titan] 2025-07-10 11:34:57,884 - root - INFO - lr: 3.0000e-05 gnorm: 1.12 [21:59:53< 0:00:11] +[titan] 2025-07-10 11:35:01,788 - root - INFO - step: 99990 loss: 16.1088 memory: 44.58GiB(31.99%) tps: 83,932 tflops: 289.66 mfu: 29.29% global_avg_ntp_loss: 2.4552 global_avg_mtp_loss: 13.6536 +[titan] 2025-07-10 11:35:01,789 - root - INFO - lr: 3.0000e-05 gnorm: 1.11 [21:59:57< 0:00:07] +[titan] 2025-07-10 11:35:05,692 - root - INFO - step: 99995 loss: 16.2336 memory: 44.58GiB(31.99%) tps: 83,946 tflops: 289.71 mfu: 29.29% global_avg_ntp_loss: 2.4709 global_avg_mtp_loss: 13.7626 +[titan] 2025-07-10 11:35:05,693 - root - INFO - lr: 3.0000e-05 gnorm: 1.09 [22:00:01< 0:00:03] +[titan] 2025-07-10 11:35:08,827 - root - INFO - [GC] Peforming periodical GC collection. 0.00 seconds. +[titan] 2025-07-10 11:35:09,625 - root - INFO - step: 100000 loss: 16.1056 memory: 44.58GiB(31.99%) tps: 83,328 tflops: 287.58 mfu: 29.08% global_avg_ntp_loss: 2.4515 global_avg_mtp_loss: 13.6541 +[titan] 2025-07-10 11:35:09,625 - root - INFO - lr: 3.0000e-05 gnorm: 1.02 [22:00:04< 0:00:00] +[titan] 2025-07-10 11:35:09,625 - root - INFO - Saving the checkpoint (or staging if async is enabled). +[titan] 2025-07-10 11:35:09,626 - root - INFO - Saving a full checkpoint at last step, step 100000. +[titan] 2025-07-10 11:35:12,142 - root - INFO - [GC] GC collection invoked by checkpointer. 0.01 seconds. +[titan] 2025-07-10 11:35:12,143 - root - INFO - Finished saving the checkpoint (or staging if async is enabled)in 2.52 seconds. +[titan] 2025-07-10 11:35:12,143 - root - INFO - Training completed