# HELP python_gc_objects_collected_total Objects collected during gc # TYPE python_gc_objects_collected_total counter python_gc_objects_collected_total{generation="0"} 75677.0 python_gc_objects_collected_total{generation="1"} 7214.0 python_gc_objects_collected_total{generation="2"} 428.0 # HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC # TYPE python_gc_objects_uncollectable_total counter python_gc_objects_uncollectable_total{generation="0"} 0.0 python_gc_objects_uncollectable_total{generation="1"} 0.0 python_gc_objects_uncollectable_total{generation="2"} 0.0 # HELP python_gc_collections_total Number of times this generation was collected # TYPE python_gc_collections_total counter python_gc_collections_total{generation="0"} 58656.0 python_gc_collections_total{generation="1"} 5332.0 python_gc_collections_total{generation="2"} 215.0 # HELP python_info Python platform information # TYPE python_info gauge python_info{implementation="CPython",major="3",minor="9",patchlevel="19",version="3.9.19"} 1.0 # HELP process_virtual_memory_bytes Virtual memory size in bytes. # TYPE process_virtual_memory_bytes gauge process_virtual_memory_bytes 4.79070208e+09 # HELP process_resident_memory_bytes Resident memory size in bytes. # TYPE process_resident_memory_bytes gauge process_resident_memory_bytes 9.5608832e+07 # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. # TYPE process_start_time_seconds gauge process_start_time_seconds 1.76529432615e+09 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter process_cpu_seconds_total 1589.14 # HELP process_open_fds Number of open file descriptors. # TYPE process_open_fds gauge process_open_fds 45.0 # HELP process_max_fds Maximum number of open file descriptors. # TYPE process_max_fds gauge process_max_fds 65536.0 # HELP ray_event_aggregator_agent_events_received_total Total number of events received. # TYPE ray_event_aggregator_agent_events_received_total counter ray_event_aggregator_agent_events_received_total{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 0.0 # HELP ray_event_aggregator_agent_events_received_created Total number of events received. # TYPE ray_event_aggregator_agent_events_received_created gauge ray_event_aggregator_agent_events_received_created{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 1.765294328544141e+09 # HELP ray_event_aggregator_agent_events_dropped_at_core_worker_total Total number of events dropped at core worker. # TYPE ray_event_aggregator_agent_events_dropped_at_core_worker_total counter ray_event_aggregator_agent_events_dropped_at_core_worker_total{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 0.0 # HELP ray_event_aggregator_agent_events_dropped_at_core_worker_created Total number of events dropped at core worker. # TYPE ray_event_aggregator_agent_events_dropped_at_core_worker_created gauge ray_event_aggregator_agent_events_dropped_at_core_worker_created{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 1.7652943285441704e+09 # HELP ray_event_aggregator_agent_events_dropped_at_event_aggregator_total Total number of events dropped at the event aggregator. # TYPE ray_event_aggregator_agent_events_dropped_at_event_aggregator_total counter ray_event_aggregator_agent_events_dropped_at_event_aggregator_total{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 0.0 # HELP ray_event_aggregator_agent_events_dropped_at_event_aggregator_created Total number of events dropped at the event aggregator. # TYPE ray_event_aggregator_agent_events_dropped_at_event_aggregator_created gauge ray_event_aggregator_agent_events_dropped_at_event_aggregator_created{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 1.765294328544189e+09 # HELP ray_event_aggregator_agent_events_published_total Total number of events successfully published to the external server. # TYPE ray_event_aggregator_agent_events_published_total counter ray_event_aggregator_agent_events_published_total{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 0.0 # HELP ray_event_aggregator_agent_events_published_created Total number of events successfully published to the external server. # TYPE ray_event_aggregator_agent_events_published_created gauge ray_event_aggregator_agent_events_published_created{Component="event_aggregator_agent",SessionName="session_2025-11-13_20-48-21_325201_1",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 1.7652943285442083e+09 # HELP ray_node_mem_shared_bytes Total shared memory usage on a ray node # TYPE ray_node_mem_shared_bytes gauge ray_node_mem_shared_bytes{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 8.0072704e+07 # HELP ray_component_cpu_percentage Total CPU usage of the components on a node. # TYPE ray_component_cpu_percentage gauge ray_component_cpu_percentage{Component="raylet",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="36"} 0.8 ray_component_cpu_percentage{Component="ray::IDLE",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_cpu_percentage{Component="agent",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 6.3 ray_component_cpu_percentage{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_cpu_percentage{Component="ray::ProxyActor",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.6 ray_component_cpu_percentage{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 1.2 # HELP ray_component_mem_shared_bytes SHM usage of all components of the node. It is equivalent to the top command's SHR column. # TYPE ray_component_mem_shared_bytes gauge ray_component_mem_shared_bytes{Component="raylet",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="36"} 1.4635008e+07 ray_component_mem_shared_bytes{Component="ray::IDLE",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_mem_shared_bytes{Component="agent",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 1.9759104e+07 ray_component_mem_shared_bytes{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_mem_shared_bytes{Component="ray::ProxyActor",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 1.9116032e+07 ray_component_mem_shared_bytes{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 3.9170048e+07 # HELP ray_component_rss_mb RSS usage of all components on the node. # TYPE ray_component_rss_mb gauge ray_component_rss_mb{Component="raylet",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="36"} 61.37856 ray_component_rss_mb{Component="ray::IDLE",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_rss_mb{Component="agent",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 95.608832 ray_component_rss_mb{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_rss_mb{Component="ray::ProxyActor",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 198.811648 ray_component_rss_mb{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 14680.895488 # HELP ray_component_uss_mb USS usage of all components on the node. # TYPE ray_component_uss_mb gauge ray_component_uss_mb{Component="raylet",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="36"} 51.798016 ray_component_uss_mb{Component="ray::IDLE",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_uss_mb{Component="agent",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 76.353536 ray_component_uss_mb{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_uss_mb{Component="ray::ProxyActor",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 182.300672 ray_component_uss_mb{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 14642.044928 # HELP ray_component_num_fds Number of open fds of all components on the node (Not available on Windows). # TYPE ray_component_num_fds gauge ray_component_num_fds{Component="raylet",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="36"} 67.0 ray_component_num_fds{Component="ray::IDLE",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_num_fds{Component="agent",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid="100"} 44.0 ray_component_num_fds{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 0.0 ray_component_num_fds{Component="ray::ProxyActor",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 104.0 ray_component_num_fds{Component="ray::ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",SessionName="",Version="2.48.0.1",ip="10.201.73.96",pid=""} 164.0 # HELP ray_node_cpu_utilization Total CPU usage on a ray node # TYPE ray_node_cpu_utilization gauge ray_node_cpu_utilization{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 0.0 # HELP ray_node_cpu_count Total CPUs available on a ray node # TYPE ray_node_cpu_count gauge ray_node_cpu_count{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 192.0 # HELP ray_node_mem_used Memory usage on a ray node # TYPE ray_node_mem_used gauge ray_node_mem_used{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 61440.0 # HELP ray_node_mem_available Memory available on a ray node # TYPE ray_node_mem_available gauge ray_node_mem_available{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 7.9823132672e+011 # HELP ray_node_mem_total Total memory on a ray node # TYPE ray_node_mem_total gauge ray_node_mem_total{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 7.9823138816e+011 # HELP ray_node_disk_io_read Total read from disk # TYPE ray_node_disk_io_read gauge ray_node_disk_io_read{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 7.8648193843712e+013 # HELP ray_node_disk_io_write Total written to disk # TYPE ray_node_disk_io_write gauge ray_node_disk_io_write{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 9.8004274010112e+013 # HELP ray_node_disk_io_read_count Total read ops from disk # TYPE ray_node_disk_io_read_count gauge ray_node_disk_io_read_count{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 8.47264711e+08 # HELP ray_node_disk_io_write_count Total write ops to disk # TYPE ray_node_disk_io_write_count gauge ray_node_disk_io_write_count{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 1.60211624e+09 # HELP ray_node_disk_io_read_speed Disk read speed # TYPE ray_node_disk_io_read_speed gauge ray_node_disk_io_read_speed{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 2.118611092434714e+06 # HELP ray_node_disk_io_write_speed Disk write speed # TYPE ray_node_disk_io_write_speed gauge ray_node_disk_io_write_speed{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 1.6717582825654387e+06 # HELP ray_node_disk_read_iops Disk read iops # TYPE ray_node_disk_read_iops gauge ray_node_disk_read_iops{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 17.933412211149218 # HELP ray_node_disk_write_iops Disk write iops # TYPE ray_node_disk_write_iops gauge ray_node_disk_write_iops{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 59.11030727043687 # HELP ray_node_disk_usage Total disk usage (bytes) on a ray node # TYPE ray_node_disk_usage gauge ray_node_disk_usage{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 6.30235951104e+011 # HELP ray_node_disk_free Total disk free (bytes) on a ray node # TYPE ray_node_disk_free gauge ray_node_disk_free{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 3.71796013056e+011 # HELP ray_node_disk_utilization_percentage Total disk utilization (percentage) on a ray node # TYPE ray_node_disk_utilization_percentage gauge ray_node_disk_utilization_percentage{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 62.895793112979646 # HELP ray_node_network_sent Total network sent # TYPE ray_node_network_sent gauge ray_node_network_sent{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 7.52271175e+08 # HELP ray_node_network_received Total network received # TYPE ray_node_network_received gauge ray_node_network_received{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 2.53568864e+08 # HELP ray_node_network_send_speed Network send speed # TYPE ray_node_network_send_speed gauge ray_node_network_send_speed{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 32165.70518831695 # HELP ray_node_network_receive_speed Network receive speed # TYPE ray_node_network_receive_speed gauge ray_node_network_receive_speed{IsHeadNode="false",SessionName="",Version="2.48.0.1",ip="10.201.73.96"} 5715.8935803418735 # HELP ray_object_directory_removed_locations Number of object locations removed per second. If this is high, a lot of objects have been removed from this node. # TYPE ray_object_directory_removed_locations gauge ray_object_directory_removed_locations{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_directory_added_locations Number of object locations added per second., If this is high, a lot of objects have been added on this node. # TYPE ray_object_directory_added_locations gauge ray_object_directory_added_locations{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_directory_lookups Number of object location lookups per second. If this is high, the raylet is waiting on a lot of objects. # TYPE ray_object_directory_lookups gauge ray_object_directory_lookups{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_directory_subscriptions Number of object location subscriptions. If this is high, the raylet is attempting to pull a lot of objects. # TYPE ray_object_directory_subscriptions gauge ray_object_directory_subscriptions{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_manager_num_pull_requests Number of active pull requests for objects. # TYPE ray_object_manager_num_pull_requests gauge ray_object_manager_num_pull_requests{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_store_num_local_objects Number of objects currently in the object store. # TYPE ray_object_store_num_local_objects gauge ray_object_store_num_local_objects{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_store_fallback_memory Amount of memory in fallback allocations in the filesystem. # TYPE ray_object_store_fallback_memory gauge ray_object_store_fallback_memory{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_store_used_memory Amount of memory currently occupied in the object store. # TYPE ray_object_store_used_memory gauge ray_object_store_used_memory{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_pull_manager_usage_bytes The total number of bytes usage broken per type {Available, BeingPulled, Pinned} # TYPE ray_pull_manager_usage_bytes gauge ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="Available",Version="2.48.0.1"} 1.5461882265e+010 ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="BeingPulled",Version="2.48.0.1"} 0.0 ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="Pinned",Version="2.48.0.1"} 0.0 # HELP ray_scheduler_failed_worker_startup_total Number of tasks that fail to be scheduled because workers were not available. Labels are broken per reason {JobConfigMissing, RegistrationTimedOut, RateLimited} # TYPE ray_scheduler_failed_worker_startup_total gauge ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.201.73.96",Reason="JobConfigMissing",Version="2.48.0.1"} 0.0 ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.201.73.96",Reason="RegistrationTimedOut",Version="2.48.0.1"} 0.0 ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.201.73.96",Reason="RateLimited",Version="2.48.0.1"} 0.0 # HELP ray_grpc_server_req_failed_total Failed request count in grpc server # TYPE ray_grpc_server_req_failed_total counter ray_grpc_server_req_failed_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 2.0 # HELP ray_grpc_server_req_finished_total Finished request number in grpc server # TYPE ray_grpc_server_req_finished_total counter ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1142.0 ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 21760.0 ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1273.0 ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 9.0 ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 65500.0 ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1765.0 ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 414.0 ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.0 # HELP ray_grpc_server_req_process_time_ms Request latency in grpc server # TYPE ray_grpc_server_req_process_time_ms histogram ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 908.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 991.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 1002.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 1034.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 1082.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 1142.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1142.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.5857749352076003e+07 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 15.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 21356.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 21758.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 21760.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 21760.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 21760.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 21760.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 21760.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 5791.985103000007 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 0.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 824.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 1233.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 1249.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 1271.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 1271.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 1273.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1273.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 71261.35601600015 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 0.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 6.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 8.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 9.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 9.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 9.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 9.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 9.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 63.63842199999999 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 8410.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 64712.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 65497.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 65500.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 65500.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 65500.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 65500.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 65500.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 12754.323989999995 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 0.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 1658.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 1765.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 1765.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 1765.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 1765.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 1765.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1765.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 757.4237210000003 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 0.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 383.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 413.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 414.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 414.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 414.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 414.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 414.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 226.7582280000001 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="0.1"} 0.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 1.0 ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 1.0 ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.0 ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.374052 # HELP ray_object_manager_received_chunks Number object chunks received broken per type {Total, FailedTotal, FailedCancelled, FailedPlasmaFull}. # TYPE ray_object_manager_received_chunks gauge ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="FailedPlasmaFull",Version="2.48.0.1"} 0.0 ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="FailedTotal",Version="2.48.0.1"} 0.0 ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="FailedCancelled",Version="2.48.0.1"} 0.0 ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="Total",Version="2.48.0.1"} 414.0 # HELP ray_object_directory_updates Number of object location updates per second., If this is high, the raylet is attempting to pull a lot of objects and/or the locations for objects are frequently changing (e.g. due to many object copies or evictions). # TYPE ray_object_directory_updates gauge ray_object_directory_updates{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_pull_manager_requests Number of pull requests broken per type {Queued, Active, Pinned}. # TYPE ray_pull_manager_requests gauge ray_pull_manager_requests{Component="raylet",NodeAddress="10.201.73.96",Type="Pinned",Version="2.48.0.1"} 0.0 ray_pull_manager_requests{Component="raylet",NodeAddress="10.201.73.96",Type="Queued",Version="2.48.0.1"} 0.0 ray_pull_manager_requests{Component="raylet",NodeAddress="10.201.73.96",Type="Active",Version="2.48.0.1"} 0.0 # HELP ray_grpc_server_req_new_total New request number in grpc server # TYPE ray_grpc_server_req_new_total counter ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 41.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelResourceReserve",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetObjectsInfo",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CommitBundleResources",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 33.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.FormatGlobalMemoryInfo",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReleaseUnusedActorWorkers",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 21792.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.IsLocalWorkerDead",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PushMutableObject",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 65532.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ShutdownRaylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PrestartWorkers",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.RegisterMutableObject",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.DrainRaylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PrepareBundleResources",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReleaseUnusedBundles",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorker",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelTasksWithResourceShapes",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1829.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.NotifyGCSRestart",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GlobalGC",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 478.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1305.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1175.0 ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetTaskFailureCause",NodeAddress="10.201.73.96",Version="2.48.0.1"} 32.0 ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.201.73.96",Version="2.48.0.1"} 64.0 # HELP ray_spill_manager_objects Number of local objects broken per state {Pinned, PendingRestore, PendingSpill}. # TYPE ray_spill_manager_objects gauge ray_spill_manager_objects{Component="raylet",NodeAddress="10.201.73.96",State="PendingRestore",Version="2.48.0.1"} 0.0 ray_spill_manager_objects{Component="raylet",NodeAddress="10.201.73.96",State="Pinned",Version="2.48.0.1"} 0.0 ray_spill_manager_objects{Component="raylet",NodeAddress="10.201.73.96",State="PendingSpill",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_infeasible_scheduling_classes The number of unique scheduling classes that are infeasible. # TYPE ray_internal_num_infeasible_scheduling_classes gauge ray_internal_num_infeasible_scheduling_classes{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_started_from_cache_total The total number of workers started from a cached worker process. # TYPE ray_internal_num_processes_started_from_cache_total counter ray_internal_num_processes_started_from_cache_total{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_started_from_cache (DEPRECATED, use ray_internal_num_processes_started_from_cache_total metric instead) The total number of workers started from a cached worker process. # TYPE ray_internal_num_processes_started_from_cache gauge ray_internal_num_processes_started_from_cache{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_pull_manager_requested_bundles Number of requested bundles broken per type {Get, Wait, TaskArgs}. # TYPE ray_pull_manager_requested_bundles gauge ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.201.73.96",Type="Get",Version="2.48.0.1"} 0.0 ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.201.73.96",Type="Wait",Version="2.48.0.1"} 0.0 ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.201.73.96",Type="TaskArgs",Version="2.48.0.1"} 0.0 ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.201.73.96",Type="CumulativeTotal",Version="2.48.0.1"} 829.0 # HELP ray_pull_manager_active_bundles Number of active bundle requests # TYPE ray_pull_manager_active_bundles gauge ray_pull_manager_active_bundles{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_object_store_memory Object store memory by various sub-kinds on this node # TYPE ray_object_store_memory gauge ray_object_store_memory{Component="raylet",Location="SPILLED",NodeAddress="10.201.73.96",ObjectState="",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="raylet",Location="MMAP_DISK",NodeAddress="10.201.73.96",ObjectState="UNSEALED",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="raylet",Location="MMAP_SHM",NodeAddress="10.201.73.96",ObjectState="SEALED",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="raylet",Location="MMAP_DISK",NodeAddress="10.201.73.96",ObjectState="SEALED",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="raylet",Location="MMAP_SHM",NodeAddress="10.201.73.96",ObjectState="UNSEALED",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.201.73.96",ObjectState="",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.201.73.96",ObjectState="",Version="2.48.0.1"} 0.0 ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.201.73.96",ObjectState="",Version="2.48.0.1"} 50148.0 # HELP ray_pull_manager_object_request_time_ms Time between initial object pull request and local pinning of the object. # TYPE ray_pull_manager_object_request_time_ms histogram ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="1.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="10.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="100.0"} 66.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="1000.0"} 78.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="10000.0"} 78.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1",le="+Inf"} 492.0 ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1"} 492.0 ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.201.73.96",Type="MemoryAvailableToPin",Version="2.48.0.1"} 1.5876073500000007e+07 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="1.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="10.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="100.0"} 94.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="1000.0"} 379.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="10000.0"} 413.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1",le="+Inf"} 828.0 ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1"} 828.0 ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.201.73.96",Type="StartToCancel",Version="2.48.0.1"} 1.6533021500000004e+07 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="1.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="10.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="100.0"} 0.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="1000.0"} 66.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="10000.0"} 78.0 ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1",le="+Inf"} 492.0 ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1"} 492.0 ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.201.73.96",Type="StartToPin",Version="2.48.0.1"} 1.636924674999998e+07 # HELP ray_object_store_dist The distribution of object size in bytes # TYPE ray_object_store_dist histogram ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="33554432.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="67108864.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="134217728.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="268435456.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="536870912.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="1073741824.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="2147483648.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="4294967296.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="8589934592.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="17179869184.0"} 9.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1",le="+Inf"} 9.0 ray_object_store_dist_count{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1"} 9.0 ray_object_store_dist_sum{Component="raylet",NodeAddress="10.201.73.96",Source="CreatedByWorker",Version="2.48.0.1"} 72.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="33554432.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="67108864.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="134217728.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="268435456.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="536870912.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="1073741824.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="2147483648.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="4294967296.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="8589934592.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="17179869184.0"} 414.0 ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1",le="+Inf"} 414.0 ray_object_store_dist_count{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1"} 414.0 ray_object_store_dist_sum{Component="raylet",NodeAddress="10.201.73.96",Source="ReceivedFromRemoteRaylet",Version="2.48.0.1"} 8.487655400000001e+07 # HELP ray_pull_manager_num_object_pins Number of object pin attempts by the pull manager, can be {Success, Failure}. # TYPE ray_pull_manager_num_object_pins gauge ray_pull_manager_num_object_pins{Component="raylet",NodeAddress="10.201.73.96",Type="Failure",Version="2.48.0.1"} 414.0 ray_pull_manager_num_object_pins{Component="raylet",NodeAddress="10.201.73.96",Type="Success",Version="2.48.0.1"} 492.0 # HELP ray_object_manager_bytes Number of bytes pushed or received by type {PushedFromLocalPlasma, PushedFromLocalDisk, Received}. # TYPE ray_object_manager_bytes gauge ray_object_manager_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="PushedFromLocalDisk",Version="2.48.0.1"} 0.0 ray_object_manager_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="Received",Version="2.48.0.1"} 8.4876554e+07 ray_object_manager_bytes{Component="raylet",NodeAddress="10.201.73.96",Type="PushedFromLocalPlasma",Version="2.48.0.1"} 0.0 # HELP ray_pull_manager_retries_total Number of cumulative pull retries. # TYPE ray_pull_manager_retries_total gauge ray_pull_manager_retries_total{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 414.0 # HELP ray_push_manager_num_pushes_remaining Number of pushes not completed. # TYPE ray_push_manager_num_pushes_remaining gauge ray_push_manager_num_pushes_remaining{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_spill_manager_objects_bytes Byte size of local objects broken per state {Pinned, PendingSpill}. # TYPE ray_spill_manager_objects_bytes gauge ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.201.73.96",State="Restored",Version="2.48.0.1"} 0.0 ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.201.73.96",State="Spilled",Version="2.48.0.1"} 0.0 ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.201.73.96",State="PendingRestore",Version="2.48.0.1"} 0.0 ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.201.73.96",State="Pinned",Version="2.48.0.1"} 0.0 ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.201.73.96",State="PendingSpill",Version="2.48.0.1"} 0.0 # HELP ray_worker_register_time_ms end to end latency of register a worker process. # TYPE ray_worker_register_time_ms histogram ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1.0"} 0.0 ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10.0"} 0.0 ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="100.0"} 0.0 ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="1000.0"} 6.0 ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="10000.0"} 7.0 ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1",le="+Inf"} 7.0 ray_worker_register_time_ms_count{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 7.0 ray_worker_register_time_ms_sum{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 5553.0 # HELP ray_push_manager_chunks Number of object chunks transfer broken per type {InFlight, Remaining}. # TYPE ray_push_manager_chunks gauge ray_push_manager_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="InFlight",Version="2.48.0.1"} 0.0 ray_push_manager_chunks{Component="raylet",NodeAddress="10.201.73.96",Type="Remaining",Version="2.48.0.1"} 0.0 # HELP ray_scheduler_tasks Number of tasks waiting for scheduling broken per state {Cancelled, Executing, Waiting, Dispatched, Received}. # TYPE ray_scheduler_tasks gauge ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="Received",Version="2.48.0.1"} 0.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="Cancelled",Version="2.48.0.1"} 0.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="Dispatched",Version="2.48.0.1"} 1.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="SpilledUnschedulable",Version="2.48.0.1"} 146.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="SpilledWaiting",Version="2.48.0.1"} 0.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="Executing",Version="2.48.0.1"} 0.0 ray_scheduler_tasks{Component="raylet",NodeAddress="10.201.73.96",State="Waiting",Version="2.48.0.1"} 0.0 # HELP ray_scheduler_unscheduleable_tasks Number of pending tasks (not scheduleable tasks) broken per reason {Infeasible, WaitingForResources, WaitingForPlasmaMemory, WaitingForRemoteResources, WaitingForWorkers}. # TYPE ray_scheduler_unscheduleable_tasks gauge ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.201.73.96",Reason="WaitingForRemoteResources",Version="2.48.0.1"} 1.0 ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.201.73.96",Reason="Infeasible",Version="2.48.0.1"} 0.0 ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.201.73.96",Reason="WaitingForResources",Version="2.48.0.1"} 0.0 ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.201.73.96",Reason="WaitingForPlasmaMemory",Version="2.48.0.1"} 0.0 ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.201.73.96",Reason="WaitingForWorkers",Version="2.48.0.1"} 0.0 # HELP ray_grpc_server_req_succeeded_total Succeeded request count in grpc server # TYPE ray_grpc_server_req_succeeded_total counter ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1142.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 21760.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1271.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 9.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 65500.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1765.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 414.0 ray_grpc_server_req_succeeded_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.0 # HELP ray_grpc_server_req_handling_total Request number are handling in grpc server # TYPE ray_grpc_server_req_handling_total counter ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1143.0 ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.201.73.96",Version="2.48.0.1"} 21760.0 ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1273.0 ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.201.73.96",Version="2.48.0.1"} 9.0 ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.201.73.96",Version="2.48.0.1"} 65500.0 ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1765.0 ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.201.73.96",Version="2.48.0.1"} 414.0 ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.0 # HELP ray_resources Logical Ray resources broken per state {AVAILABLE, USED} # TYPE ray_resources gauge ray_resources{Component="raylet",Name="object_store_memory",NodeAddress="10.201.73.96",State="USED",Version="2.48.0.1"} 0.0 ray_resources{Component="raylet",Name="object_store_memory",NodeAddress="10.201.73.96",State="AVAILABLE",Version="2.48.0.1"} 1.6320875724e+010 ray_resources{Component="raylet",Name="memory",NodeAddress="10.201.73.96",State="AVAILABLE",Version="2.48.0.1"} 1.7179869184e+010 ray_resources{Component="raylet",Name="memory",NodeAddress="10.201.73.96",State="USED",Version="2.48.0.1"} 0.0 ray_resources{Component="raylet",Name="CPU",NodeAddress="10.201.73.96",State="AVAILABLE",Version="2.48.0.1"} 0.0 ray_resources{Component="raylet",Name="CPU",NodeAddress="10.201.73.96",State="USED",Version="2.48.0.1"} 4.0 # HELP ray_spill_manager_request_total Number of {spill, restore} requests. # TYPE ray_spill_manager_request_total gauge ray_spill_manager_request_total{Component="raylet",NodeAddress="10.201.73.96",Type="Restored",Version="2.48.0.1"} 0.0 ray_spill_manager_request_total{Component="raylet",NodeAddress="10.201.73.96",Type="Spilled",Version="2.48.0.1"} 0.0 ray_spill_manager_request_total{Component="raylet",NodeAddress="10.201.73.96",Type="FailedDeletion",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_started_total The total number of worker processes the worker pool has created. # TYPE ray_internal_num_processes_started_total counter ray_internal_num_processes_started_total{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 7.0 # HELP ray_internal_num_processes_started (DEPRECATED, use ray_internal_num_processes_started_total metric instead) The total number of worker processes the worker pool has created. # TYPE ray_internal_num_processes_started gauge ray_internal_num_processes_started{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 7.0 # HELP ray_internal_num_processes_skipped_job_mismatch_total The total number of cached workers skipped due to job mismatch. # TYPE ray_internal_num_processes_skipped_job_mismatch_total counter ray_internal_num_processes_skipped_job_mismatch_total{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_skipped_job_mismatch (DEPRECATED, use ray_internal_num_processes_skipped_job_mismatch_total metric instead) The total number of cached workers skipped due to job mismatch. # TYPE ray_internal_num_processes_skipped_job_mismatch gauge ray_internal_num_processes_skipped_job_mismatch{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_skipped_runtime_environment_mismatch_total The total number of cached workers skipped due to runtime environment mismatch. # TYPE ray_internal_num_processes_skipped_runtime_environment_mismatch_total counter ray_internal_num_processes_skipped_runtime_environment_mismatch_total{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_processes_skipped_runtime_environment_mismatch (DEPRECATED, use ray_internal_num_processes_skipped_runtime_environment_mismatch_total metric instead) The total number of cached workers skipped due to runtime environment mismatch. # TYPE ray_internal_num_processes_skipped_runtime_environment_mismatch gauge ray_internal_num_processes_skipped_runtime_environment_mismatch{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_internal_num_spilled_tasks The cumulative number of lease requeusts that this raylet has spilled to other raylets. # TYPE ray_internal_num_spilled_tasks gauge ray_internal_num_spilled_tasks{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.0 # HELP ray_object_store_available_memory Amount of memory currently available in the object store. # TYPE ray_object_store_available_memory gauge ray_object_store_available_memory{Component="raylet",NodeAddress="10.201.73.96",Version="2.48.0.1"} 1.6320875724e+010 # HELP ray_serve_replica_processing_queries The current number of queries being processed. # TYPE ray_serve_replica_processing_queries gauge ray_serve_replica_processing_queries{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw"} 0.0 ray_serve_replica_processing_queries{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="tv3z0dl0"} 0.0 # HELP ray_serve_deployment_replica_starts_total The number of times this replica has been restarted due to failure. # TYPE ray_serve_deployment_replica_starts_total counter ray_serve_deployment_replica_starts_total{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw"} 1.0 ray_serve_deployment_replica_starts_total{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="tv3z0dl0"} 1.0 # HELP ray_serve_deployment_replica_starts (DEPRECATED, use ray_serve_deployment_replica_starts_total metric instead) The number of times this replica has been restarted due to failure. # TYPE ray_serve_deployment_replica_starts gauge ray_serve_deployment_replica_starts{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw"} 1.0 ray_serve_deployment_replica_starts{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="tv3z0dl0"} 1.0 # HELP ray_total_lineage_bytes Total amount of memory used to store task specs for lineage reconstruction. # TYPE ray_total_lineage_bytes gauge ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1"} 0.0 # HELP ray_actors Current number of actors currently in a particular state. # TYPE ray_actors gauge ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="IDLE",Version="2.48.0.1"} 1.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_TASK",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_TASK",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="IDLE",Version="2.48.0.1"} 1.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_TASK",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.201.73.96",Source="executor",State="IDLE",Version="2.48.0.1"} 1.0 ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 # HELP ray_tasks Current number of tasks currently in a particular state. # TYPE ray_tasks gauge ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 365.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.is_allocated",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 365.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.record_routing_stats",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 1.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 1.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 647.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="owner",State="FAILED",Version="2.48.0.1"} 78.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 1278.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.pong",NodeAddress="10.201.73.96",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 2.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.201.73.96",Source="executor",State="RUNNING",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 1.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:mmcls-unite-dinov3:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 725.0 ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="FAILED",Version="2.48.0.1"} 2.0 ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.48.0.1"} 0.0 ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.201.73.96",Source="owner",State="FINISHED",Version="2.48.0.1"} 2.0 # HELP ray_serve_deployment_processing_latency_ms The latency for queries to be processed. # TYPE ray_serve_deployment_processing_latency_ms histogram ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="1.0",replica="jw2b3fyw",route="/"} 0.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="2.0",replica="jw2b3fyw",route="/"} 0.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="5.0",replica="jw2b3fyw",route="/"} 0.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="10.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="20.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="50.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="100.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="200.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="300.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="400.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="500.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="1000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="2000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="5000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="10000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="60000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="120000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="300000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="600000.0",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",le="+Inf",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_count{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw",route="/"} 1.0 ray_serve_deployment_processing_latency_ms_sum{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw",route="/"} 6.49571418762207 # HELP ray_serve_deployment_request_counter_total The number of queries that have been processed in this replica. # TYPE ray_serve_deployment_request_counter_total counter ray_serve_deployment_request_counter_total{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw",route="/"} 1.0 # HELP ray_serve_deployment_request_counter (DEPRECATED, use ray_serve_deployment_request_counter_total metric instead) The number of queries that have been processed in this replica. # TYPE ray_serve_deployment_request_counter gauge ray_serve_deployment_request_counter{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",replica="jw2b3fyw",route="/"} 1.0 # HELP ray_serve_num_scheduling_tasks The number of request routing tasks in the router. # TYPE ray_serve_num_scheduling_tasks gauge ray_serve_num_scheduling_tasks{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",actor_id="d8812e3327d88ea5ea8bbdfb01000000",app="mmcls-unite-dinov3",deployment="EmbeddingRayServe"} 0.0 # HELP ray_serve_deployment_queued_queries The current number of queries to this deployment waiting to be assigned to a replica. # TYPE ray_serve_deployment_queued_queries gauge ray_serve_deployment_queued_queries{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",actor_id="d8812e3327d88ea5ea8bbdfb01000000",application="mmcls-unite-dinov3",deployment="EmbeddingRayServe",handle="vdb0otic"} 0.0 # HELP ray_serve_num_scheduling_tasks_in_backoff The number of request routing tasks in the router that are undergoing backoff. # TYPE ray_serve_num_scheduling_tasks_in_backoff gauge ray_serve_num_scheduling_tasks_in_backoff{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",actor_id="d8812e3327d88ea5ea8bbdfb01000000",app="mmcls-unite-dinov3",deployment="EmbeddingRayServe"} 0.0 # HELP ray_serve_http_request_latency_ms The end-to-end latency of HTTP requests (measured from the Serve HTTP proxy). # TYPE ray_serve_http_request_latency_ms histogram ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="1.0",method="GET",route="/-/healthz",status_code="200"} 729.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="2.0",method="GET",route="/-/healthz",status_code="200"} 731.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="5.0",method="GET",route="/-/healthz",status_code="200"} 731.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="10.0",method="GET",route="/-/healthz",status_code="200"} 731.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="20.0",method="GET",route="/-/healthz",status_code="200"} 731.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="50.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="100.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="200.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="300.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="400.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="500.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="1000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="2000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="5000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="10000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="60000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="120000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="300000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="600000.0",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",le="+Inf",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",method="GET",route="/-/healthz",status_code="200"} 732.0 ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",method="GET",route="/-/healthz",status_code="200"} 104.02917861938474 # HELP ray_serve_num_http_requests_total The number of HTTP requests processed. # TYPE ray_serve_num_http_requests_total counter ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",method="GET",route="/-/healthz",status_code="200"} 732.0 # HELP ray_serve_num_http_requests (DEPRECATED, use ray_serve_num_http_requests_total metric instead) The number of HTTP requests processed. # TYPE ray_serve_num_http_requests gauge ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.201.73.96",Version="2.48.0.1",application="",method="GET",route="/-/healthz",status_code="200"} 732.0