# HELP python_gc_objects_collected_total Objects collected during gc
# TYPE python_gc_objects_collected_total counter
python_gc_objects_collected_total{generation="0"} 192682.0
python_gc_objects_collected_total{generation="1"} 21229.0
python_gc_objects_collected_total{generation="2"} 1096.0
# HELP python_gc_objects_uncollectable_total Uncollectable objects found during GC
# TYPE python_gc_objects_uncollectable_total counter
python_gc_objects_uncollectable_total{generation="0"} 0.0
python_gc_objects_uncollectable_total{generation="1"} 0.0
python_gc_objects_uncollectable_total{generation="2"} 0.0
# HELP python_gc_collections_total Number of times this generation was collected
# TYPE python_gc_collections_total counter
python_gc_collections_total{generation="0"} 190847.0
python_gc_collections_total{generation="1"} 17349.0
python_gc_collections_total{generation="2"} 969.0
# HELP python_info Python platform information
# TYPE python_info gauge
python_info{implementation="CPython",major="3",minor="9",patchlevel="19",version="3.9.19"} 1.0
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 2.160439296e+09
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 1.17436416e+08
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.76598972588e+09
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 3365.64
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 34.0
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 65536.0
# HELP ray_node_mem_shared_bytes Total shared memory usage on a ray node
# TYPE ray_node_mem_shared_bytes gauge
ray_node_mem_shared_bytes{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 7.7811712e+07
# HELP ray_component_cpu_percentage Total CPU usage of the components on a node.
# TYPE ray_component_cpu_percentage gauge
ray_component_cpu_percentage{Component="raylet",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="22"} 1.7
ray_component_cpu_percentage{Component="ray::IDLE",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="agent",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="52"} 8.4
ray_component_cpu_percentage{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#gvo4xms5:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#cgo8bpm2:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 330.29999999999995
ray_component_cpu_percentage{Component="ray::ProxyActor",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 4.3
ray_component_cpu_percentage{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ProxyActor.receive_asgi_messages",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ProxyActor.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_cpu_percentage{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
# HELP ray_component_mem_shared_bytes SHM usage of all components of the node. It is equivalent to the top command's SHR column.
# TYPE ray_component_mem_shared_bytes gauge
ray_component_mem_shared_bytes{Component="raylet",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="22"} 2.2355968e+07
ray_component_mem_shared_bytes{Component="ray::IDLE",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="agent",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="52"} 4.8484352e+07
ray_component_mem_shared_bytes{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#gvo4xms5:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#cgo8bpm2:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 6.32438784e+08
ray_component_mem_shared_bytes{Component="ray::ProxyActor",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 9.578496e+07
ray_component_mem_shared_bytes{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ProxyActor.receive_asgi_messages",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ProxyActor.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_mem_shared_bytes{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
# HELP ray_component_rss_mb RSS usage of all components on the node.
# TYPE ray_component_rss_mb gauge
ray_component_rss_mb{Component="raylet",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="22"} 175.128576
ray_component_rss_mb{Component="ray::IDLE",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="agent",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="52"} 117.436416
ray_component_rss_mb{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#gvo4xms5:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#cgo8bpm2:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 9957.236735999999
ray_component_rss_mb{Component="ray::ProxyActor",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 349.425664
ray_component_rss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ProxyActor.receive_asgi_messages",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ProxyActor.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_rss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
# HELP ray_component_uss_mb USS usage of all components on the node.
# TYPE ray_component_uss_mb gauge
ray_component_uss_mb{Component="raylet",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="22"} 156.450816
ray_component_uss_mb{Component="ray::IDLE",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="agent",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="52"} 69.050368
ray_component_uss_mb{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#gvo4xms5:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#cgo8bpm2:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 9326.022656000001
ray_component_uss_mb{Component="ray::ProxyActor",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 253.755392
ray_component_uss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ProxyActor.receive_asgi_messages",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ProxyActor.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_uss_mb{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
# HELP ray_component_num_fds Number of open fds of all components on the node (Not available on Windows).
# TYPE ray_component_num_fds gauge
ray_component_num_fds{Component="raylet",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="22"} 1343.0
ray_component_num_fds{Component="ray::IDLE",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="agent",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid="52"} 33.0
ray_component_num_fds{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#gvo4xms5:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::SERVE_REPLICA::video_embedding#EmbeddingRayServe#cgo8bpm2:ServeReplica:video_embedding:EmbeddingRayServe.__init__",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 917.0
ray_component_num_fds{Component="ray::ProxyActor",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 906.0
ray_component_num_fds{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ProxyActor.receive_asgi_messages",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ProxyActor.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
ray_component_num_fds{Component="ray::ServeReplica:video_embedding:EmbeddingRayServe.check_health",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243",pid=""} 0.0
# HELP ray_node_cpu_utilization Total CPU usage on a ray node
# TYPE ray_node_cpu_utilization gauge
ray_node_cpu_utilization{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 0.0
# HELP ray_node_cpu_count Total CPUs available on a ray node
# TYPE ray_node_cpu_count gauge
ray_node_cpu_count{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 64.0
# HELP ray_node_mem_used Memory usage on a ray node
# TYPE ray_node_mem_used gauge
ray_node_mem_used{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 3.93216e+06
# HELP ray_node_mem_available Memory available on a ray node
# TYPE ray_node_mem_available gauge
ray_node_mem_available{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 5.32074643456e+011
# HELP ray_node_mem_total Total memory on a ray node
# TYPE ray_node_mem_total gauge
ray_node_mem_total{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 5.32072075264e+011
# HELP ray_node_disk_io_read Total read from disk
# TYPE ray_node_disk_io_read gauge
ray_node_disk_io_read{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 4.5893104432128e+013
# HELP ray_node_disk_io_write Total written to disk
# TYPE ray_node_disk_io_write gauge
ray_node_disk_io_write{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 4.92747814651392e+014
# HELP ray_node_disk_io_read_count Total read ops from disk
# TYPE ray_node_disk_io_read_count gauge
ray_node_disk_io_read_count{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 9.197719148e+09
# HELP ray_node_disk_io_write_count Total write ops to disk
# TYPE ray_node_disk_io_write_count gauge
ray_node_disk_io_write_count{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 5.71787352e+09
# HELP ray_node_disk_io_read_speed Disk read speed
# TYPE ray_node_disk_io_read_speed gauge
ray_node_disk_io_read_speed{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 4.673955222529455e+06
# HELP ray_node_disk_io_write_speed Disk write speed
# TYPE ray_node_disk_io_write_speed gauge
ray_node_disk_io_write_speed{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 2.75379264948381e+07
# HELP ray_node_disk_read_iops Disk read iops
# TYPE ray_node_disk_read_iops gauge
ray_node_disk_read_iops{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 1141.0714928011178
# HELP ray_node_disk_write_iops Disk write iops
# TYPE ray_node_disk_write_iops gauge
ray_node_disk_write_iops{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 263.23636844473594
# HELP ray_node_disk_usage Total disk usage (bytes) on a ray node
# TYPE ray_node_disk_usage gauge
ray_node_disk_usage{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 1.9137032192e+011
# HELP ray_node_disk_free Total disk free (bytes) on a ray node
# TYPE ray_node_disk_free gauge
ray_node_disk_free{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 1.03399923712e+011
# HELP ray_node_disk_utilization_percentage Total disk utilization (percentage) on a ray node
# TYPE ray_node_disk_utilization_percentage gauge
ray_node_disk_utilization_percentage{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 64.92185855112135
# HELP ray_node_network_sent Total network sent
# TYPE ray_node_network_sent gauge
ray_node_network_sent{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 3.061222951e+09
# HELP ray_node_network_received Total network received
# TYPE ray_node_network_received gauge
ray_node_network_received{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 4.608871182e+09
# HELP ray_node_network_send_speed Network send speed
# TYPE ray_node_network_send_speed gauge
ray_node_network_send_speed{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 61073.42942091468
# HELP ray_node_network_receive_speed Network receive speed
# TYPE ray_node_network_receive_speed gauge
ray_node_network_receive_speed{IsHeadNode="false",SessionName="",Version="2.40.0.2.dev20250317",ip="10.85.24.243"} 89933.89838299647
# HELP ray_object_directory_removed_locations Number of object locations removed per second. If this is high, a lot of objects have been removed from this node.
# TYPE ray_object_directory_removed_locations gauge
ray_object_directory_removed_locations{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_directory_lookups Number of object location lookups per second. If this is high, the raylet is waiting on a lot of objects.
# TYPE ray_object_directory_lookups gauge
ray_object_directory_lookups{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_directory_subscriptions Number of object location subscriptions. If this is high, the raylet is attempting to pull a lot of objects.
# TYPE ray_object_directory_subscriptions gauge
ray_object_directory_subscriptions{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_store_fallback_memory Amount of memory in fallback allocations in the filesystem.
# TYPE ray_object_store_fallback_memory gauge
ray_object_store_fallback_memory{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_pull_manager_active_bundles Number of active bundle requests
# TYPE ray_pull_manager_active_bundles gauge
ray_pull_manager_active_bundles{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_store_memory Object store memory by various sub-kinds on this node
# TYPE ray_object_store_memory gauge
ray_object_store_memory{Component="raylet",Location="SPILLED",NodeAddress="10.85.24.243",ObjectState="",Version="2.40.0.2.dev20250317"} 0.0
ray_object_store_memory{Component="raylet",Location="MMAP_SHM",NodeAddress="10.85.24.243",ObjectState="SEALED",Version="2.40.0.2.dev20250317"} 0.0
ray_object_store_memory{Component="raylet",Location="MMAP_SHM",NodeAddress="10.85.24.243",ObjectState="UNSEALED",Version="2.40.0.2.dev20250317"} 0.0
ray_object_store_memory{Component="raylet",Location="MMAP_DISK",NodeAddress="10.85.24.243",ObjectState="SEALED",Version="2.40.0.2.dev20250317"} 0.0
ray_object_store_memory{Component="raylet",Location="MMAP_DISK",NodeAddress="10.85.24.243",ObjectState="UNSEALED",Version="2.40.0.2.dev20250317"} 0.0
ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.85.24.243",ObjectState="",Version="2.40.0.2.dev20250317"} 153.0
ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.85.24.243",ObjectState="",Version="2.40.0.2.dev20250317"} 351.0
ray_object_store_memory{Component="core_worker",Location="WORKER_HEAP",NodeAddress="10.85.24.243",ObjectState="",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_pull_manager_object_request_time_ms Time between initial object pull request and local pinning of the object. 
# TYPE ray_pull_manager_object_request_time_ms histogram
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="10.0"} 5.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="100.0"} 734.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="1000.0"} 2576.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="10000.0"} 2672.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317",le="+Inf"} 5251.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317"} 5251.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.85.24.243",Type="StartToCancel",Version="2.40.0.2.dev20250317"} 7.047458274999988e+07
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="10.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="100.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="1000.0"} 447.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="10000.0"} 504.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317",le="+Inf"} 3083.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317"} 3083.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.85.24.243",Type="StartToPin",Version="2.40.0.2.dev20250317"} 6.968571799999993e+07
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="1.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="10.0"} 0.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="100.0"} 431.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="1000.0"} 474.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="10000.0"} 504.0
ray_pull_manager_object_request_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317",le="+Inf"} 3083.0
ray_pull_manager_object_request_time_ms_count{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317"} 3083.0
ray_pull_manager_object_request_time_ms_sum{Component="raylet",NodeAddress="10.85.24.243",Type="MemoryAvailableToPin",Version="2.40.0.2.dev20250317"} 6.742995050000003e+07
# HELP ray_object_store_used_memory Amount of memory currently occupied in the object store.
# TYPE ray_object_store_used_memory gauge
ray_object_store_used_memory{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_push_manager_in_flight_pushes Number of in flight object push requests.
# TYPE ray_push_manager_in_flight_pushes gauge
ray_push_manager_in_flight_pushes{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_pull_manager_usage_bytes The total number of bytes usage broken per type {Available, BeingPulled, Pinned}
# TYPE ray_pull_manager_usage_bytes gauge
ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="Available",Version="2.40.0.2.dev20250317"} 1.5461882265e+010
ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="BeingPulled",Version="2.40.0.2.dev20250317"} 0.0
ray_pull_manager_usage_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="Pinned",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_scheduler_failed_worker_startup_total Number of tasks that fail to be scheduled because workers were not available. Labels are broken per reason {JobConfigMissing, RegistrationTimedOut, RateLimited}
# TYPE ray_scheduler_failed_worker_startup_total gauge
ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.85.24.243",Reason="JobConfigMissing",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.85.24.243",Reason="RegistrationTimedOut",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_failed_worker_startup_total{Component="raylet",NodeAddress="10.85.24.243",Reason="RateLimited",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_grpc_server_req_finished_total Finished request number in grpc server
# TYPE ray_grpc_server_req_finished_total counter
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54419.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5066.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 165112.0
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 25034.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3423.0
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 2608.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54.0
ray_grpc_server_req_finished_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 51.0
ray_grpc_server_req_finished_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 16.0
# HELP ray_object_manager_received_chunks Number object chunks received broken per type {Total, FailedTotal, FailedCancelled, FailedPlasmaFull}.
# TYPE ray_object_manager_received_chunks gauge
ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="FailedCancelled",Version="2.40.0.2.dev20250317"} 0.0
ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="FailedTotal",Version="2.40.0.2.dev20250317"} 0.0
ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="Total",Version="2.40.0.2.dev20250317"} 2608.0
ray_object_manager_received_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="FailedPlasmaFull",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_directory_updates Number of object location updates per second., If this is high, the raylet is attempting to pull a lot of objects and/or the locations for objects are frequently changing (e.g. due to many object copies or evictions).
# TYPE ray_object_directory_updates gauge
ray_object_directory_updates{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_pull_manager_requests Number of pull requests broken per type {Queued, Active, Pinned}.
# TYPE ray_pull_manager_requests gauge
ray_pull_manager_requests{Component="raylet",NodeAddress="10.85.24.243",Type="Active",Version="2.40.0.2.dev20250317"} 0.0
ray_pull_manager_requests{Component="raylet",NodeAddress="10.85.24.243",Type="Pinned",Version="2.40.0.2.dev20250317"} 0.0
ray_pull_manager_requests{Component="raylet",NodeAddress="10.85.24.243",Type="Queued",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_grpc_server_req_new_total New request number in grpc server
# TYPE ray_grpc_server_req_new_total counter
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetTasksInfo",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CommitBundleResources",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ShutdownRaylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetObjectsInfo",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReleaseUnusedBundles",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.FormatGlobalMemoryInfo",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3455.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 48.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 86.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReleaseUnusedActorWorkers",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GlobalGC",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PushMutableObject",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelResourceReserve",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5099.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 2672.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.DrainRaylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetTaskFailureCause",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54451.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReturnWorker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 25098.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.RegisterMutableObject",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.PrepareBundleResources",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 35.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.NotifyGCSRestart",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 32.0
ray_grpc_server_req_new_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 165144.0
ray_grpc_server_req_new_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 115.0
# HELP ray_spill_manager_objects Number of local objects broken per state {Pinned, PendingRestore, PendingSpill}.
# TYPE ray_spill_manager_objects gauge
ray_spill_manager_objects{Component="raylet",NodeAddress="10.85.24.243",State="Pinned",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects{Component="raylet",NodeAddress="10.85.24.243",State="PendingSpill",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects{Component="raylet",NodeAddress="10.85.24.243",State="PendingRestore",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_infeasible_scheduling_classes The number of unique scheduling classes that are infeasible.
# TYPE ray_internal_num_infeasible_scheduling_classes gauge
ray_internal_num_infeasible_scheduling_classes{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_grpc_server_req_process_time_ms Request latency in grpc server
# TYPE ray_grpc_server_req_process_time_ms histogram
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 5.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 54149.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 54413.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 54419.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 54419.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 54419.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 54419.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54419.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 13846.546709000026
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 3.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 3.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 3.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 3.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 3.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 3.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.5974999999999999
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 2967.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 3109.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 3117.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 3211.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 3795.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 5066.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5066.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5.9442800899537e+07
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 10353.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 164675.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 165107.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 165112.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 165112.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 165112.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 165112.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 165112.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 29198.830607000134
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 24838.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 25033.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 25034.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 25034.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 25034.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 25034.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 25034.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 7568.334535000009
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 2150.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 3345.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 3379.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 3423.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 3423.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 3423.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3423.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 11832.473328999993
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 2556.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 2608.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 2608.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 2608.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 2608.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 2608.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 2608.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 1162.4690850000006
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 54.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 54.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 54.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 54.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 54.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 26.726774999999996
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 2.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 51.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 51.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 51.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 8.521823
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="0.1"} 0.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 16.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 16.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 16.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 16.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 16.0
ray_grpc_server_req_process_time_ms_bucket{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 16.0
ray_grpc_server_req_process_time_ms_count{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 16.0
ray_grpc_server_req_process_time_ms_sum{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 4.451033000000001
# HELP ray_internal_num_processes_started_from_cache_total The total number of workers started from a cached worker process.
# TYPE ray_internal_num_processes_started_from_cache_total counter
ray_internal_num_processes_started_from_cache_total{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_processes_started_from_cache (DEPRECATED, use ray_internal_num_processes_started_from_cache_total metric instead) The total number of workers started from a cached worker process.
# TYPE ray_internal_num_processes_started_from_cache gauge
ray_internal_num_processes_started_from_cache{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_directory_added_locations Number of object locations added per second., If this is high, a lot of objects have been added on this node.
# TYPE ray_object_directory_added_locations gauge
ray_object_directory_added_locations{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_processes_skipped_runtime_enviornment_mismatch_total The total number of cached workers skipped due to runtime environment mismatch.
# TYPE ray_internal_num_processes_skipped_runtime_enviornment_mismatch_total counter
ray_internal_num_processes_skipped_runtime_enviornment_mismatch_total{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_processes_skipped_runtime_enviornment_mismatch (DEPRECATED, use ray_internal_num_processes_skipped_runtime_enviornment_mismatch_total metric instead) The total number of cached workers skipped due to runtime environment mismatch.
# TYPE ray_internal_num_processes_skipped_runtime_enviornment_mismatch gauge
ray_internal_num_processes_skipped_runtime_enviornment_mismatch{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_pull_manager_requested_bundles Number of requested bundles broken per type {Get, Wait, TaskArgs}.
# TYPE ray_pull_manager_requested_bundles gauge
ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.85.24.243",Type="CumulativeTotal",Version="2.40.0.2.dev20250317"} 5252.0
ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.85.24.243",Type="Wait",Version="2.40.0.2.dev20250317"} 0.0
ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.85.24.243",Type="Get",Version="2.40.0.2.dev20250317"} 0.0
ray_pull_manager_requested_bundles{Component="raylet",NodeAddress="10.85.24.243",Type="TaskArgs",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_store_dist The distribution of object size in bytes
# TYPE ray_object_store_dist histogram
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="33554432.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="67108864.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="134217728.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="268435456.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="536870912.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="1073741824.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="2147483648.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="4294967296.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="8589934592.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="17179869184.0"} 57.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317",le="+Inf"} 57.0
ray_object_store_dist_count{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317"} 57.0
ray_object_store_dist_sum{Component="raylet",NodeAddress="10.85.24.243",Source="CreatedByWorker",Version="2.40.0.2.dev20250317"} 6.100001e+06
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="33554432.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="67108864.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="134217728.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="268435456.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="536870912.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="1073741824.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="2147483648.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="4294967296.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="8589934592.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="17179869184.0"} 2608.0
ray_object_store_dist_bucket{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317",le="+Inf"} 2608.0
ray_object_store_dist_count{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317"} 2608.0
ray_object_store_dist_sum{Component="raylet",NodeAddress="10.85.24.243",Source="ReceivedFromRemoteRaylet",Version="2.40.0.2.dev20250317"} 6.5488757e+08
# HELP ray_pull_manager_num_object_pins Number of object pin attempts by the pull manager, can be {Success, Failure}.
# TYPE ray_pull_manager_num_object_pins gauge
ray_pull_manager_num_object_pins{Component="raylet",NodeAddress="10.85.24.243",Type="Failure",Version="2.40.0.2.dev20250317"} 2608.0
ray_pull_manager_num_object_pins{Component="raylet",NodeAddress="10.85.24.243",Type="Success",Version="2.40.0.2.dev20250317"} 3083.0
# HELP ray_worker_register_time_ms end to end latency of register a worker process.
# TYPE ray_worker_register_time_ms histogram
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="100.0"} 0.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="1000.0"} 3.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="10000.0"} 3.0
ray_worker_register_time_ms_bucket{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",le="+Inf"} 3.0
ray_worker_register_time_ms_count{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
ray_worker_register_time_ms_sum{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 1254.0
# HELP ray_push_manager_chunks Number of object chunks transfer broken per type {InFlight, Remaining}.
# TYPE ray_push_manager_chunks gauge
ray_push_manager_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="Remaining",Version="2.40.0.2.dev20250317"} 0.0
ray_push_manager_chunks{Component="raylet",NodeAddress="10.85.24.243",Type="InFlight",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_manager_bytes Number of bytes pushed or received by type {PushedFromLocalPlasma, PushedFromLocalDisk, Received}.
# TYPE ray_object_manager_bytes gauge
ray_object_manager_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="PushedFromLocalDisk",Version="2.40.0.2.dev20250317"} 0.0
ray_object_manager_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="Received",Version="2.40.0.2.dev20250317"} 6.5488757e+08
ray_object_manager_bytes{Component="raylet",NodeAddress="10.85.24.243",Type="PushedFromLocalPlasma",Version="2.40.0.2.dev20250317"} 5.79055e+06
# HELP ray_pull_manager_retries_total Number of cumulative pull retries.
# TYPE ray_pull_manager_retries_total gauge
ray_pull_manager_retries_total{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 2608.0
# HELP ray_scheduler_tasks Number of tasks waiting for scheduling broken per state {Cancelled, Executing, Waiting, Dispatched, Received}.
# TYPE ray_scheduler_tasks gauge
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="Dispatched",Version="2.40.0.2.dev20250317"} 1.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="Cancelled",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="Received",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="SpilledUnschedulable",Version="2.40.0.2.dev20250317"} 1938.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="Executing",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="SpilledWaiting",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_tasks{Component="raylet",NodeAddress="10.85.24.243",State="Waiting",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_scheduler_unscheduleable_tasks Number of pending tasks (not scheduleable tasks) broken per reason {Infeasible, WaitingForResources, WaitingForPlasmaMemory, WaitingForRemoteResources, WaitingForWorkers}.
# TYPE ray_scheduler_unscheduleable_tasks gauge
ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.85.24.243",Reason="Infeasible",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.85.24.243",Reason="WaitingForResources",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.85.24.243",Reason="WaitingForPlasmaMemory",Version="2.40.0.2.dev20250317"} 0.0
ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.85.24.243",Reason="WaitingForRemoteResources",Version="2.40.0.2.dev20250317"} 1.0
ray_scheduler_unscheduleable_tasks{Component="raylet",NodeAddress="10.85.24.243",Reason="WaitingForWorkers",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_spill_manager_objects_bytes Byte size of local objects broken per state {Pinned, PendingSpill}.
# TYPE ray_spill_manager_objects_bytes gauge
ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.85.24.243",State="Restored",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.85.24.243",State="Pinned",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.85.24.243",State="PendingSpill",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.85.24.243",State="PendingRestore",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_objects_bytes{Component="raylet",NodeAddress="10.85.24.243",State="Spilled",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_grpc_server_req_handling_total Request number are handling in grpc server
# TYPE ray_grpc_server_req_handling_total counter
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetResourceLoad",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54419.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetSystemConfig",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.RequestWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5067.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.ReportWorkerBacklog",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 165112.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.FreeObjects",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 25034.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.GetNodeStats",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3423.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.Push",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 2608.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.PinObjectIDs",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 54.0
ray_grpc_server_req_handling_total{Component="raylet",Method="ObjectManagerService.grpc_server.Pull",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 51.0
ray_grpc_server_req_handling_total{Component="raylet",Method="NodeManagerService.grpc_server.CancelWorkerLease",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 16.0
# HELP ray_resources Logical Ray resources broken per state {AVAILABLE, USED}
# TYPE ray_resources gauge
ray_resources{Component="raylet",Name="object_store_memory",NodeAddress="10.85.24.243",State="AVAILABLE",Version="2.40.0.2.dev20250317"} 1.6320875724e+010
ray_resources{Component="raylet",Name="CPU",NodeAddress="10.85.24.243",State="AVAILABLE",Version="2.40.0.2.dev20250317"} 0.0
ray_resources{Component="raylet",Name="object_store_memory",NodeAddress="10.85.24.243",State="USED",Version="2.40.0.2.dev20250317"} 0.0
ray_resources{Component="raylet",Name="CPU",NodeAddress="10.85.24.243",State="USED",Version="2.40.0.2.dev20250317"} 4.0
ray_resources{Component="raylet",Name="memory",NodeAddress="10.85.24.243",State="USED",Version="2.40.0.2.dev20250317"} 0.0
ray_resources{Component="raylet",Name="memory",NodeAddress="10.85.24.243",State="AVAILABLE",Version="2.40.0.2.dev20250317"} 1.7179869184e+010
# HELP ray_spill_manager_request_total Number of {spill, restore} requests.
# TYPE ray_spill_manager_request_total gauge
ray_spill_manager_request_total{Component="raylet",NodeAddress="10.85.24.243",Type="FailedDeletion",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_request_total{Component="raylet",NodeAddress="10.85.24.243",Type="Spilled",Version="2.40.0.2.dev20250317"} 0.0
ray_spill_manager_request_total{Component="raylet",NodeAddress="10.85.24.243",Type="Restored",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_processes_started_total The total number of worker processes the worker pool has created.
# TYPE ray_internal_num_processes_started_total counter
ray_internal_num_processes_started_total{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
# HELP ray_internal_num_processes_started (DEPRECATED, use ray_internal_num_processes_started_total metric instead) The total number of worker processes the worker pool has created.
# TYPE ray_internal_num_processes_started gauge
ray_internal_num_processes_started{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 3.0
# HELP ray_internal_num_processes_skipped_job_mismatch_total The total number of cached workers skipped due to job mismatch.
# TYPE ray_internal_num_processes_skipped_job_mismatch_total counter
ray_internal_num_processes_skipped_job_mismatch_total{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_processes_skipped_job_mismatch (DEPRECATED, use ray_internal_num_processes_skipped_job_mismatch_total metric instead) The total number of cached workers skipped due to job mismatch.
# TYPE ray_internal_num_processes_skipped_job_mismatch gauge
ray_internal_num_processes_skipped_job_mismatch{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_internal_num_spilled_tasks The cumulative number of lease requeusts that this raylet has spilled to other raylets.
# TYPE ray_internal_num_spilled_tasks gauge
ray_internal_num_spilled_tasks{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 5.0
# HELP ray_object_store_available_memory Amount of memory currently available in the object store.
# TYPE ray_object_store_available_memory gauge
ray_object_store_available_memory{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 1.6320875724e+010
# HELP ray_object_store_num_local_objects Number of objects currently in the object store.
# TYPE ray_object_store_num_local_objects gauge
ray_object_store_num_local_objects{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_object_manager_num_pull_requests Number of active pull requests for objects.
# TYPE ray_object_manager_num_pull_requests gauge
ray_object_manager_num_pull_requests{Component="raylet",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_serve_deployment_replica_starts_total The number of times this replica has been restarted due to failure.
# TYPE ray_serve_deployment_replica_starts_total counter
ray_serve_deployment_replica_starts_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2"} 1.0
ray_serve_deployment_replica_starts_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5"} 1.0
# HELP ray_serve_deployment_replica_starts (DEPRECATED, use ray_serve_deployment_replica_starts_total metric instead) The number of times this replica has been restarted due to failure.
# TYPE ray_serve_deployment_replica_starts gauge
ray_serve_deployment_replica_starts{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2"} 1.0
ray_serve_deployment_replica_starts{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5"} 1.0
# HELP ray_total_lineage_bytes Total amount of memory used to store task specs for lineage reconstruction.
# TYPE ray_total_lineage_bytes gauge
ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
ray_total_lineage_bytes{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_actors Current number of actors currently in a particular state.
# TYPE ray_actors gauge
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="IDLE",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_TASK",Version="2.40.0.2.dev20250317"} 1.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="IDLE",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_TASK",Version="2.40.0.2.dev20250317"} 1.0
ray_actors{Component="core_worker",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_TASK",Version="2.40.0.2.dev20250317"} 1.0
ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.85.24.243",Source="executor",State="IDLE",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_actors{Component="core_worker",JobId="01000000",Name="ProxyActor",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
# HELP ray_tasks Current number of tasks currently in a particular state.
# TYPE ray_tasks gauge
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 20243.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 2.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 5.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} -5.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 18.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.initialize_and_get_metadata",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.is_allocated",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 20251.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 1.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 5.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} -5.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 13.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 1.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="Unknown task",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 4291.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_http_config",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 2.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.__init__",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.ready",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 1.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.check_health",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 349152.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.list_deployment_ids",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.get_root_url",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 1.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 2754.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 16.0
ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="1",JobId="",Name="ServeController.listen_for_change",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 16.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="owner",State="PENDING_ARGS_AVAIL",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="owner",State="PENDING_NODE_ASSIGNMENT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="owner",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} 7.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="executor",State="RUNNING",Version="2.40.0.2.dev20250317"} 4.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="executor",State="SUBMITTED_TO_WORKER",Version="2.40.0.2.dev20250317"} -4.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_WAIT",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="01000000",Name="ProxyActor.receive_asgi_messages",NodeAddress="10.85.24.243",Source="executor",State="RUNNING_IN_RAY_GET",Version="2.40.0.2.dev20250317"} 0.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="owner",State="FINISHED",Version="2.40.0.2.dev20250317"} 19275.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.handle_request_with_rejection",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 85.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.get_num_ongoing_requests",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 479.0
ray_tasks{Component="core_worker",IsRetry="0",JobId="",Name="ServeReplica:video_embedding:EmbeddingRayServe.push_proxy_handle",NodeAddress="10.85.24.243",Source="owner",State="FAILED",Version="2.40.0.2.dev20250317"} 174.0
# HELP ray_serve_deployment_processing_latency_ms The latency for queries to be processed.
# TYPE ray_serve_deployment_processing_latency_ms histogram
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="1.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="2.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="5.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="10.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="20.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="50.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="100.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="200.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="300.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="400.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="500.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="1000.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="2000.0",replica="cgo8bpm2",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="5000.0",replica="cgo8bpm2",route="/embedding"} 1.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="10000.0",replica="cgo8bpm2",route="/embedding"} 3.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="60000.0",replica="cgo8bpm2",route="/embedding"} 10132.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="120000.0",replica="cgo8bpm2",route="/embedding"} 10133.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="300000.0",replica="cgo8bpm2",route="/embedding"} 10133.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="600000.0",replica="cgo8bpm2",route="/embedding"} 10133.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="+Inf",replica="cgo8bpm2",route="/embedding"} 10133.0
ray_serve_deployment_processing_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 10133.0
ray_serve_deployment_processing_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 2.7124902237200755e+08
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="1.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="2.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="5.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="10.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="20.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="50.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="100.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="200.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="300.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="400.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="500.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="1000.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="2000.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="5000.0",replica="gvo4xms5",route="/embedding"} 0.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="10000.0",replica="gvo4xms5",route="/embedding"} 1.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="60000.0",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="120000.0",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="300000.0",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="600000.0",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",le="+Inf",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 10134.0
ray_serve_deployment_processing_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 2.714188902065758e+08
# HELP ray_serve_replica_processing_queries The current number of queries being processed.
# TYPE ray_serve_replica_processing_queries gauge
ray_serve_replica_processing_queries{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2"} 5.0
ray_serve_replica_processing_queries{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5"} 5.0
# HELP ray_serve_deployment_request_counter_total The number of queries that have been processed in this replica.
# TYPE ray_serve_deployment_request_counter_total counter
ray_serve_deployment_request_counter_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 10112.0
ray_serve_deployment_request_counter_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 10112.0
# HELP ray_serve_deployment_request_counter (DEPRECATED, use ray_serve_deployment_request_counter_total metric instead) The number of queries that have been processed in this replica.
# TYPE ray_serve_deployment_request_counter gauge
ray_serve_deployment_request_counter{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 10112.0
ray_serve_deployment_request_counter{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 10112.0
# HELP ray_serve_deployment_error_counter_total The number of exceptions that have occurred in this replica.
# TYPE ray_serve_deployment_error_counter_total counter
ray_serve_deployment_error_counter_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 21.0
ray_serve_deployment_error_counter_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 22.0
# HELP ray_serve_deployment_error_counter (DEPRECATED, use ray_serve_deployment_error_counter_total metric instead) The number of exceptions that have occurred in this replica.
# TYPE ray_serve_deployment_error_counter gauge
ray_serve_deployment_error_counter{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="cgo8bpm2",route="/embedding"} 21.0
ray_serve_deployment_error_counter{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",replica="gvo4xms5",route="/embedding"} 22.0
# HELP ray_serve_http_request_latency_ms The end-to-end latency of HTTP requests (measured from the Serve HTTP proxy).
# TYPE ray_serve_http_request_latency_ms histogram
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="1.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="2.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="5.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="10.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="20.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="50.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="100.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="200.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="300.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="400.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="500.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="1000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="2000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="5000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="10000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="60000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="120000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="300000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="600000.0",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="+Inf",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="503"} 0.048160552978515625
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="1.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="2.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="5.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="10.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="20.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="50.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="100.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="200.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="300.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="400.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="500.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="1000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="2000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="5000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="10000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="60000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="120000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="300000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="600000.0",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",le="+Inf",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="200"} 165.82870483398466
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1.0",method="POST",route="/embedding",status_code="503"} 24772.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2.0",method="POST",route="/embedding",status_code="503"} 49301.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5.0",method="POST",route="/embedding",status_code="503"} 51061.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10.0",method="POST",route="/embedding",status_code="503"} 51141.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="20.0",method="POST",route="/embedding",status_code="503"} 51175.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="50.0",method="POST",route="/embedding",status_code="503"} 51235.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="100.0",method="POST",route="/embedding",status_code="503"} 51294.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="200.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="400.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="500.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="60000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="120000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="600000.0",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="+Inf",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="503"} 64969.77138519318
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="20.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="50.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="100.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="200.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="400.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="500.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1000.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2000.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5000.0",method="POST",route="/embedding",status_code="200"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10000.0",method="POST",route="/embedding",status_code="200"} 9.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="60000.0",method="POST",route="/embedding",status_code="200"} 17924.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="120000.0",method="POST",route="/embedding",status_code="200"} 18241.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300000.0",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="600000.0",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="+Inf",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="200"} 6.004929303505431e+08
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="20.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="50.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="100.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="200.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="400.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="500.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="1000.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="2000.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="5000.0",method="POST",route="/embedding",status_code="500"} 0.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="10000.0",method="POST",route="/embedding",status_code="500"} 6.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="60000.0",method="POST",route="/embedding",status_code="500"} 69.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="120000.0",method="POST",route="/embedding",status_code="500"} 76.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="300000.0",method="POST",route="/embedding",status_code="500"} 85.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="600000.0",method="POST",route="/embedding",status_code="500"} 85.0
ray_serve_http_request_latency_ms_bucket{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",le="+Inf",method="POST",route="/embedding",status_code="500"} 85.0
ray_serve_http_request_latency_ms_count{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="500"} 85.0
ray_serve_http_request_latency_ms_sum{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="500"} 3.5523753044605246e+06
# HELP ray_serve_num_scheduling_tasks The number of request scheduling tasks in the router.
# TYPE ray_serve_num_scheduling_tasks gauge
ray_serve_num_scheduling_tasks{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",app="video_embedding",deployment="EmbeddingRayServe"} 2.0
# HELP ray_serve_num_http_error_requests_total The number of errored HTTP responses.
# TYPE ray_serve_num_http_error_requests_total counter
ray_serve_num_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",error_code="503",method="GET",route="/-/healthz"} 1.0
ray_serve_num_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",error_code="503",method="POST",route="/embedding"} 51302.0
ray_serve_num_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",error_code="500",method="POST",route="/embedding"} 85.0
# HELP ray_serve_num_http_error_requests (DEPRECATED, use ray_serve_num_http_error_requests_total metric instead) The number of errored HTTP responses.
# TYPE ray_serve_num_http_error_requests gauge
ray_serve_num_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",error_code="503",method="GET",route="/-/healthz"} 1.0
ray_serve_num_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",error_code="503",method="POST",route="/embedding"} 51302.0
ray_serve_num_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",error_code="500",method="POST",route="/embedding"} 85.0
# HELP ray_serve_num_http_requests_total The number of HTTP requests processed.
# TYPE ray_serve_num_http_requests_total counter
ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_num_http_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="500"} 85.0
# HELP ray_serve_num_http_requests (DEPRECATED, use ray_serve_num_http_requests_total metric instead) The number of HTTP requests processed.
# TYPE ray_serve_num_http_requests gauge
ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="503"} 1.0
ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",method="GET",route="/-/healthz",status_code="200"} 1835.0
ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="503"} 51302.0
ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="200"} 18243.0
ray_serve_num_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",method="POST",route="/embedding",status_code="500"} 85.0
# HELP ray_serve_deployment_queued_queries The current number of queries to this deployment waiting to be assigned to a replica.
# TYPE ray_serve_deployment_queued_queries gauge
ray_serve_deployment_queued_queries{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",application="video_embedding",deployment="EmbeddingRayServe",handle="7e6akuk0"} 2.0
# HELP ray_serve_num_deployment_http_error_requests_total The number of errored HTTP responses returned by each deployment.
# TYPE ray_serve_num_deployment_http_error_requests_total counter
ray_serve_num_deployment_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",deployment="",error_code="503",method="GET",route="/-/healthz"} 1.0
ray_serve_num_deployment_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",error_code="503",method="POST",route="/embedding"} 51302.0
ray_serve_num_deployment_http_error_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",error_code="500",method="POST",route="/embedding"} 85.0
# HELP ray_serve_num_deployment_http_error_requests (DEPRECATED, use ray_serve_num_deployment_http_error_requests_total metric instead) The number of errored HTTP responses returned by each deployment.
# TYPE ray_serve_num_deployment_http_error_requests gauge
ray_serve_num_deployment_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="",deployment="",error_code="503",method="GET",route="/-/healthz"} 1.0
ray_serve_num_deployment_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",error_code="503",method="POST",route="/embedding"} 51302.0
ray_serve_num_deployment_http_error_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",error_code="500",method="POST",route="/embedding"} 85.0
# HELP ray_serve_num_scheduling_tasks_in_backoff The number of request scheduling tasks in the router that are undergoing backoff.
# TYPE ray_serve_num_scheduling_tasks_in_backoff gauge
ray_serve_num_scheduling_tasks_in_backoff{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",app="video_embedding",deployment="EmbeddingRayServe"} 2.0
# HELP ray_serve_num_router_requests_total The number of requests processed by the router.
# TYPE ray_serve_num_router_requests_total counter
ray_serve_num_router_requests_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",application="video_embedding",deployment="EmbeddingRayServe",handle="7e6akuk0",route="/embedding"} 18337.0
# HELP ray_serve_num_router_requests (DEPRECATED, use ray_serve_num_router_requests_total metric instead) The number of requests processed by the router.
# TYPE ray_serve_num_router_requests gauge
ray_serve_num_router_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",application="video_embedding",deployment="EmbeddingRayServe",handle="7e6akuk0",route="/embedding"} 18337.0
# HELP ray_serve_handle_request_counter_total The number of handle.remote() calls that have been made on this handle.
# TYPE ray_serve_handle_request_counter_total counter
ray_serve_handle_request_counter_total{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",handle="video_embedding#EmbeddingRayServe#7e6akuk0",route="/embedding"} 69639.0
# HELP ray_serve_handle_request_counter (DEPRECATED, use ray_serve_handle_request_counter_total metric instead) The number of handle.remote() calls that have been made on this handle.
# TYPE ray_serve_handle_request_counter gauge
ray_serve_handle_request_counter{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",application="video_embedding",deployment="EmbeddingRayServe",handle="video_embedding#EmbeddingRayServe#7e6akuk0",route="/embedding"} 69639.0
# HELP ray_serve_num_ongoing_http_requests The number of ongoing requests in this HTTP proxy.
# TYPE ray_serve_num_ongoing_http_requests gauge
ray_serve_num_ongoing_http_requests{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",node_id="6d7be4adc802a5f42f9b6abf496d2f0f29a9df2e30e67117829fc9da",node_ip_address="10.85.24.243"} 9.0
# HELP ray_serve_num_ongoing_requests_at_replicas The current number of requests to this deployment that have been submitted to a replica.
# TYPE ray_serve_num_ongoing_requests_at_replicas gauge
ray_serve_num_ongoing_requests_at_replicas{Component="core_worker",NodeAddress="10.85.24.243",Version="2.40.0.2.dev20250317",actor_id="cfa3240df7a186e814122d0301000000",application="video_embedding",deployment="EmbeddingRayServe",handle="7e6akuk0"} 7.0