{ "nodes": { "sensors": { "title": "Robot + wrist camera", "body": "The robot supplies measured feedback. The wrist RGB image is the only image input to the expert; no simulator object pose enters this boundary.", "inputs": "D415 RGB; joint encoders; EE pose; gripper feedback", "outputs": "RGB 320 \u00d7 180; 16-value robot state; gripper busy / validity / age / contact; camera frame ID", "source": "TubeV3Rig.read \u00b7 app.Runner._observe", "kind": "input" }, "calibration": { "title": "Fixed calibration + task priors", "body": "Intrinsics K and hand-eye transform map camera observations into the robot/workcell frames. Table height, source and destination search centres and tube geometry are still configured priors. Real calibration remains provisional.", "inputs": "K; T_EE_camera; T_world_base; table height; search regions", "outputs": "Metric camera rays; configured search / placement regions", "source": "Workcell \u00b7 Calibration", "kind": "input" }, "labobs": { "title": "Lab observation adapter", "body": "The real camera stream is 1280 \u00d7 720 at 30 fps. It is area-resized to 320 \u00d7 180. The 20 Hz observer records and supplies the latest measured robot state. The API worker requests updates at up to 5 Hz.", "inputs": "RGB + measured feedback", "outputs": "Latest observation snapshot; no object labels", "source": "RemoteExpertPolicy._predict_feedback", "kind": "process" }, "api": { "title": "A100 observation API", "body": "A session holds the expert state and fixed calibration. Each request includes a strictly increasing sequence and observation time. Invalid data or an old sequence is rejected.", "inputs": "RGB bytes; q[6]; gap_m; EE xyz[3]; two rotation columns[6]; metadata", "outputs": "Observation + per-session FSM state", "source": "server.observation \u00b7 /session \u00b7 /act", "kind": "process" }, "inspect_gate": { "title": "Inspection state?", "body": "Heavy perception runs only on the acquisition path. Normal approach, descent, transport and release ticks use the cached object estimate and measured robot feedback.", "inputs": "Current phase; cached candidates", "outputs": "Perception call OR ordinary FSM update", "source": "VisionExpert.act", "kind": "decision" }, "models": { "title": "SAM3 + FoundationStereo", "body": "SAM3 segments red tube instances in the left wrist image. FoundationStereo estimates disparity from two rectified wrist views separated by a commanded 50 mm translation. Their outputs meet in metric geometry fitting.", "inputs": "Temporal image pair; camera calibration; measured camera poses", "outputs": "3D tube centres, axes, masks and scores", "source": "stereo_tubes \u00b7 SamSegmenter \u00b7 FoundationDepth", "kind": "model" }, "planner": { "title": "Geometric grasp search", "body": "Searches axial grasp offsets and scores configured pad/wall clearance, other perceived tubes and failed locations. This is a small geometric grasp planner, not a global collision-free arm planner.", "inputs": "Tube candidates; source region; failed grasp history", "outputs": "Selected tube, offset, grasp orientation and clearance", "source": "choose_grasp", "kind": "process" }, "fsm": { "title": "Scripted finite-state expert", "body": "Code selects goals and gripper commands. Measured arrival gates most motions. Real-only guards wait for descent progress and gripper close completion. The placement revision additionally compensates the carried object and terminates after two confirmed releases.", "inputs": "Observation; perceived candidates; phase history; gripper feedback", "outputs": "Absolute EE target + physical width + speed caps + status", "source": "VisionExpert.act \u00b7 RealExpert._act_legacy", "kind": "process" }, "command": { "title": "Shared motion command", "body": "The API returns an absolute pose in robot base, open_width_mm, speed caps and session/sequence. Zero width means closed; values above the hardware maximum are clipped by the driver.", "inputs": "FSM goal in workcell coordinates", "outputs": "T_base_EE; open_width_mm; m/s and rad/s caps", "source": "WidthCommand \u00b7 PoseCommand", "kind": "output" }, "ttl": { "title": "Fresh, valid command?", "body": "Expiry is request-start + 350 ms, not response-arrival + 350 ms. Slow model/network responses do not renew an old target. Local observations also have a 200 ms watchdog.", "inputs": "Validated command; local monotonic clock", "outputs": "Follow a fresh target OR hold", "source": "RemoteExpertPolicy._predict_feedback", "kind": "decision" }, "hold": { "title": "Hold / normal completion / fault", "body": "On expiry, hold reseeds the arm reference from measured joints and issues no new gripper command. Network/protocol faults stop the runner. In v2, a completed outcome holds and stops normally; it does not become a fault.", "inputs": "Expired, failed or completed command", "outputs": "Measured-position hold or stopped servo stream", "source": "TrajectoryController.step \u00b7 Runner.stop / _fail", "kind": "stop" }, "controller": { "title": "Local persistent controller \u00b7 20 Hz", "body": "The shared controller advances a persistent joint reference using DLS IK. It gates reference motion against measured tracking and configured workspace/joint bounds. NN deltas are adapted at this same boundary.", "inputs": "PoseCommand + measured q + robot kinematics", "outputs": "Bounded joint reference", "source": "TrajectoryController.step", "kind": "process" }, "driver": { "title": "Robot / gripper drivers", "body": "Joint targets are interpolated and streamed at 125 Hz to the existing UR servoJ driver. Gripper width is executed independently; RG2 clips above 100.8 mm. Arm speed scale zero also inhibits new gripper commands.", "inputs": "Joint reference; open_width_mm", "outputs": "Arm motion; gripper motion; measured feedback", "source": "TubeV3Runner._stream \u00b7 TubeV3Rig.command_open_width", "kind": "output" }, "nn": { "title": "NN compatibility adapter", "body": "The local NN retains its existing base-frame delta meaning. Its delta is applied once to the observation pose; its legacy gripper value is converted at the adapter. It shares the local controller with the scripted expert.", "inputs": "NN action + observation pose", "outputs": "Same PoseCommand boundary", "source": "motion_contract.policy_command", "kind": "input" }, "settle": { "title": "settle \u2192 overview", "body": "After 3 seconds, construct a top-down search pose that aims the wrist camera at the configured source centre. This uses the current wrist yaw and fixed hand-eye transform.", "inputs": "Initial measured pose; calibration; source prior", "outputs": "Overview EE goal; open gripper", "source": "VisionExpert.act: settle", "kind": "process" }, "overview": { "title": "overview \u2192 inspect", "body": "Wait until translation error < 4 mm and orientation error < 0.06 rad. Unless explicitly listed otherwise, a false guard holds the existing goal and stays in the same state.", "inputs": "Measured EE pose; overview goal", "outputs": "Enter inspect", "source": "VisionExpert.arrived", "kind": "process" }, "left": { "title": "inspect: save left view", "body": "If no cached stereo candidates exist, save the entire current observation as the left view. An empty candidate list is distinct from an uncomputed estimate.", "inputs": "RGB_left; measured T_base_EE; K and hand-eye", "outputs": "Left observation + stereo-motion goal", "source": "VisionExpert.act: inspect", "kind": "process" }, "baseline": { "title": "stereo_move", "body": "Translate the EE by 50 mm along the current camera X direction. The right view is accepted only after measured arrival within 1 mm and 0.06 rad, then a 0.4 s settling wait.", "inputs": "Left camera pose; measured EE feedback", "outputs": "Right RGB + right measured camera pose", "source": "VisionExpert.act: stereo_move / stereo_wait", "kind": "process" }, "rectify": { "title": "Rectify measured camera pair", "body": "Compute the relative camera transform from measured EE poses and hand-eye calibration. Reject a measured baseline outside 20\u2013120 mm. Rectify images using K and that transform.", "inputs": "Left/right RGB; K; measured T_base_camera", "outputs": "Rectified stereo pair; Q; image remap", "source": "FoundationDepth.estimate", "kind": "process" }, "fs": { "title": "FoundationStereo", "body": "Dense stereo inference on the rectified pair. For 320-pixel input, images are upscaled 2\u00d7 for inference; disparity is resized back. It uses 32 iterations. This is stereo depth, not a single-image depth model.", "inputs": "Rectified left and right RGB", "outputs": "Dense disparity \u2192 camera-frame point cloud", "source": "FoundationDepth.estimate", "kind": "model" }, "sam": { "title": "SAM3 instance segmentation", "body": "Prompt: \u201cred tube\u201d. Segment the left image into instance masks and scores. Confidence threshold is 0.55. No simulator segmentation or instance IDs are supplied.", "inputs": "Left wrist RGB", "outputs": "Tube masks + confidence scores", "source": "SamSegmenter.masks", "kind": "model" }, "fuse": { "title": "Mask + metric point-cloud fusion", "body": "Remap each mask into the rectified left frame and erode it with a 3 \u00d7 3 kernel. Select valid 3D points in that instance and transform them to the workcell frame.", "inputs": "SAM3 masks; rectification map; stereo XYZ; camera pose", "outputs": "3D point set per tube candidate", "source": "stereo_tubes", "kind": "process" }, "fit": { "title": "Fit tube centre and axis", "body": "Require at least 20 valid points. Keep heights between table_z and table_z + 150 mm and centres within 140 mm of the source prior. Use planar PCA and robust cylinder fitting with a 10 mm radius prior. Reject fit RMS > 5 mm.", "inputs": "Filtered tube surface points; geometry prior", "outputs": "Centre xyz; horizontal axis; mask; score", "source": "stereo_tubes", "kind": "process" }, "cache": { "title": "Cache candidates; return to left pose", "body": "Record perception latency. Wait until perception_wait age reaches that latency; return to the left-view pose and re-enter inspect. The cached estimate is then used for grasp selection. Slow API results meanwhile expire into a local hold.", "inputs": "Candidate list + saved left pose", "outputs": "Cached candidates at a repeatable view", "source": "VisionExpert.act: perception_wait / stereo_return", "kind": "process" }, "candidates": { "title": "Any candidates?", "body": "With a computed nonempty list, choose a grasp. An empty list waits until inspect age > 2 s, then enters no_detection. Empty-source perception is a fault/stop in v1, not proof of task completion.", "inputs": "Cached candidates; inspect age", "outputs": "Grasp candidate OR no_detection", "source": "VisionExpert.act: inspect", "kind": "decision" }, "no_detection": { "title": "no_detection", "body": "No usable tube candidate was found. The API marks this terminal and the real client raises a policy stop. This is not labelled success.", "inputs": "Empty candidate list after inspect wait", "outputs": "Terminal stop", "source": "server.py \u00b7 RemoteExpertPolicy.validate_result", "kind": "stop" }, "grasp_offsets": { "title": "Search axial grasp offsets", "body": "Try 0, \u00b112, \u00b124 and \u00b130 mm along each perceived tube. Score pad footprint clearance to the configured source box and other tubes; penalize proximity to failed grasp points. There is no continuous arm collision planner here.", "inputs": "Candidates + source dimensions + failures", "outputs": "Selected tube, grasp XY, offset and clearance", "source": "choose_grasp", "kind": "process" }, "retry_gate": { "title": "Near a failed grasp?", "body": "If the selected tube centre is within 45 mm of a failed point AND fewer than 3 nudges have occurred, try a nudge. Otherwise approach the chosen grasp directly.", "inputs": "Selected tube; failed points; nudge count", "outputs": "nudge_approach OR approach", "source": "VisionExpert.act: inspect", "kind": "decision" }, "approach": { "title": "approach", "body": "Set XY to perceived centre + chosen axial offset; height = table + 100 mm. Align jaws across the tube, selecting the nearer equivalent wrist yaw. Open to the legacy-equivalent physical gap (~28.04 mm).", "inputs": "Chosen tube + offset + current wrist orientation", "outputs": "Approach goal + opening width", "source": "VisionExpert.act: inspect / approach", "kind": "process" }, "refine": { "title": "refine \u00b7 0.35 s wait", "body": "With FoundationStereo enabled, this state does NOT rerun SAM3 or depth. After age > 0.35 s it uses the cached tube to set descent Z = max(table + 10 mm, centre_z \u2212 5 mm). The color-only fallback can re-detect here, but that is not the deployed stereo path.", "inputs": "Cached tube geometry; measured arrival", "outputs": "Descend goal", "source": "VisionExpert.act: refine", "kind": "process" }, "descend": { "title": "descend: measured arrival?", "body": "Descend at a requested 35 mm/s. Require < 2 mm translation error and < 0.06 rad rotation error before close. Otherwise use progress checks, not the frozen expert\u2019s unconditional 6.5 s abort.", "inputs": "Measured pose + descent goal", "outputs": "Close OR continue descent / recover", "source": "RealExpert._act_legacy", "kind": "decision" }, "progress": { "title": "Progress and travel budget", "body": "Recover if distance improves by less than 0.5 mm over a window of at least 1 s, or travel time exceeds min(18 s, max(6.5 s, initial_distance / 0.008 + 2 s)). These are heuristic guards, not force/contact estimation.", "inputs": "Distance history + elapsed time", "outputs": "Continue goal OR recover_up", "source": "RealExpert.guarded_motion", "kind": "decision" }, "close": { "title": "close: gripper completion", "body": "Request ~11.72 mm (legacy-equivalent close target). Wait \u2265 0.8 s, not busy, valid feedback younger than 0.6 s, and a post-command sample. Closure evidence: saw busy, contact detected, OR measured gap < 24 mm. Fault after 14 s without confirmation.", "inputs": "Gap; busy; validity; feedback age; contact", "outputs": "Lift OR wait / hardware_fault", "source": "RealExpert._act_legacy: close", "kind": "decision" }, "lift": { "title": "lift: grasp gap plausible?", "body": "Lift to table + 140 mm. On measured arrival (< 4 mm, < 0.06 rad), a gap outside [14, 32] mm is treated as a failed grasp: record the location, open, return and re-perceive. A gap in range allows transport; this is not visual grasp verification.", "inputs": "Measured EE pose + measured gripper gap", "outputs": "Transport OR failed-grasp return", "source": "VisionExpert.act: lift", "kind": "decision" }, "transport": { "title": "transport", "body": "Move at clearance height to the destination, rotating the wrist into a tube-aligned placement orientation. In v1, the destination target is the EE centre. In v2, target XY compensates the measured object-in-gripper offset and selects an interior slot.", "inputs": "Measured grasp; destination prior; placement orientation", "outputs": "Destination clearance goal", "source": "VisionExpert.act: lift \u00b7 RealExpert.plan_placement", "kind": "process" }, "recover": { "title": "recover_up", "body": "Record the failed grasp, increment retries, open to ~28.04 mm and lift at the current XY to table + 130 mm. On arrival, return to the source overview.", "inputs": "Stalled / budget-exhausted contact motion", "outputs": "Clearance goal \u2192 return", "source": "RealExpert.recover_motion", "kind": "process" }, "return": { "title": "return \u2192 inspect", "body": "Return to the saved source overview. On arrival, increment the old cycle counter, clear stereo candidates and inspect again. A cycle can include a failure, so it is not a successful-transfer count.", "inputs": "Saved home/overview pose; measured arrival", "outputs": "Fresh temporal stereo acquisition", "source": "VisionExpert.act: return", "kind": "process" }, "fault": { "title": "hardware_fault / API fault", "body": "Missing gripper completion, invalid inputs, failed protocol checks or controller/watchdog failures stop execution. Faults do not count as completed transfers.", "inputs": "Fault condition", "outputs": "Robot hold and fault status", "source": "RealExpert \u00b7 RemoteExpertPolicy \u00b7 Runner", "kind": "stop" }, "nudge_approach": { "title": "nudge_approach", "body": "Aim over the tube centre at table + 100 mm. Use a narrow ~5.11 mm opening. Push direction points toward the source centre, or perpendicular to the tube if already near the centre.", "inputs": "Tube centre/axis; source prior", "outputs": "Nudge approach goal", "source": "VisionExpert.act: inspect", "kind": "process" }, "nudge_down": { "title": "nudge_down", "body": "On approach arrival, lower to tube centre_z + 11 mm. Require measured arrival within 3 mm; v1/v2 real guards recover on stall or travel-budget exhaustion. The frozen 5 s early-advance condition is bypassed.", "inputs": "Measured pose + tube estimate", "outputs": "Contact approach goal", "source": "RealExpert._act_legacy \u00b7 guarded_motion", "kind": "process" }, "nudge_press": { "title": "nudge_press", "body": "Set the target to tube centre_z + 5 mm. After > 0.5 s, advance to the push goal. This state remains time-based; it is not a force-controlled contact confirmation.", "inputs": "Tube estimate + elapsed time", "outputs": "45 mm push target at contact height", "source": "VisionExpert.act: nudge_press", "kind": "process" }, "nudge_push": { "title": "nudge_push", "body": "Push 45 mm along the chosen direction. Require measured arrival within 5 mm, or recover on stall/travel budget. On arrival increment nudge count, remove failed points within 60 mm of the tube centre, then rise and re-perceive.", "inputs": "Measured pose; push target; history", "outputs": "Recovery rise + updated history", "source": "RealExpert._act_legacy \u00b7 VisionExpert.act", "kind": "process" }, "lower_v1": { "title": "lower \u00b7 previous v1", "body": "After transport arrival (< 4 mm / 0.06 rad), set destination EE Z to table + 50 mm. On arrival at that goal, request release. The gripper centre, not the tube centre, targets the same XY on every transfer.", "inputs": "Fixed destination EE goal + measured arrival", "outputs": "Release goal; ~49.54 mm opening", "source": "VisionExpert.act: transport / lower", "kind": "process" }, "release_v1": { "title": "release \u00b7 timer only", "body": "After release age > 0.7 s, raise the wrist. This does not wait for measured gripper completion. In the latest real run, opening took 1.79 s and 2.20 s.", "inputs": "Elapsed release time", "outputs": "Retreat goal", "source": "VisionExpert.act: release", "kind": "warning" }, "retreat_v1": { "title": "retreat \u2192 return", "body": "Retreat to table + 140 mm. On measured arrival, return to the source overview and open further. There is no \u201ctwo tubes done\u201d guard in v1.", "inputs": "Measured EE arrival", "outputs": "Source return \u2192 inspect again", "source": "VisionExpert.act: retreat / return", "kind": "warning" }, "object_offset": { "title": "Remember object in measured grasp", "body": "At confirmed close, transform the perceived tube centre into the measured EE frame: c_EE = R_grasp\u1d40(c_world \u2212 p_grasp). This preserves the axial grasp offset when the wrist rotates. It assumes no subsequent slip.", "inputs": "Perceived tube centre + measured grasp pose", "outputs": "Carried object centre relative to EE", "source": "held_centre_in_ee \u00b7 RealExpert close", "kind": "revision" }, "slot": { "title": "Choose object-centre slot", "body": "Default task: two 80 \u00d7 20 mm tubes, configured destination half-extents 74.5 \u00d7 47 mm. Slots are \u00b115 mm across its short axis. Geometry checks require 12 mm edge margin and reject overlapping slots. Box centre/orientation are still configured, not observed online.", "inputs": "Confirmed release count + configured dimensions", "outputs": "Desired object centre in one of two slots", "source": "PlacementTask.slot", "kind": "revision" }, "compensate": { "title": "Compensate the wrist target", "body": "XY_EE = XY_slot \u2212 (R_place \u00b7 c_EE)_xy. The wrist goal now centres the carried tube at the slot even after a 180\u00b0 yaw change. Existing clearance/release heights are retained.", "inputs": "Object-in-EE vector; placement rotation; slot centre", "outputs": "Compensated destination EE XY", "source": "placement_ee_xy \u00b7 RealExpert.plan_placement", "kind": "revision" }, "lower_v2": { "title": "transport \u2192 lower", "body": "Transport at table + 140 mm, then descend at the compensated XY to table + 50 mm. Each transition requires < 2 mm translation error and < 0.06 rad orientation error.", "inputs": "Compensated XY + measured EE pose", "outputs": "Release pose; ~49.54 mm opening", "source": "RealExpert._act_legacy: transport / lower", "kind": "revision" }, "release_v2": { "title": "Release feedback clear?", "body": "Require valid feedback age < 0.6 s, a post-command sample, not busy, grip_detected = false and measured gap \u2265 requested width \u2212 3 mm. Keep these conditions true for 0.4 s. Otherwise stay at the release pose. Fault after 14 s.", "inputs": "Measured gripper gap; busy/contact/age/validity", "outputs": "Confirmed release OR wait / hardware_fault", "source": "RealExpert.wait_for_release", "kind": "decision" }, "count": { "title": "Count one confirmed release", "body": "Increment released_count only once, when release feedback and the settling interval pass. This counter is separate from attempts, nudges and source-return cycles. It does not visually certify final containment.", "inputs": "Confirmed release", "outputs": "released_count + 1; clearance retreat target", "source": "RealExpert.wait_for_release", "kind": "revision" }, "retreat_v2": { "title": "Retreat at compensated XY", "body": "Lift to table + 140 mm. Completion is checked only after the measured wrist has arrived at clearance (< 4 mm / 0.06 rad).", "inputs": "Release confirmed + measured pose", "outputs": "Safe-height completion check", "source": "RealExpert._act_legacy: retreat", "kind": "revision" }, "limit": { "title": "Requested transfer count reached?", "body": "Default expected_transfers = 2. If fewer releases are confirmed, return to the source and acquire fresh stereo views. Otherwise enter the absorbing completed state.", "inputs": "released_count; expected_transfers", "outputs": "Return for next transfer OR completed", "source": "RealExpert._act_legacy: retreat", "kind": "decision" }, "completed": { "title": "completed \u2192 normal stop", "body": "Hold at the measured clearance pose; never re-enter inspect. API outcome = completed. The generic lab runner stops normally and records completion. Two release confirmations are explicitly not a visual success claim.", "inputs": "Transfer limit reached after retreat", "outputs": "Hold, stop servo, keep camera; no third pick", "source": "RealExpert \u00b7 server \u00b7 RemoteExpertPolicy \u00b7 Runner.stop", "kind": "stop" }, "validate": { "title": "Validate command + feedback", "body": "Check SE(3), finite nonnegative width, positive finite speed caps, joint feedback and timing. Reject targets outside configured workspace or near-singular Jacobians.", "inputs": "PoseCommand + measured q + local clock", "outputs": "Validated request OR exception/fault", "source": "PoseCommand.validate \u00b7 TrajectoryController.step", "kind": "decision" }, "expired": { "title": "Expired / hold / zero scale?", "body": "If true, reset the reference to measured joints. If the previous reference already exceeds tracking bounds, reseed and hold for this tick. Missed ticks never accumulate catch-up motion.", "inputs": "TTL; hold flag; arm scale; reference tracking", "outputs": "Reseed/hold OR advance persistent reference", "source": "TrajectoryController.step", "kind": "decision" }, "reference": { "title": "Persistent reference + DLS IK", "body": "Compute FK/Jacobian at the reference, not freshly measured q every tick. Clip Cartesian error to capped speed \u00d7 dt, solve damped least squares, then apply joint speed/acceleration and braking bounds.", "inputs": "Previous q_ref; goal; robot kinematics; dt = 50 ms", "outputs": "Candidate q_ref_next", "source": "TrajectoryController.step", "kind": "process" }, "bounds": { "title": "Measured tracking and FK bounds", "body": "Check candidate FK against workspace/joint bounds, \u2264 8 mm position tracking, \u2264 2.5\u00b0 rotation tracking and \u2264 0.08 rad joint tracking. If needed, binary-search a smaller advancement. Normal caps: 80 mm/s, 0.24 rad/s, 0.5 rad/s joint speed, 1 rad/s\u00b2 nominal acceleration. Hard bounds/holds take precedence.", "inputs": "Candidate reference + measured joints/pose", "outputs": "Bounded joint reference OR hold/fault", "source": "Limits \u00b7 TrajectoryController.step", "kind": "decision" }, "interpolate": { "title": "Interpolate + servo watchdog", "body": "Interpolate the 20 Hz reference at 125 Hz. Stop on > 200 ms old local action/camera observation or joint tracking error > 0.08 rad. The existing UR servoJ parameters are unchanged.", "inputs": "Bounded q target + local observation timestamps", "outputs": "125 Hz robot command stream", "source": "TubeV3Runner._stream", "kind": "process" }, "width_driver": { "title": "Execute physical opening width", "body": "open_width_mm = 0 closes. The driver clips widths above 100.8 mm. Nonzero arm speed scale does not rescale the absolute width. Busy commands finish asynchronously; no new command starts while busy.", "inputs": "Absolute requested width", "outputs": "RG2 command + busy/completion feedback", "source": "TubeV3Rig.command_open_width \u00b7 _grip_width", "kind": "output" } }, "graphs": { "system": { "title": "From observation to motion", "subtitle": "A100 owns perception and task decisions. The lab PC owns I/O, the common controller and the drivers.", "nodes": [ "sensors", "labobs", "api", "calibration", "inspect_gate", "models", "planner", "fsm", "command", "ttl", "hold", "controller", "driver", "nn" ], "edges": [ [ "sensors", "labobs", "RGB + measured state" ], [ "labobs", "api", "async request \u00b7 up to 5 Hz" ], [ "calibration", "api", "session configuration" ], [ "api", "inspect_gate", "Observation" ], [ "inspect_gate", "models", "yes \u00b7 inspect / stereo_wait" ], [ "inspect_gate", "fsm", "no \u00b7 reuse estimate" ], [ "models", "planner", "perceived tube geometry" ], [ "planner", "fsm", "chosen grasp" ], [ "fsm", "command", "goal + width + caps" ], [ "command", "ttl", "API reply" ], [ "ttl", "controller", "fresh and valid" ], [ "ttl", "hold", "expired / terminal" ], [ "nn", "controller", "same command interface" ], [ "controller", "driver", "joint reference \u00b7 20 Hz" ], [ "driver", "sensors", "physical feedback loop" ] ], "direction": "TB" }, "perception": { "title": "Where SAM3 and FS are used", "subtitle": "Two wrist views produce one cached 3D estimate. No continuous learned visual-servo loop runs during descent.", "nodes": [ "left", "baseline", "rectify", "fs", "sam", "fuse", "fit", "cache", "candidates", "planner", "no_detection" ], "edges": [ [ "left", "baseline", "command 50 mm camera-X shift" ], [ "baseline", "rectify", "arrival < 1 mm, then wait > 0.4 s" ], [ "left", "sam", "left RGB only" ], [ "rectify", "fs", "rectified stereo pair" ], [ "rectify", "fuse", "left remap + camera transforms" ], [ "fs", "fuse", "dense XYZ" ], [ "sam", "fuse", "masks + scores" ], [ "fuse", "fit", "instance point sets" ], [ "fit", "cache", "accepted candidates" ], [ "cache", "candidates", "back at left pose" ], [ "candidates", "planner", "nonempty" ], [ "candidates", "no_detection", "empty and inspect age > 2 s" ] ], "direction": "TB" }, "grasp": { "title": "Grasp FSM and feedback guards", "subtitle": "State names match the implementation. False arrival guards remain in the same state and keep the current goal.", "nodes": [ "grasp_offsets", "retry_gate", "approach", "refine", "descend", "progress", "close", "lift", "transport", "nudge_approach", "recover", "return", "fault" ], "edges": [ [ "grasp_offsets", "retry_gate", "chosen tube + offset" ], [ "retry_gate", "approach", "no" ], [ "retry_gate", "nudge_approach", "yes \u2192 Recovery branch" ], [ "approach", "refine", "arrived < 4 mm / 0.06 rad" ], [ "refine", "descend", "age > 0.35 s" ], [ "descend", "close", "arrived < 2 mm / 0.06 rad" ], [ "descend", "progress", "not arrived" ], [ "progress", "descend", "progress + budget OK" ], [ "progress", "recover", "stall OR budget exhausted" ], [ "close", "lift", "fresh completion evidence" ], [ "close", "fault", "age > 14 s without evidence" ], [ "lift", "transport", "arrived and gap \u2208 [14, 32] mm" ], [ "lift", "return", "arrived and gap outside range" ], [ "recover", "return", "arrived at recovery height" ] ], "direction": "TB" }, "recovery": { "title": "Recovery and nudge branches", "subtitle": "Recovery refreshes perception after returning. Nudge contact remains partly time-based.", "nodes": [ "retry_gate", "nudge_approach", "nudge_down", "nudge_press", "nudge_push", "progress", "recover", "return", "left" ], "edges": [ [ "retry_gate", "nudge_approach", "near failed point AND nudges < 3" ], [ "nudge_approach", "nudge_down", "arrived < 4 mm / 0.06 rad" ], [ "nudge_down", "nudge_press", "arrived < 3 mm / 0.06 rad" ], [ "nudge_down", "progress", "not arrived" ], [ "nudge_press", "nudge_push", "age > 0.5 s" ], [ "nudge_push", "recover", "arrived < 5 mm; update history" ], [ "nudge_push", "progress", "not arrived" ], [ "progress", "recover", "stall OR budget exhausted" ], [ "recover", "return", "arrived" ], [ "return", "left", "arrived; clear cached candidates" ] ], "direction": "TB" }, "place-v1": { "title": "Placement FSM \u00b7 previous v1", "subtitle": "This explains the latest real run: off-centre grasp is not compensated, release is time-based, and the loop has no transfer limit.", "nodes": [ "transport", "lower_v1", "release_v1", "retreat_v1", "return", "left" ], "edges": [ [ "transport", "lower_v1", "arrived < 4 mm / 0.06 rad" ], [ "lower_v1", "release_v1", "arrived < 4 mm / 0.06 rad" ], [ "release_v1", "retreat_v1", "release age > 0.7 s" ], [ "retreat_v1", "return", "arrived < 4 mm / 0.06 rad" ], [ "return", "left", "arrived \u2192 inspect, no completion gate" ] ], "direction": "TB" }, "place-v2": { "title": "Placement FSM \u00b7 deployed revision", "subtitle": "The replacement handles object-centre placement, measured release and an explicit two-transfer endpoint.", "nodes": [ "object_offset", "slot", "compensate", "lower_v2", "release_v2", "count", "retreat_v2", "limit", "return", "completed", "fault" ], "edges": [ [ "object_offset", "compensate", "held offset" ], [ "slot", "compensate", "slot centre" ], [ "compensate", "lower_v2", "corrected EE goal + R_place" ], [ "lower_v2", "release_v2", "arrived; request opening" ], [ "release_v2", "count", "clear continuously for \u2265 0.4 s" ], [ "release_v2", "release_v2", "not clear \u00b7 hold pose" ], [ "release_v2", "fault", "unconfirmed after 14 s" ], [ "count", "retreat_v2", "rise to clearance" ], [ "retreat_v2", "limit", "arrived" ], [ "limit", "completed", "yes \u00b7 count \u2265 2" ], [ "limit", "return", "no \u00b7 acquire next tube" ] ], "direction": "TB" }, "control": { "title": "Shared local control loop", "subtitle": "The NN and the scripted expert share this small lab-side boundary. No tube FSM or perception model runs inside it.", "nodes": [ "command", "nn", "validate", "expired", "hold", "reference", "bounds", "interpolate", "width_driver", "sensors" ], "edges": [ [ "command", "validate", "absolute target + width + caps" ], [ "nn", "validate", "adapted NN command" ], [ "validate", "expired", "valid" ], [ "validate", "hold", "invalid \u2192 fault" ], [ "expired", "hold", "yes / tracking already exceeded" ], [ "expired", "reference", "no" ], [ "reference", "bounds", "candidate reference" ], [ "bounds", "reference", "shrink step if needed" ], [ "bounds", "interpolate", "safe bounded reference" ], [ "bounds", "hold", "cannot advance safely" ], [ "interpolate", "sensors", "servoJ \u2192 measured motion" ], [ "bounds", "width_driver", "accepted tick; scale > 0; not busy" ], [ "width_driver", "sensors", "measured width / busy / contact" ], [ "sensors", "bounds", "measured tracking feedback" ] ], "direction": "TB" }, "fsm-v1": { "title": "FSM map \u00b7 previous real policy", "subtitle": "Follow acquisition \u2192 grasp \u2192 place. Red paths recover and re-perceive. The v1 placement loop has no completion condition.", "nodes": [ "settle", "overview", "left", "baseline", "models", "cache", "candidates", "no_detection", "grasp_offsets", "retry_gate", "approach", "refine", "descend", "progress", "close", "lift", "transport", "lower_v1", "release_v1", "retreat_v1", "return", "nudge_approach", "nudge_down", "nudge_press", "nudge_push", "recover", "fault" ], "edges": [ [ "settle", "overview", "age \u2265 3 s" ], [ "overview", "left", "arrived" ], [ "left", "baseline", "no cached stereo estimate" ], [ "baseline", "models", "arrival + 0.4 s wait" ], [ "models", "cache", "candidates" ], [ "cache", "candidates", "return to left view; inspect" ], [ "candidates", "grasp_offsets", "nonempty" ], [ "candidates", "no_detection", "empty; inspect age > 2 s" ], [ "grasp_offsets", "retry_gate", "selected grasp" ], [ "retry_gate", "approach", "no" ], [ "retry_gate", "nudge_approach", "near failed point; nudges < 3" ], [ "approach", "refine", "arrived" ], [ "refine", "descend", "age > 0.35 s" ], [ "descend", "close", "arrived < 2 mm" ], [ "descend", "progress", "not arrived" ], [ "progress", "recover", "stall / budget exhausted" ], [ "close", "lift", "fresh close evidence" ], [ "close", "fault", "timeout > 14 s" ], [ "lift", "transport", "arrived; gap 14\u201332 mm" ], [ "lift", "return", "arrived; implausible gap" ], [ "transport", "lower_v1", "arrived" ], [ "lower_v1", "release_v1", "arrived" ], [ "release_v1", "retreat_v1", "age > 0.7 s" ], [ "retreat_v1", "return", "arrived" ], [ "return", "left", "arrived; clear cached estimate" ], [ "nudge_approach", "nudge_down", "arrived" ], [ "nudge_down", "nudge_press", "arrived < 3 mm" ], [ "nudge_down", "progress", "not arrived" ], [ "nudge_press", "nudge_push", "age > 0.5 s" ], [ "nudge_push", "recover", "arrived < 5 mm" ], [ "nudge_push", "progress", "not arrived" ], [ "recover", "return", "arrived" ] ], "direction": "TB" }, "fsm-v2": { "title": "FSM map \u00b7 deployed revision", "subtitle": "The new placement subgraph replaces the timer-only release loop and adds an absorbing completion state.", "nodes": [ "settle", "overview", "left", "baseline", "models", "cache", "candidates", "no_detection", "grasp_offsets", "retry_gate", "approach", "refine", "descend", "progress", "close", "lift", "transport", "return", "nudge_approach", "nudge_down", "nudge_press", "nudge_push", "recover", "fault", "object_offset", "slot", "compensate", "lower_v2", "release_v2", "count", "retreat_v2", "limit", "completed" ], "edges": [ [ "settle", "overview", "age \u2265 3 s" ], [ "overview", "left", "arrived" ], [ "left", "baseline", "no cached stereo estimate" ], [ "baseline", "models", "arrival + 0.4 s wait" ], [ "models", "cache", "candidates" ], [ "cache", "candidates", "return to left view; inspect" ], [ "candidates", "grasp_offsets", "nonempty" ], [ "candidates", "no_detection", "empty; inspect age > 2 s" ], [ "grasp_offsets", "retry_gate", "selected grasp" ], [ "retry_gate", "approach", "no" ], [ "retry_gate", "nudge_approach", "near failed point; nudges < 3" ], [ "approach", "refine", "arrived" ], [ "refine", "descend", "age > 0.35 s" ], [ "descend", "close", "arrived < 2 mm" ], [ "descend", "progress", "not arrived" ], [ "progress", "recover", "stall / budget exhausted" ], [ "close", "lift", "fresh close evidence" ], [ "close", "fault", "timeout > 14 s" ], [ "lift", "transport", "arrived; gap 14\u201332 mm" ], [ "lift", "return", "arrived; implausible gap" ], [ "return", "left", "arrived; clear cached estimate" ], [ "nudge_approach", "nudge_down", "arrived" ], [ "nudge_down", "nudge_press", "arrived < 3 mm" ], [ "nudge_down", "progress", "not arrived" ], [ "nudge_press", "nudge_push", "age > 0.5 s" ], [ "nudge_push", "recover", "arrived < 5 mm" ], [ "nudge_push", "progress", "not arrived" ], [ "recover", "return", "arrived" ], [ "close", "object_offset", "confirmed grasp pose" ], [ "transport", "compensate", "placement rotation" ], [ "object_offset", "compensate", "held object offset" ], [ "slot", "compensate", "release-count slot" ], [ "compensate", "lower_v2", "corrected target" ], [ "lower_v2", "release_v2", "arrived" ], [ "release_v2", "count", "fresh release + 0.4 s clear" ], [ "release_v2", "fault", "timeout > 14 s" ], [ "count", "retreat_v2", "clearance retreat" ], [ "retreat_v2", "limit", "arrived" ], [ "limit", "completed", "count \u2265 2" ], [ "limit", "return", "count < 2" ] ], "direction": "TB" } } }