commit 2e4c6964b21c4d7354ef718d99d640f94badea61 Author: Reza Adousi <91836545+RezaAdousi@users.noreply.github.com> Date: Sat Oct 31 23:04:47 2026 +0330 Initial commit diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..c9ebf2d --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:system" +} \ No newline at end of file diff --git a/__pycache__/calibration.cpython-312.pyc b/__pycache__/calibration.cpython-312.pyc new file mode 100644 index 0000000..39ed637 Binary files /dev/null and b/__pycache__/calibration.cpython-312.pyc differ diff --git a/__pycache__/camera.cpython-312.pyc b/__pycache__/camera.cpython-312.pyc new file mode 100644 index 0000000..41bfe8b Binary files /dev/null and b/__pycache__/camera.cpython-312.pyc differ diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..419a9c4 Binary files /dev/null and b/__pycache__/config.cpython-312.pyc differ diff --git a/__pycache__/detection.cpython-312.pyc b/__pycache__/detection.cpython-312.pyc new file mode 100644 index 0000000..49e2be0 Binary files /dev/null and b/__pycache__/detection.cpython-312.pyc differ diff --git a/__pycache__/effects.cpython-312.pyc b/__pycache__/effects.cpython-312.pyc new file mode 100644 index 0000000..c632c64 Binary files /dev/null and b/__pycache__/effects.cpython-312.pyc differ diff --git a/__pycache__/geometry.cpython-312.pyc b/__pycache__/geometry.cpython-312.pyc new file mode 100644 index 0000000..a0f0b17 Binary files /dev/null and b/__pycache__/geometry.cpython-312.pyc differ diff --git a/__pycache__/imaging.cpython-312.pyc b/__pycache__/imaging.cpython-312.pyc new file mode 100644 index 0000000..5cb74b5 Binary files /dev/null and b/__pycache__/imaging.cpython-312.pyc differ diff --git a/__pycache__/launcher.cpython-312.pyc b/__pycache__/launcher.cpython-312.pyc new file mode 100644 index 0000000..be0f957 Binary files /dev/null and b/__pycache__/launcher.cpython-312.pyc differ diff --git a/__pycache__/main.cpython-312.pyc b/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000..6ef4e3a Binary files /dev/null and b/__pycache__/main.cpython-312.pyc differ diff --git a/__pycache__/physics.cpython-312.pyc b/__pycache__/physics.cpython-312.pyc new file mode 100644 index 0000000..9c1a5fd Binary files /dev/null and b/__pycache__/physics.cpython-312.pyc differ diff --git a/__pycache__/rendering.cpython-312.pyc b/__pycache__/rendering.cpython-312.pyc new file mode 100644 index 0000000..e0e465d Binary files /dev/null and b/__pycache__/rendering.cpython-312.pyc differ diff --git a/__pycache__/vision.cpython-312.pyc b/__pycache__/vision.cpython-312.pyc new file mode 100644 index 0000000..b600bdb Binary files /dev/null and b/__pycache__/vision.cpython-312.pyc differ diff --git a/calibration.py b/calibration.py new file mode 100644 index 0000000..52f77f0 --- /dev/null +++ b/calibration.py @@ -0,0 +1,1580 @@ +# calibration.py + +import math + +import cv2 +import numpy as np + +import imaging + +from config import ( + SCREEN_WIDTH, + SCREEN_HEIGHT, + CALIBRATION_TARGET_HUE_DEGREES, + CALIBRATION_TARGET_HUE_TOLERANCE_DEGREES, + CALIBRATION_MIN_TARGET_SATURATION, + CALIBRATION_MIN_TARGET_VALUE, + CALIBRATION_ADAPTIVE_HUE_ENABLED, + CALIBRATION_ADAPTIVE_HUE_BLEND, + CALIBRATION_ADAPTIVE_MAX_DRIFT_DEGREES, + CALIBRATION_WHITE_ADAPTIVE_OFFSET, + CALIBRATION_WHITE_MIN_ABSOLUTE_LIGHTNESS, + CALIBRATION_WHITE_MAX_GREEN_DOMINANCE, + CALIBRATION_WHITE_REFERENCE_BLEND, + CALIBRATION_WHITE_REFERENCE_MAX_OFFSET, + CALIBRATION_MIN_SCREEN_AREA_RATIO, + CALIBRATION_MAX_SCREEN_AREA_RATIO, + CALIBRATION_MIN_OUTER_TARGET_RATIO, + CALIBRATION_MIN_INNER_WHITE_RATIO, + CALIBRATION_MIN_FULL_TARGET_RATIO, + CALIBRATION_MIN_QUALITY, + CALIBRATION_REQUIRED_STABLE_FRAMES, + CALIBRATION_MAX_CORNER_JUMP_RATIO, + CALIBRATION_SMOOTHING_ALPHA, + CALIBRATION_MASK_CLOSE_KERNEL, + CALIBRATION_OUTER_BORDER_SIZE, + CALIBRATION_INNER_BORDER_SIZE, +) + + +class Calibration: + """ + Detects the monitor in two phases, matching what Renderer draws + in each: + + 1. NOT YET CALIBRATED: the whole screen is filled solid pink + (Renderer.draw_full_calibration_screen). We just look for + the single largest, best-shaped, (almost) entirely-pink + quad in the camera frame - the simplest, most unambiguous + target possible for the very first lock-on. + + 2. CALIBRATED (locked on): the screen instead shows a + saturated green outer ring with a white inner ring around + a black interior (Renderer.draw_calibration_border), drawn + every frame during gameplay so the system can keep quietly + re-verifying the monitor's position without interrupting + play, while leaving the black interior free for the ball + and the detected obstacles. + + Compared to the previous four-marker approach this is far more + robust in a busy, noisy exhibition environment: + - it's one large, high-contrast shape instead of four tiny + independent ones, so a person briefly walking in front of + part of the screen doesn't break detection the way losing + one out of four small markers would; + - requiring BOTH the green AND white ring to be present, in + the right relative position, makes it very unlikely a + random green/white object or a reflection in the room gets + mistaken for the screen; + - once locked, calibration is only re-verified/refined by + new stable detections and is never silently dropped just + because a frame or two failed - it only clears on an + explicit reset() call (bound to the "C" key in main.py). + + Color detection is NOT a single fixed HSV threshold (see + config.py's "Target-color detection" section for the full + reasoning). In short: a loose, wide hue pre-filter rules out + everything nowhere near green, and a per-frame Otsu threshold + on green-dominance (G compared to R and B) then finds the exact + cutoff that best separates "clearly green" from "not" IN THE + CURRENT frame - so it keeps adapting to the room's lighting + instead of needing a fixed brightness/saturation number that + only ever matched the lighting at setup time. On top of that, + once locked on, the system slowly learns the actual hue it is + seeing and re-centers the pre-filter on it, bounded so it can + never drift onto some unrelated color. + + Public API is intentionally identical to the previous + marker-based Calibration class, so main.py needs no changes + beyond how the pattern itself is drawn. + """ + + def __init__(self): + self.matrix = None + + self.history = [] + self.hue_history = [] + self.previous_points = None + self.reference_points = None + + # What hue (0-360 degrees) the camera is ACTUALLY showing + # for the calibration green right now, learned on the fly. + # None until the first successful lock, and never reset by + # reset() below - a "C" key reset is about re-finding the + # screen's position, not about the room's lighting having + # changed, so there is no reason to throw this away with it. + self._learned_hue_center = None + + # What the camera ACTUALLY records for the white inner ring + # right now, as a (a*, b*) offset from LAB neutral (128, + # 128) - i.e. exactly what a photographer gets by custom + # white-balancing off a gray card, except sampled + # continuously off the ring that is drawn every single + # frame anyway. detection.py uses this to judge how "white" + # a candidate sticker really is relative to what white + # currently looks like to this camera, instead of relative + # to an absolute assumption that colored ambient light can + # never actually hold true. Also never cleared by reset(). + self._white_reference_ab = None + + self.failed_frame_count = 0 + + # ======================================================== + # Public API + # ======================================================== + + def is_calibrated(self): + return ( + self.matrix is not None + and np.isfinite(self.matrix).all() + ) + + def reset(self): + self.matrix = None + + self.history = [] + self.hue_history = [] + self.previous_points = None + self.reference_points = None + + self.failed_frame_count = 0 + + def update(self, frame): + if frame is None: + return self.is_calibrated() + + if frame.ndim != 3: + return self.is_calibrated() + + # Before we're locked on, the screen is filled solid pink + # (Renderer.draw_full_calibration_screen) - look for one + # big pink quad. Once locked on, the screen instead shows + # the pink/white border ring around a black interior + # (Renderer.draw_calibration_border) - look for that + # specific pattern instead, which also lets us tell a real + # match from a random pink object/reflection in the room. + if self.is_calibrated(): + candidate = self._find_best_border_candidate(frame) + else: + candidate = self._find_best_full_screen_candidate( + frame + ) + + if candidate is None: + self._handle_failed_detection() + return self.is_calibrated() + + self.failed_frame_count = 0 + + self._update_white_reference(candidate.get("white_ab")) + + points = candidate["points"] + hue = candidate.get("hue_degrees") + + if self._stable_against_previous( + points, + frame.shape, + ): + self.history.append(points) + self.hue_history.append(hue) + + if ( + len(self.history) + > CALIBRATION_REQUIRED_STABLE_FRAMES + ): + self.history.pop(0) + self.hue_history.pop(0) + else: + self.history = [points] + self.hue_history = [hue] + + self.previous_points = points.copy() + + if ( + len(self.history) + >= CALIBRATION_REQUIRED_STABLE_FRAMES + ): + self._accept_or_refine(frame.shape) + + return self.is_calibrated() + + def transform_frame(self, frame, scale=1.0): + """ + Warp the camera frame into screen space. With scale < 1 the + result is a proportionally smaller (SCREEN_WIDTH*scale by + SCREEN_HEIGHT*scale) image of the same screen - the warp is + composed with the scaling, so it's a single cheap pass and + much faster than warping full size and shrinking after. + """ + if not self.is_calibrated(): + raise RuntimeError( + "Calibration has not been completed." + ) + + matrix = self.matrix + + if scale != 1.0: + matrix = ( + np.diag([scale, scale, 1.0]) @ matrix + ) + + return cv2.warpPerspective( + frame, + matrix, + ( + int(round(SCREEN_WIDTH * scale)), + int(round(SCREEN_HEIGHT * scale)), + ), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0, 0, 0), + ) + + def transform_point(self, x, y): + if not self.is_calibrated(): + raise RuntimeError( + "Calibration has not been completed." + ) + + point = np.float32( + [[[x, y]]] + ) + + transformed = cv2.perspectiveTransform( + point, + self.matrix, + ) + + return tuple(transformed[0][0]) + + # ======================================================== + # Candidate detection + # ======================================================== + + def _find_best_border_candidate(self, frame): + hsv = cv2.cvtColor( + frame, + cv2.COLOR_BGR2HSV, + ) + + lightness, a_channel, b_channel = imaging.to_lab(frame) + + target_mask = self._make_target_mask(frame, hsv) + white_mask = self._make_white_mask(frame) + + contours, _ = cv2.findContours( + target_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_SIMPLE, + ) + + outer_depth = float(CALIBRATION_OUTER_BORDER_SIZE) + + inner_depth = float( + CALIBRATION_OUTER_BORDER_SIZE + + CALIBRATION_INNER_BORDER_SIZE + ) + + best = None + + for contour in contours: + quad = self._quad_from_contour( + contour, + frame.shape, + ) + + if quad is None: + continue + + quality = self._quad_quality( + quad, + frame.shape, + ) + + if quality <= 0: + continue + + target_ratio = self._band_ratio( + target_mask, + quad, + 0.0, + outer_depth, + ) + + if ( + target_ratio + < CALIBRATION_MIN_OUTER_TARGET_RATIO + ): + continue + + white_ratio = self._band_ratio( + white_mask, + quad, + outer_depth, + inner_depth, + ) + + if ( + white_ratio + < CALIBRATION_MIN_INNER_WHITE_RATIO + ): + continue + + score = ( + quality * 0.45 + + target_ratio * 0.30 + + white_ratio * 0.25 + ) + + if score < CALIBRATION_MIN_QUALITY: + continue + + if ( + best is None + or score > best["score"] + ): + best = { + "points": quad, + "score": float(score), + "quality": float(quality), + "target_ratio": float(target_ratio), + "white_ratio": float(white_ratio), + "hue_degrees": self._band_mean_hue( + hsv, + target_mask, + quad, + 0.0, + outer_depth, + ), + "white_ab": self._band_mean_ab( + a_channel, + b_channel, + white_mask, + quad, + outer_depth, + inner_depth, + ), + } + + return best + + def _find_best_full_screen_candidate(self, frame): + """ + Used only before we're locked on, while the monitor is + showing one solid green fill (no white ring, no black + interior). We just need the largest, best-shaped green quad + whose own interior is (almost) entirely green - there's no + separate inner-ring check needed since the "ring" is the + whole screen. + """ + hsv = cv2.cvtColor( + frame, + cv2.COLOR_BGR2HSV, + ) + + target_mask = self._make_target_mask(frame, hsv) + + contours, _ = cv2.findContours( + target_mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_SIMPLE, + ) + + best = None + + for contour in contours: + quad = self._quad_from_contour( + contour, + frame.shape, + ) + + if quad is None: + continue + + quality = self._quad_quality( + quad, + frame.shape, + ) + + if quality <= 0: + continue + + fill_ratio = self._full_quad_ratio( + target_mask, + quad, + ) + + if ( + fill_ratio + < CALIBRATION_MIN_FULL_TARGET_RATIO + ): + continue + + score = ( + quality * 0.5 + + fill_ratio * 0.5 + ) + + if score < CALIBRATION_MIN_QUALITY: + continue + + if ( + best is None + or score > best["score"] + ): + best = { + "points": quad, + "score": float(score), + "quality": float(quality), + "target_ratio": float(fill_ratio), + "white_ratio": 1.0, + "hue_degrees": self._quad_mean_hue( + hsv, + target_mask, + quad, + ), + } + + return best + + def _quad_from_contour( + self, + contour, + frame_shape, + ): + frame_area = ( + frame_shape[0] + * frame_shape[1] + ) + + area = cv2.contourArea(contour) + + if area <= 0: + return None + + area_ratio = area / frame_area + + if area_ratio < CALIBRATION_MIN_SCREEN_AREA_RATIO: + return None + + if area_ratio > CALIBRATION_MAX_SCREEN_AREA_RATIO: + return None + + perimeter = cv2.arcLength( + contour, + True, + ) + + if perimeter <= 0: + return None + + for epsilon_ratio in ( + 0.005, + 0.008, + 0.012, + 0.018, + 0.025, + 0.035, + ): + approx = cv2.approxPolyDP( + contour, + epsilon_ratio * perimeter, + True, + ) + + if len(approx) != 4: + continue + + points = approx.reshape( + 4, 2 + ).astype(np.float32) + + if not cv2.isContourConvex( + points.reshape((-1, 1, 2)) + ): + continue + + return self._order_points(points) + + return None + + # ======================================================== + # Color masks + # ======================================================== + + def _make_target_mask(self, frame_bgr, hsv): + """ + Finds the calibration green robustly under unpredictable, + possibly-shifting exhibition lighting, by combining a loose + hue pre-filter with a per-frame auto-threshold on a + brightness-INVARIANT color signal, rather than one fixed + HSV box (see config.py's "Target-color detection" section + and the class docstring for the full reasoning). + """ + pre_filter = self._make_hue_prefilter_mask(hsv) + + if cv2.countNonZero(pre_filter) < 64: + # Nothing even loosely green-ish in view - there is no + # meaningful split for Otsu to find, so don't ask it to + # invent one out of noise. + return self._clean_mask(pre_filter) + + # The "how green is this pixel" signal is measured on LAB's + # a* axis (green-red), NOT by comparing raw R/G/B levels. + # LAB deliberately separates brightness (L) from true color + # (a, b), so a* barely moves when a glare highlight, a + # shadow, or the camera's own auto-exposure/white-balance + # changes how BRIGHT a pixel looks - which is most of what + # "the camera doesn't quite see what the monitor shows" + # actually is. Comparing raw channel levels (as a plain + # "G minus R/B" measure would) conflates color with + # brightness and is far more easily fooled by exactly that. + _lightness, a_channel, _b_channel = imaging.to_lab( + frame_bgr + ) + + dominance = imaging.green_dominance(a_channel) + + # Otsu needs to see BOTH classes to find a meaningful split + # point - "pattern green" and "everything else" - so it is + # run over the WHOLE frame's dominance histogram (that is + # genuinely bimodal: a bright green cluster from the + # pattern vs. a large, much less green-dominant cluster for + # the room/people/background). Restricting it to only the + # pixels the loose pre-filter already approved would feed + # Otsu an almost-uniform population with no real second + # class to split against, which produces a meaningless + # threshold instead of an adaptive one. + threshold, _ = cv2.threshold( + dominance, + 0, + 255, + cv2.THRESH_BINARY + cv2.THRESH_OTSU, + ) + + strong = cv2.inRange( + dominance, + int(threshold), + 255, + ) + + # The hue pre-filter is then applied on top as a precision + # pass: it rules out anything that passed the brightness- + # based Otsu split but isn't actually the right hue (e.g. a + # bright white highlight, which has near-zero dominance + # anyway, or a saturated cyan/yellow object, which does + # not). + mask = cv2.bitwise_and(pre_filter, strong) + + mask = self._clean_mask(mask) + + # A specular glare spot on the monitor's glass can punch a + # small hole clean through the middle of the pattern, or + # even a thin streak that splits it into two separate + # blobs. Filling any fully-enclosed hole (see + # imaging.fill_holes) repairs exactly that without touching + # a real gap caused by, say, a person actually standing in + # front of part of the screen (which stays open to the + # mask's edge and so is left alone). + return imaging.fill_holes(mask) + + def _make_hue_prefilter_mask(self, hsv): + center = self._effective_hue_center() + + mask = np.zeros( + hsv.shape[:2], + dtype=np.uint8, + ) + + for lower_h, upper_h in self._hue_ranges( + center, + CALIBRATION_TARGET_HUE_TOLERANCE_DEGREES, + ): + current = cv2.inRange( + hsv, + np.array( + [ + lower_h, + CALIBRATION_MIN_TARGET_SATURATION, + CALIBRATION_MIN_TARGET_VALUE, + ], + dtype=np.uint8, + ), + np.array( + [upper_h, 255, 255], + dtype=np.uint8, + ), + ) + + mask = cv2.bitwise_or(mask, current) + + return mask + + @staticmethod + def _hue_ranges(center_degrees, tolerance_degrees): + """ + A (center +/- tolerance) window in standard 0-360 degree + hue, converted to OpenCV's 0-179 H channel and split into + one or two (lower, upper) ranges so a window that wraps + past 0/360 (relevant for colors near red, not for the green + this is tuned for, but kept general) still works correctly. + """ + raw_low = center_degrees - tolerance_degrees + raw_high = center_degrees + tolerance_degrees + + if raw_low < 0.0: + spans = [ + (0.0, raw_high), + (raw_low + 360.0, 360.0), + ] + elif raw_high > 360.0: + spans = [ + (raw_low, 360.0), + (0.0, raw_high - 360.0), + ] + else: + spans = [(raw_low, raw_high)] + + return [ + ( + int(round(lo / 2.0)), + int(round(hi / 2.0)), + ) + for lo, hi in spans + ] + + def _make_white_mask(self, frame_bgr): + """ + Finds the white inner ring the same way detection.py finds + white stickers (see its module docstring): LOCALLY adaptive + brightness rather than one fixed V floor for the whole frame + (so a ring sitting in a dimmer part of the hall is still + found), and LAB chroma rather than HSV saturation for "is + this actually neutral" (stable across the whole brightness + range, unlike HSV's saturation ratio). A fixed absolute HSV + box here was exactly as fragile under real exhibition + lighting as the old fixed box was for stickers. + """ + lightness, a_channel, _b_channel = imaging.to_lab( + frame_bgr + ) + + block = self._white_ring_adaptive_block_size( + lightness.shape + ) + + local = cv2.adaptiveThreshold( + lightness, + 255, + cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY, + block, + -CALIBRATION_WHITE_ADAPTIVE_OFFSET, + ) + + mask = local & cv2.inRange( + lightness, + CALIBRATION_WHITE_MIN_ABSOLUTE_LIGHTNESS, + 255, + ) + + # The only other bright thing near the white ring is the + # outer green ring itself, bleeding in right at the band + # boundary - so it is enough to exclude anything clearly + # GREEN-dominant (one-sided), rather than pinning an exact + # "how white" chroma number. A strong ambient color cast can + # push even genuinely white paper's overall chroma quite + # far (see config.py's comment), but it does not make white + # look GREEN - a one-sided test stays reliable under warm, + # cool, or any other cast a two-sided chroma ceiling cannot + # universally cover. + mask &= cv2.inRange( + imaging.green_dominance(a_channel), + 0, + CALIBRATION_WHITE_MAX_GREEN_DOMINANCE, + ) + + kernel = cv2.getStructuringElement( + cv2.MORPH_RECT, + (9, 9), + ) + + return cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + kernel, + ) + + @staticmethod + def _white_ring_adaptive_block_size(shape): + # Tied to the frame's own size (calibration has no learned + # "ring thickness" the way detection.py learns sticker + # size), large enough that the sampled neighborhood is + # mostly the ring's dark/green surroundings rather than the + # ring's own pixels. + size = int(round(min(shape[:2]) * 0.05)) + + size = max(15, size) + + if size % 2 == 0: + size += 1 + + return size + + def _clean_mask(self, mask): + open_kernel = cv2.getStructuringElement( + cv2.MORPH_RECT, + (5, 5), + ) + + close_kernel = cv2.getStructuringElement( + cv2.MORPH_RECT, + ( + CALIBRATION_MASK_CLOSE_KERNEL, + CALIBRATION_MASK_CLOSE_KERNEL, + ), + ) + + mask = cv2.morphologyEx( + mask, + cv2.MORPH_OPEN, + open_kernel, + ) + + mask = cv2.morphologyEx( + mask, + cv2.MORPH_CLOSE, + close_kernel, + ) + + return mask + + # ======================================================== + # Full-quad sampling (used during the solid-pink full-screen + # calibration phase, before we're locked on) + # ======================================================== + + def _full_quad_ratio(self, mask, points): + polygon = points.astype(np.int32) + + fill = np.zeros_like(mask) + + cv2.fillPoly( + fill, + [polygon], + 255, + ) + + total = cv2.countNonZero(fill) + + if total == 0: + return 0.0 + + inside = cv2.bitwise_and( + mask, + fill, + ) + + return cv2.countNonZero(inside) / total + + # ======================================================== + # Band sampling (used to confirm the pink outer / white inner + # ring at the right relative depth, instead of trusting the + # outer pink contour alone) + # ======================================================== + + # points is always [top_left, top_right, bottom_right, + # bottom_left] (see _order_points), so edge 0 (top) and edge 2 + # (bottom) run the screen's WIDTH and edge 1 (right) and edge 3 + # (left) run its HEIGHT. An absolute screen-space pixel depth + # (e.g. CALIBRATION_OUTER_BORDER_SIZE) converts to that edge's + # own camera-space pixels via (edge's camera length / that + # edge's TRUE screen-space length) - using the same reference + # (SCREEN_WIDTH) for every edge, as if top/bottom and left/right + # were interchangeable, silently undersizes the left/right band + # whenever the screen isn't exactly square, which in practice is + # always: a 1920x1080 screen's left/right edges are only + # 1080/1920 as long as its top/bottom edges, so a band meant to + # be the same physical width on every side would come out + # visibly too shallow on the sides, easily shallow enough to + # land entirely short of where the white ring actually starts. + _EDGE_REFERENCE_LENGTH = ( + SCREEN_WIDTH, + SCREEN_HEIGHT, + SCREEN_WIDTH, + SCREEN_HEIGHT, + ) + + def _band_ratio( + self, + mask, + points, + depth_start_px, + depth_end_px, + ): + center = np.mean(points, axis=0) + + values = [] + + for i in range(4): + p1 = points[i] + p2 = points[(i + 1) % 4] + + edge = p2 - p1 + length = np.linalg.norm(edge) + + if length < 1: + continue + + reference = self._EDGE_REFERENCE_LENGTH[i] + + depth_start = ( + depth_start_px * length / reference + ) + depth_end = ( + depth_end_px * length / reference + ) + + if depth_end <= depth_start: + continue + + normal = np.array( + [-edge[1], edge[0]], + dtype=np.float32, + ) + + normal_length = np.linalg.norm(normal) + + if normal_length < 1e-6: + continue + + normal /= normal_length + + midpoint = (p1 + p2) * 0.5 + + if ( + np.dot(normal, center - midpoint) + < 0 + ): + normal = -normal + + a = p1 + normal * depth_start + b = p2 + normal * depth_start + c = p2 + normal * depth_end + d = p1 + normal * depth_end + + polygon = np.array( + [a, b, c, d], + dtype=np.int32, + ) + + band = np.zeros_like(mask) + + cv2.fillPoly( + band, + [polygon], + 255, + ) + + total = cv2.countNonZero(band) + + if total == 0: + continue + + inside = cv2.bitwise_and( + mask, + band, + ) + + values.append( + cv2.countNonZero(inside) / total + ) + + if not values: + return 0.0 + + return float(np.mean(values)) + + def _band_polygon_mask( + self, + points, + depth_start_px, + depth_end_px, + shape, + ): + """ + Same ring-band geometry as _band_ratio above (including the + same per-edge reference-length correction), but returned as + a single combined mask (union of the 4 edge trapezoids) + instead of an averaged ratio - used for pooling pixels + together (e.g. to measure their mean hue) rather than just + counting them. + """ + center = np.mean(points, axis=0) + + mask = np.zeros(shape, dtype=np.uint8) + + drew_any = False + + for i in range(4): + p1 = points[i] + p2 = points[(i + 1) % 4] + + edge = p2 - p1 + length = np.linalg.norm(edge) + + if length < 1: + continue + + reference = self._EDGE_REFERENCE_LENGTH[i] + + depth_start = ( + depth_start_px * length / reference + ) + depth_end = ( + depth_end_px * length / reference + ) + + if depth_end <= depth_start: + continue + + normal = np.array( + [-edge[1], edge[0]], + dtype=np.float32, + ) + + normal_length = np.linalg.norm(normal) + + if normal_length < 1e-6: + continue + + normal /= normal_length + + midpoint = (p1 + p2) * 0.5 + + if ( + np.dot(normal, center - midpoint) + < 0 + ): + normal = -normal + + a = p1 + normal * depth_start + b = p2 + normal * depth_start + c = p2 + normal * depth_end + d = p1 + normal * depth_end + + polygon = np.array( + [a, b, c, d], + dtype=np.int32, + ) + + cv2.fillPoly( + mask, + [polygon], + 255, + ) + + drew_any = True + + return mask if drew_any else None + + def _band_mean_hue( + self, + hsv, + color_mask, + points, + depth_start_px, + depth_end_px, + ): + """ + Mean hue (0-360 degrees) of the pixels that are BOTH inside + the given ring band AND already matched by color_mask - the + measurement used to learn what hue the camera is actually + recording for the calibration green right now (see + Calibration._update_learned_hue). + """ + band = self._band_polygon_mask( + points, + depth_start_px, + depth_end_px, + hsv.shape[:2], + ) + + if band is None: + return None + + return self._masked_mean_hue( + hsv, + cv2.bitwise_and(color_mask, band), + ) + + def _quad_mean_hue(self, hsv, color_mask, points): + polygon = points.astype(np.int32) + + fill = np.zeros(hsv.shape[:2], dtype=np.uint8) + + cv2.fillPoly(fill, [polygon], 255) + + return self._masked_mean_hue( + hsv, + cv2.bitwise_and(color_mask, fill), + ) + + def _band_mean_ab( + self, + a_channel, + b_channel, + color_mask, + points, + depth_start_px, + depth_end_px, + ): + """ + Mean LAB (a*, b*) of the pixels that are BOTH inside the + given ring band AND already matched by color_mask - used to + learn what the camera currently records for a KNOWN-white + surface (see white_reference / _update_white_reference). + """ + band = self._band_polygon_mask( + points, + depth_start_px, + depth_end_px, + a_channel.shape[:2], + ) + + if band is None: + return None + + combined = cv2.bitwise_and(color_mask, band) + + if cv2.countNonZero(combined) < 32: + return None + + mean_a = cv2.mean(a_channel, mask=combined)[0] + mean_b = cv2.mean(b_channel, mask=combined)[0] + + return float(mean_a), float(mean_b) + + @staticmethod + def _masked_mean_hue(hsv, mask): + # A minimum pixel count keeps a tiny sliver of matched + # pixels (e.g. a stray reflection) from producing a + # confident-looking but meaningless hue measurement. + if cv2.countNonZero(mask) < 32: + return None + + mean_hue_cv_units = cv2.mean( + hsv[:, :, 0], + mask=mask, + )[0] + + return float(mean_hue_cv_units) * 2.0 + + # ======================================================== + # Live white reference ("gray card" for the sticker detector) + # ======================================================== + + def white_reference(self): + """ + (a*, b*) in LAB's 0-255 encoding (128 = neutral) that the + camera is CURRENTLY recording for the calibration ring's + white band, or None before enough samples have come in. + detection.py measures how "white" a candidate sticker is + relative to this, instead of relative to absolute LAB + neutral - exactly like a photographer custom white-balancing + off a gray card, except this one is sampled continuously off + the ring that is drawn on the monitor every single frame + anyway, so it tracks the room's actual current lighting + automatically rather than needing to be re-tuned on-site. + """ + return self._white_reference_ab + + def _update_white_reference(self, measured_ab): + if measured_ab is None: + return + + measured_a, measured_b = measured_ab + + if self._white_reference_ab is None: + self._white_reference_ab = (measured_a, measured_b) + return + + current_a, current_b = self._white_reference_ab + + blended_a = ( + current_a + + CALIBRATION_WHITE_REFERENCE_BLEND + * (measured_a - current_a) + ) + blended_b = ( + current_b + + CALIBRATION_WHITE_REFERENCE_BLEND + * (measured_b - current_b) + ) + + # A real color cast from room lighting is a gentle, physical + # effect - it should never push "white" wildly far from + # actual neutral. Clamping the offset's magnitude keeps a + # run of bad samples (someone's bright shirt briefly + # overlapping the band, say) from teaching the sticker + # detector a wildly wrong idea of white. + offset_a = blended_a - 128.0 + offset_b = blended_b - 128.0 + + magnitude = math.hypot(offset_a, offset_b) + + limit = CALIBRATION_WHITE_REFERENCE_MAX_OFFSET + + if magnitude > limit: + scale = limit / magnitude + + offset_a *= scale + offset_b *= scale + + self._white_reference_ab = ( + 128.0 + offset_a, + 128.0 + offset_b, + ) + + # ======================================================== + # Hue learning + # ======================================================== + # + # Slowly re-centers the loose hue pre-filter on whatever hue + # the camera is ACTUALLY recording for the calibration green, + # instead of forever trusting the single fixed + # CALIBRATION_TARGET_HUE_DEGREES value from config.py. This is + # what lets the system track a slow lighting change over a long + # show without needing to be re-tuned on-site - while + # ADAPTIVE_MAX_DRIFT_DEGREES keeps it from ever wandering far + # enough to start matching some unrelated color. + + def _effective_hue_center(self): + if ( + CALIBRATION_ADAPTIVE_HUE_ENABLED + and self._learned_hue_center is not None + ): + return self._learned_hue_center + + return CALIBRATION_TARGET_HUE_DEGREES + + def _update_learned_hue(self, measured_hue_degrees): + if measured_hue_degrees is None: + return + + if not CALIBRATION_ADAPTIVE_HUE_ENABLED: + return + + current = ( + self._learned_hue_center + if self._learned_hue_center is not None + else CALIBRATION_TARGET_HUE_DEGREES + ) + + blended = self._circular_blend_degrees( + current, + measured_hue_degrees, + CALIBRATION_ADAPTIVE_HUE_BLEND, + period=360.0, + ) + + drift = self._circular_diff_degrees( + blended, + CALIBRATION_TARGET_HUE_DEGREES, + period=360.0, + ) + + max_drift = CALIBRATION_ADAPTIVE_MAX_DRIFT_DEGREES + + if abs(drift) > max_drift: + blended = ( + CALIBRATION_TARGET_HUE_DEGREES + + math.copysign(max_drift, drift) + ) % 360.0 + + self._learned_hue_center = blended + + @staticmethod + def _representative_hue(samples): + """ + Circular mean (vector average, not a plain arithmetic mean + - hue wraps around, so naively averaging e.g. 350 degrees + and 10 degrees must give 0, not 180) of a list of hue + samples in degrees, skipping any that are None. Returns + None if nothing usable is in the list. + """ + valid = [ + value + for value in samples + if value is not None + ] + + if not valid: + return None + + radians = np.radians(valid) + + mean_sin = float(np.mean(np.sin(radians))) + mean_cos = float(np.mean(np.cos(radians))) + + return float( + np.degrees( + np.arctan2(mean_sin, mean_cos) + ) + % 360.0 + ) + + @staticmethod + def _circular_diff_degrees(a, b, period): + """Shortest signed a-b distance on a circle of this period.""" + return ( + (a - b + period / 2.0) % period + ) - period / 2.0 + + @classmethod + def _circular_blend_degrees(cls, old, new, alpha, period): + diff = cls._circular_diff_degrees( + new, + old, + period, + ) + + return (old + alpha * diff) % period + + # ======================================================== + # Geometry / quality scoring + # ======================================================== + + def _quad_quality( + self, + points, + frame_shape, + ): + height, width = frame_shape[:2] + + frame_area = width * height + + area = self._polygon_area(points) + + area_ratio = area / frame_area + + if area_ratio < CALIBRATION_MIN_SCREEN_AREA_RATIO: + return 0.0 + + if area_ratio > CALIBRATION_MAX_SCREEN_AREA_RATIO: + return 0.0 + + contour = points.reshape( + (-1, 1, 2) + ).astype(np.float32) + + if not cv2.isContourConvex(contour): + return 0.0 + + sides = self._side_lengths(points) + + if min(sides) < 50: + return 0.0 + + top_bottom = ( + min(sides[0], sides[2]) + / max(sides[0], sides[2]) + ) + + left_right = ( + min(sides[1], sides[3]) + / max(sides[1], sides[3]) + ) + + side_quality = ( + top_bottom + left_right + ) / 2.0 + + angle_quality = 1.0 + + for i in range(4): + p1 = points[i - 1] + p2 = points[i] + p3 = points[(i + 1) % 4] + + v1 = p1 - p2 + v2 = p3 - p2 + + n1 = np.linalg.norm(v1) + n2 = np.linalg.norm(v2) + + if n1 < 1e-6 or n2 < 1e-6: + return 0.0 + + cos_angle = np.clip( + np.dot(v1, v2) / (n1 * n2), + -1.0, + 1.0, + ) + + angle = np.degrees( + np.arccos(cos_angle) + ) + + if angle < 45 or angle > 135: + angle_quality *= 0.3 + elif angle < 60 or angle > 120: + angle_quality *= 0.7 + + area_quality = min( + area_ratio / 0.40, + 1.0, + ) + + return float( + area_quality * 0.35 + + side_quality * 0.35 + + angle_quality * 0.30 + ) + + @staticmethod + def _polygon_area(points): + return abs( + cv2.contourArea( + points.astype(np.float32) + ) + ) + + @staticmethod + def _side_lengths(points): + return [ + np.linalg.norm(points[1] - points[0]), + np.linalg.norm(points[2] - points[1]), + np.linalg.norm(points[3] - points[2]), + np.linalg.norm(points[0] - points[3]), + ] + + @staticmethod + def _order_points(points): + points = np.asarray( + points, + dtype=np.float32, + ) + + s = points.sum(axis=1) + d = np.diff(points, axis=1).reshape(-1) + + top_left = points[np.argmin(s)] + top_right = points[np.argmin(d)] + bottom_right = points[np.argmax(s)] + bottom_left = points[np.argmax(d)] + + return np.array( + [top_left, top_right, bottom_right, bottom_left], + dtype=np.float32, + ) + + # ======================================================== + # Temporal stability + # ======================================================== + + def _stable_against_previous( + self, + points, + frame_shape, + ): + if self.previous_points is None: + return True + + height, width = frame_shape[:2] + + diagonal = np.sqrt( + width * width + height * height + ) + + distance = self._corners_distance( + points, + self.previous_points, + ) + + return ( + distance + <= diagonal * CALIBRATION_MAX_CORNER_JUMP_RATIO + ) + + @staticmethod + def _corners_distance(a, b): + return float( + np.mean( + np.linalg.norm(a - b, axis=1) + ) + ) + + # ======================================================== + # Accepting / refining calibration + # ======================================================== + + def _accept_or_refine(self, frame_shape): + final_points = self._order_points( + np.median( + np.stack(self.history, axis=0), + axis=0, + ) + ) + + quality = self._quad_quality( + final_points, + frame_shape, + ) + + if quality < CALIBRATION_MIN_QUALITY: + return + + area_ratio = self._polygon_area( + final_points + ) / (frame_shape[0] * frame_shape[1]) + + if area_ratio < CALIBRATION_MIN_SCREEN_AREA_RATIO: + return + + if self.reference_points is not None: + final_points = self._smooth( + self.reference_points, + final_points, + ) + + matrix = self._calculate_matrix( + final_points + ) + + if matrix is None: + return + + self.matrix = matrix + self.reference_points = final_points + + self._update_learned_hue( + self._representative_hue(self.hue_history) + ) + + @staticmethod + def _smooth(old_points, new_points): + alpha = CALIBRATION_SMOOTHING_ALPHA + + return ( + old_points * (1.0 - alpha) + + new_points * alpha + ).astype(np.float32) + + def _calculate_matrix(self, points): + destination = np.float32( + [ + [0, 0], + [SCREEN_WIDTH, 0], + [SCREEN_WIDTH, SCREEN_HEIGHT], + [0, SCREEN_HEIGHT], + ] + ) + + try: + matrix = cv2.getPerspectiveTransform( + points.astype(np.float32), + destination, + ) + except cv2.error: + return None + + if matrix is None: + return None + + if not np.isfinite(matrix).all(): + return None + + return matrix + + # ======================================================== + # Failure handling + # ======================================================== + + def _handle_failed_detection(self): + self.failed_frame_count += 1 + + self.history = [] + self.hue_history = [] + self.previous_points = None + + # ======================================================== + # Debugging + # ======================================================== + + def debug_draw(self, frame): + """ + Returns an annotated copy of `frame` showing the best + currently-detected candidate (if any), its quality/color + scores, the hue center currently being searched around + (handy for seeing the adaptive learning track the room's + actual lighting), and whether calibration is locked. Useful + for tuning config.py's hue/saturation/value floors on-site + against your real camera and lighting. + """ + output = frame.copy() + + if self.is_calibrated(): + candidate = self._find_best_border_candidate(frame) + else: + candidate = self._find_best_full_screen_candidate( + frame + ) + + if candidate is not None: + points = candidate["points"].astype( + np.int32 + ) + + cv2.polylines( + output, + [points.reshape((-1, 1, 2))], + True, + (0, 255, 0), + 3, + ) + + text = ( + f"quality={candidate['quality']*100:.0f}% " + f"green={candidate['target_ratio']*100:.0f}% " + f"white={candidate['white_ratio']*100:.0f}% " + f"stable={len(self.history)}/" + f"{CALIBRATION_REQUIRED_STABLE_FRAMES}" + ) + else: + text = "searching..." + + status = ( + "CALIBRATED" + if self.is_calibrated() + else "NOT CALIBRATED" + ) + + hue_line = ( + f"hue center ~{self._effective_hue_center():.0f} deg " + f"(anchor {CALIBRATION_TARGET_HUE_DEGREES} deg)" + ) + + cv2.putText( + output, + f"{status} | {text}", + (20, 30), + cv2.FONT_HERSHEY_SIMPLEX, + 0.7, + (0, 255, 0), + 2, + cv2.LINE_AA, + ) + + cv2.putText( + output, + hue_line, + (20, 58), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (0, 255, 0), + 2, + cv2.LINE_AA, + ) + + return output \ No newline at end of file diff --git a/camera.py b/camera.py new file mode 100644 index 0000000..4fb0135 --- /dev/null +++ b/camera.py @@ -0,0 +1,258 @@ +import threading +import time + +import cv2 +import numpy as np + +from config import ( + CAMERA_FROZEN_DIFF_THRESHOLD, + CAMERA_FROZEN_SECONDS, + CAMERA_CONNECT_RETRY_SECONDS, +) + + +class Camera: + """ + Wrapper around cv2.VideoCapture for an HTTP/MJPEG camera stream, + with a few exhibition-day survival features: + + - Frame grabbing runs on its own background thread that reads as + fast as the stream delivers and keeps only the SINGLE most + recent frame. cv2.VideoCapture's own buffering on network + streams often ignores CAP_PROP_BUFFERSIZE; if the consumer is + ever briefly slower than the stream, frames queue up upstream + and every later read() returns an ever-staler frame ("the game + gets slower and slower"). Draining continuously and handing + out only the newest frame makes that impossible. + + - Connecting also happens on that thread and never blocks the + caller: the game window comes up immediately (and stays + responsive to the keyboard) even if the phone/app isn't + streaming yet. It keeps retrying forever, both at startup and + after a mid-show drop. + + - Every frame gets an increasing id (see read_with_id), so the + vision thread can skip a frame it has already processed + instead of redoing the same work on it. + + - Frozen-frame detection: apps that pause in the background + often keep re-sending the last JPEG, which cv2.VideoCapture + happily "reads" forever. We track whether the image content + actually changes and expose is_frozen() so callers can refuse + to trust a feed that looks alive but isn't. + """ + + MAX_CONSECUTIVE_FAILURES = 5 + RECONNECT_BACKOFF_SECONDS = 1.0 + + def __init__(self, url: str): + self.url = url + self.cap = None + + self._stop_event = threading.Event() + + self._state_lock = threading.Lock() + self._last_thumb = None + self._last_change_time = time.monotonic() + + self._frame_lock = threading.Lock() + self._latest_frame = None + self._latest_id = 0 + self._latest_error = "Camera not connected yet." + + self._thread = threading.Thread( + target=self._run, + name="camera-reader", + daemon=True, + ) + self._thread.start() + + # ======================================================== + # Connection + # ======================================================== + + def _connect_with_retry(self): + """ + Keeps trying to open the stream until it works or the + camera is released. This runs unattended in a booth: the + phone/app may not be streaming yet, or wifi may take a + moment, and nobody may be there to relaunch anything. + """ + attempt = 0 + + while not self._stop_event.is_set(): + attempt += 1 + + try: + self._open() + print("[CAMERA] connected.") + return + except Exception as error: + with self._frame_lock: + self._latest_error = ( + f"Camera not connected ({error})" + ) + + print( + f"[CAMERA] connect attempt {attempt} failed " + f"({error}) - retrying in " + f"{CAMERA_CONNECT_RETRY_SECONDS:.1f}s..." + ) + + self._stop_event.wait( + CAMERA_CONNECT_RETRY_SECONDS + ) + + def _open(self): + if self.cap is not None: + try: + self.cap.release() + except Exception: + pass + + self.cap = cv2.VideoCapture(self.url) + + # Cheap extra defense on backends that honor it; the reader + # thread is what actually guarantees we never fall behind. + try: + self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1) + except Exception: + pass + + if not self.cap.isOpened(): + raise RuntimeError( + f"Could not connect to camera stream: {self.url}" + ) + + # ======================================================== + # Background thread + # ======================================================== + + def _run(self): + self._connect_with_retry() + + consecutive_failures = 0 + + while not self._stop_event.is_set(): + try: + success, frame = self.cap.read() + except Exception as error: + success, frame = False, None + + with self._frame_lock: + self._latest_error = ( + f"Camera read error: {error}" + ) + + if not success or frame is None: + consecutive_failures += 1 + + with self._frame_lock: + self._latest_error = ( + "Failed to read frame from camera." + ) + + if ( + consecutive_failures + >= self.MAX_CONSECUTIVE_FAILURES + ): + self._stop_event.wait( + self.RECONNECT_BACKOFF_SECONDS + ) + + self._connect_with_retry() + + consecutive_failures = 0 + else: + # A failed read on some backends returns + # instantly; don't spin a core on it. + self._stop_event.wait(0.01) + + continue + + consecutive_failures = 0 + + self._update_freeze_state(frame) + + # Publish only the newest frame; anything the consumer + # hasn't taken yet is simply replaced. + with self._frame_lock: + self._latest_frame = frame + self._latest_id += 1 + self._latest_error = None + + # ======================================================== + # Public API + # ======================================================== + + def read_with_id(self): + """Newest frame and its id; raises RuntimeError if none yet.""" + with self._frame_lock: + frame = self._latest_frame + frame_id = self._latest_id + error = self._latest_error + + if frame is None: + raise RuntimeError( + error or "No frame received from camera yet." + ) + + return frame, frame_id + + def read(self): + return self.read_with_id()[0] + + def is_frozen(self): + with self._state_lock: + last_change = self._last_change_time + + return ( + time.monotonic() - last_change + ) > CAMERA_FROZEN_SECONDS + + def release(self): + self._stop_event.set() + + # Let the reader thread notice and finish before the + # capture object is torn down underneath it. + if self._thread.is_alive(): + self._thread.join(timeout=2.0) + + if self.cap is not None: + try: + self.cap.release() + except Exception: + pass + + # ======================================================== + # Frozen-frame detection + # ======================================================== + + def _update_freeze_state(self, frame): + # Shrink FIRST (INTER_AREA averages, so it also suppresses + # pixel noise) and only then convert to gray - converting a + # full frame just to throw almost all of it away is wasted + # work on the capture thread. + thumb = cv2.cvtColor( + cv2.resize( + frame, + (32, 18), + interpolation=cv2.INTER_AREA, + ), + cv2.COLOR_BGR2GRAY, + ).astype(np.int16) + + now = time.monotonic() + + with self._state_lock: + if self._last_thumb is None: + self._last_change_time = now + else: + diff = float( + np.mean(np.abs(thumb - self._last_thumb)) + ) + + if diff > CAMERA_FROZEN_DIFF_THRESHOLD: + self._last_change_time = now + + self._last_thumb = thumb diff --git a/config.py b/config.py new file mode 100644 index 0000000..a35dfa7 --- /dev/null +++ b/config.py @@ -0,0 +1,599 @@ +# config.py + +# ============================================================ +# Screen +# ============================================================ + +SCREEN_WIDTH = 1530 +SCREEN_HEIGHT = 920 + + +# ============================================================ +# Camera +# ============================================================ + +# CAMERA_URL = "http://172.30.2.62:8080/video" +# CAMERA_URL = "http://172.30.1.216:4747/video" +CAMERA_URL = "http://192.168.1.194:4747/video" + +# How long to wait between attempts to (re)connect to the camera +# when it is unreachable - both on startup (phone app not open yet) +# and whenever a reconnect is needed mid-show. The game keeps +# retrying forever instead of crashing, since this runs unattended +# in a booth and nobody may be there to relaunch it. +CAMERA_CONNECT_RETRY_SECONDS = 2.0 + + +# ============================================================ +# Detection intervals +# ============================================================ + +# Both of these run on a background vision thread (see vision.py), +# never on the render loop, so they can no longer cause frame +# hitches. Obstacle detection works on a downscaled image (see +# DETECTION_SCALE) and is cheap enough to run 4x per second, which +# makes moving a sticker feel responsive. +MONITOR_DETECTION_INTERVAL = 1.0 +OBSTACLE_DETECTION_INTERVAL = 0.25 + + +# ============================================================ +# Calibration - screen border pattern +# ============================================================ +# +# Instead of four small corner markers, the monitor draws a +# magenta/pink outer ring with a white inner ring around the edge +# of the screen, permanently (during both calibration and normal +# gameplay - see Renderer.draw_calibration_border). This is much +# more robust in a busy, noisy exhibition hall than four small +# markers: +# - it's a single large, high-contrast shape instead of four +# tiny independent ones, so a person briefly walking in front +# of one corner doesn't break detection the way losing one out +# of four markers would; +# - requiring BOTH the pink AND the white ring to be present +# (in the right relative position) makes it very unlikely that +# a random pink or white object/reflection in the room could +# be mistaken for the screen. + +# Colors drawn on the monitor itself (RGB, used by pygame). +# +# Why GREEN instead of magenta/pink: the outer ring only has to be +# the single easiest possible color for a CAMERA (not a human eye) +# to pick out of a crowded, oddly-lit exhibition hall, and green is +# the best pick available for three concrete reasons: +# 1. Every consumer camera sensor uses a Bayer filter with TWICE +# as many green photosites as red or blue (the pattern a human +# eye is most sensitive to), so the green channel is always +# the least noisy, highest-resolution channel the camera has - +# this is exactly why film/broadcast green-screens are green +# and not some other color. +# 2. Pink/magenta sits close to skin tones and to a lot of common +# clothing under warm (incandescent/halogen) exhibition +# lighting, which white-balance algorithms can shift toward +# orange/red - pushing it out of a fixed HSV window entirely. +# Saturated green has no such close neighbour in a crowd of +# people, and survives white-balance shifts far better (most +# auto white balance implementations anchor their correction +# around making genuinely neutral/gray areas look neutral, +# which leaves a strongly green-dominant pixel green-dominant +# under almost any color temperature). +# 3. It pairs well with the detection method below: "is the green +# channel clearly stronger than red AND blue" is a RELATIVE +# test, so it survives brightness/exposure changes (someone's +# shadow, a spotlight) that would otherwise shift an absolute +# HSV threshold out of range. +CALIBRATION_PATTERN_OUTER_COLOR = (40, 255, 60) # saturated green +CALIBRATION_PATTERN_INNER_COLOR = (255, 255, 255) # white + +# Thickness of each ring, in screen pixels (at the SCREEN_WIDTH x +# SCREEN_HEIGHT design resolution above). +CALIBRATION_OUTER_BORDER_SIZE = 40 +CALIBRATION_INNER_BORDER_SIZE = 26 + +# ------------------------------------------------------------ +# Target-color detection +# ------------------------------------------------------------ +# +# The exact color the camera records for CALIBRATION_PATTERN_OUTER_ +# COLOR depends on the camera's white balance/exposure and the +# room's lighting, and can drift over the course of a show (the +# hall's lighting changing, the phone's camera re-adjusting +# exposure, etc.) - a single fixed HSV box, tuned once, is exactly +# what used to make detection "work great at setup time, then get +# flaky an hour into the exhibition". So the mask that finds the +# outer ring is now built from two cooperating, auto-adapting +# pieces instead of one fixed threshold (see Calibration. +# _make_target_mask for the implementation): +# +# 1. A wide, loose hue pre-filter (this section) that only rules +# out colors nowhere near green at all (reds, blues, skin, +# gray) - it does NOT try to pin down the exact shade. +# 2. Within whatever passes that pre-filter, a per-frame Otsu +# threshold on how green-dominant each pixel is (how far G +# exceeds R and B) - Otsu finds the split point that best +# separates "clearly green" from "not" IN THAT FRAME, so it +# re-adapts itself to the current lighting automatically, +# frame by frame, with no fixed brightness/saturation number +# to go stale. +# +# CALIBRATION_TARGET_HUE_DEGREES/TOLERANCE describe the loose +# pre-filter in standard 0-360 degree hue (green = 120 degrees). +CALIBRATION_TARGET_HUE_DEGREES = 120 +CALIBRATION_TARGET_HUE_TOLERANCE_DEGREES = 45 + +# Loose floor for saturation/brightness in the pre-filter - just +# enough to exclude flat gray/black/white areas, not to pin an +# exact shade (that's Otsu's job). +CALIBRATION_MIN_TARGET_SATURATION = 50 +CALIBRATION_MIN_TARGET_VALUE = 35 + +# Once locked on, the system also WATCHES the hue it is actually +# seeing inside the confirmed ring and slowly re-centers the +# pre-filter on it (see Calibration._update_learned_hue). This is +# what lets it track a slow lighting drift over the course of a +# multi-hour show instead of needing to be re-tuned on-site. +# ADAPTIVE_MAX_DRIFT_DEGREES bounds how far that learned center may +# ever wander from the true CALIBRATION_TARGET_HUE_DEGREES, so a +# run of bad detections can never "learn" its way onto some +# unrelated color. +CALIBRATION_ADAPTIVE_HUE_ENABLED = True +CALIBRATION_ADAPTIVE_HUE_BLEND = 0.08 +CALIBRATION_ADAPTIVE_MAX_DRIFT_DEGREES = 30 + +# The white inner ring is found the same way detection.py finds +# white stickers (see config's "White obstacle detection" section +# for the full reasoning): a LOCAL adaptive brightness test rather +# than one fixed floor for the whole frame, and LAB chroma rather +# than HSV saturation for "is this actually neutral". A fixed HSV +# box here was just as fragile under real exhibition lighting as the +# old fixed box was for stickers. +CALIBRATION_WHITE_ADAPTIVE_OFFSET = 8 +CALIBRATION_WHITE_MIN_ABSOLUTE_LIGHTNESS = 50 + +# One-sided: the only other bright thing nearby is the green ring +# itself, so it's enough to exclude clearly GREEN-dominant pixels +# rather than pin an exact "how white" number - see the comment in +# Calibration._make_white_mask for why a two-sided chroma ceiling +# can't reliably cover every possible lighting cast but this can. +CALIBRATION_WHITE_MAX_GREEN_DOMINANCE = 50 + +# The white inner ring also serves as a live "gray card": its mean +# LAB (a*, b*) as the camera currently records it is continuously +# learned (slow exponential blend) and handed to detection.py, which +# judges a candidate sticker's whiteness relative to THAT instead of +# absolute neutral - see Calibration.white_reference(). Colored +# ambient light shifts a genuinely white/neutral surface's recorded +# color for physical reasons no fixed threshold can see around; this +# is what lets the sticker detector stay correct anyway, and track +# the room's lighting changing over a long show without re-tuning. +CALIBRATION_WHITE_REFERENCE_BLEND = 0.05 + +# How far (LAB a/b units) the learned white point may ever drift +# from true neutral, so a run of bad samples can never teach the +# sticker detector a wildly wrong idea of white. +CALIBRATION_WHITE_REFERENCE_MAX_OFFSET = 40.0 + +# The detected quad's area, as a fraction of the full camera frame, +# must fall in this range. Filters out both tiny false positives +# (reflections, noise) and implausibly large ones. +CALIBRATION_MIN_SCREEN_AREA_RATIO = 0.10 +CALIBRATION_MAX_SCREEN_AREA_RATIO = 0.90 + +# How much of the sampled outer/inner ring band must actually match +# its expected color for a candidate to be accepted. +CALIBRATION_MIN_OUTER_TARGET_RATIO = 0.55 +CALIBRATION_MIN_INNER_WHITE_RATIO = 0.40 + +# Combined quality/color score (0-1) required before a detection is +# even considered as a calibration candidate. +CALIBRATION_MIN_QUALITY = 0.50 + +# While NOT yet calibrated, the whole screen is filled solid green +# (Renderer.draw_full_calibration_screen) instead of just the thin +# border ring. This gives the very first lock-on the simplest, +# most unambiguous target possible: one huge solid-color rectangle +# with nothing else on screen. This is the fraction of that +# candidate quad's own interior that must actually be green for it +# to count - it naturally stays low while someone is standing in +# front of part of the monitor, so a person walking through frame +# during calibration just delays the lock instead of causing a bad +# one. +CALIBRATION_MIN_FULL_TARGET_RATIO = 0.85 + +# Temporal stability: how many consecutive matching detections are +# required before (re)locking calibration, and how far (as a +# fraction of the camera frame's diagonal) the corners are allowed +# to drift between frames and still count as "the same" detection. +# This is what lets the system ignore a person briefly walking +# through frame instead of resetting calibration. +CALIBRATION_REQUIRED_STABLE_FRAMES = 5 +CALIBRATION_MAX_CORNER_JUMP_RATIO = 0.035 + +# Exponential smoothing applied to the accepted corner positions on +# every successful re-detection once already calibrated, so small +# frame-to-frame jitter doesn't make the transform wobble. +CALIBRATION_SMOOTHING_ALPHA = 0.25 + +# Morphological closing kernel (pixels) used to bridge small gaps +# in the color mask caused by glare/reflections on the monitor +# glass. Bigger = more tolerant of glare, but also more likely to +# merge in nearby unrelated green/white blobs - tune together with +# the hue settings above. +CALIBRATION_MASK_CLOSE_KERNEL = 17 + + +# ============================================================ +# Camera feed health +# ============================================================ +# +# If the raw camera feed barely changes for this long, treat it as +# a stuck/cached/frozen frame (phone screen locked, streaming app +# paused in the background, stale network buffer, dead wifi +# re-serving the last JPEG, lens covered, etc.) rather than a live +# view, and don't trust it for calibration or obstacle detection. +CAMERA_FROZEN_DIFF_THRESHOLD = 0.6 +CAMERA_FROZEN_SECONDS = 2.0 + + +# ============================================================ +# Rendering +# ============================================================ + +BACKGROUND_COLOR = (0, 0, 0) + +TARGET_FPS = 144 + +# Only repaint (and push to the display) the small screen regions +# that actually changed since the last frame, instead of redrawing +# and re-uploading all 1920x1080 pixels 144 times a second. On most +# machines this is the single biggest rendering speed-up. If you +# ever see stale "ghost" pixels on the monitor (a driver that +# ignores partial updates), set this to False - everything still +# works, just with more CPU/GPU load. +USE_DIRTY_RECTS = True + +# Safety net for USE_DIRTY_RECTS: force one complete repaint every +# this many seconds, so even a misbehaving driver can't leave a +# smudge on the screen for long. +FULL_REPAINT_SECONDS = 10.0 + + +# ============================================================ +# Visual effects (purely cosmetic - never affect physics) +# ============================================================ +# +# Master switch. Everything below is drawn ONLY with saturated +# colors (never white blobs) and only in small, dim glows, because +# the camera is looking at this same screen and must never mistake +# an effect for a white sticker. If detection ever misbehaves and +# you suspect the visuals, flip this to False first to find out. +VISUAL_EFFECTS_ENABLED = True + +# Each ball gets its own hue (degrees, 0-360) from this range. The +# default skips magenta/pink (roughly 280-350) on purpose: that is +# the calibration color, and it must stay unique to the calibration +# border so the camera never confuses the two. +BALL_HUE_MIN_DEGREES = 15 +BALL_HUE_MAX_DEGREES = 255 + +# Every time a ball bounces off a sticker its hue shifts by this +# many degrees (staying inside the range above) - a small, very +# visible "the sticker reacted to me" cue that people love. +BALL_HUE_SHIFT_ON_BOUNCE = 22 + +# Fading comet tail behind each ball: how many past positions it +# remembers, and how often (seconds) a new one is recorded. +BALL_TRAIL_LENGTH = 16 +BALL_TRAIL_SAMPLE_SECONDS = 1.0 / 60.0 + +# Soft colored glow around each ball (brighter the faster it moves). +BALL_GLOW_ENABLED = True + +# Springy squash-and-stretch: balls elongate along their direction +# of travel with speed, and squash flat for a moment on a hard hit. +BALL_SQUASH_STRETCH_ENABLED = True +BALL_SQUASH_DURATION_SECONDS = 0.16 + +# Sparks + expanding ring on a bounce whose impact speed (px/s) +# exceeds this; gentler taps and resting contact produce nothing, +# so a ball lying still on a sticker stays calm. +IMPACT_EFFECT_MIN_SPEED = 160.0 +MAX_PARTICLES = 260 + + +# ============================================================ +# Detected obstacles (what is painted back onto the monitor) +# ============================================================ +# +# When True, every obstacle the camera currently sees is also +# painted white on the game's own screen (i.e. on the physical +# monitor itself), where it was detected - as the real tilted +# rectangle, not an axis-aligned box. When False, nothing extra is +# drawn for it. Either way this is purely visual - balls always +# collide with every currently-detected obstacle regardless of this +# flag; it never affects physics. +SHOW_DETECTED_OBSTACLES = False + +DETECTED_OBSTACLE_COLOR = (255, 255, 255) +DETECTED_OBSTACLE_BORDER_WIDTH = 0 + +# The painted white is deliberately drawn a little SMALLER than the +# sticker (this fraction of the sticker's short side is trimmed off +# each edge). Reason: the camera looks at this very screen. If the +# painted white were exactly sticker-sized, then after someone +# removes the sticker the camera would keep seeing the painted +# white and "detect" the (now nonexistent) sticker forever. Painting +# inset keeps it hidden under the real sticker, and the detector +# additionally blanks out exactly the area that was painted before +# looking for stickers (see detection.py), so a removed sticker +# really disappears. +DETECTED_OBSTACLE_PAINT_INSET_RATIO = 0.22 + + +# ============================================================ +# Ball physics +# ============================================================ + +BALL_RADIUS = 15 + +# Each ball's radius is randomly scaled by up to +/- this fraction +# (bigger balls are also heavier in ball-vs-ball collisions). +BALL_RADIUS_VARIATION = 0.25 + +BALL_GRAVITY = 900 + +# Fraction of normal speed kept after bouncing off a sticker or a +# screen edge (1.0 = perfectly elastic: with gravity, a ball just +# keeps bouncing at the same energy forever and never settles down, +# which is the default here). Lower it (e.g. ~0.75) for a "real +# rubber ball" feel that loses a little height on every bounce +# instead. Very slow contact against a STICKER is still always +# treated as fully inelastic (BALL_REST_SPEED) so a ball resting on +# one doesn't buzz at the pixel level; screen edges always bounce at +# the full BALL_RESTITUTION regardless of speed. +BALL_RESTITUTION = 1.0 +BALL_REST_SPEED = 55.0 + +# Fraction of sliding (tangential) speed lost per bounce off a +# sticker - what makes a ball roll/slide along a tilted sticker +# instead of skating on ice. 0 = no energy lost this way either (the +# default, paired with BALL_RESTITUTION = 1.0 above); raise it for +# more of a "rolling to a stop on a ramp" feel. +BALL_FRICTION = 0.0 + +# Safety cap on speed (px/s), so a freak event can never launch a +# ball fast enough to tunnel through a sticker. +BALL_MAX_SPEED = 2000.0 + +# Balls collide with each other: overlapping balls push apart and +# exchange momentum (heavier balls push lighter ones around). 1.0 = +# no energy lost in the collision either, matching the walls/ +# stickers above. +BALL_BALL_COLLISION_ENABLED = True +BALL_BALL_RESTITUTION = 1.0 + +# Physics runs in equal slices of at most this many seconds, so +# nothing tunnels even on a slow frame. +PHYSICS_MAX_STEP_SECONDS = 1.0 / 240.0 + + +# ============================================================ +# Ball spawning and stuck balls +# ============================================================ +# +# The two settings that matter most, kept together and right up +# front so they're easy to find and tune: +# +# A new ball drops in every BALL_SPAWN_INTERVAL_SECONDS, on its own +# clock, completely independent of what earlier balls are doing or +# whether they've disappeared yet. +BALL_SPAWN_INTERVAL_SECONDS = 2.5 + +# ============================================================ +# Launcher configuration +# ============================================================ +# +# Attached to top-center of the screen. +# Angles are in degrees relative to straight down (0 = straight down, +# negative = left, positive = right). +LAUNCHER_MIN_ANGLE = -70.0 +LAUNCHER_MAX_ANGLE = 70.0 + +# Dimensions of the launcher tube in screen pixels. +LAUNCHER_LENGTH = 75.0 +LAUNCHER_WIDTH = 26.0 + +# Angular rotation speed (controls continuous smooth movement). +LAUNCHER_ROTATION_SPEED = 35.0 + +# Initial ball speed when exiting the nozzle (px/s). +LAUNCHER_BALL_SPAWN_SPEED = 280.0 + +# Every ball is removed (with a little pop effect, see effects.py) +# this many seconds after it was spawned, whatever else is +# happening to it. 0 = balls live forever (until/unless the stuck- +# timeout below removes them instead). +BALL_MAX_LIFETIME_SECONDS = 20.0 + +# --- Everything below is secondary detail for the two settings above --- + +# Hard ceiling on balls alive at once - only a safety net so trapped +# balls can never accumulate without limit over a long show. +MAX_SIMULTANEOUS_BALLS = 12 + +# Random horizontal drop position. With BALL_SPAWN_RANDOM_X = False +# every ball drops from the middle of the screen (the old +# behavior). Margin keeps drops away from the side walls. +BALL_SPAWN_RANDOM_X = True +BALL_SPAWN_MARGIN = 120 +BALL_SPAWN_VELOCITY_JITTER = 60.0 + +# A ball is "stuck" if it has stayed within BALL_STUCK_DISTANCE +# pixels of one spot for BALL_STUCK_TIMEOUT_SECONDS - usually a +# pocket between stickers, but with every screen edge now a solid +# wall (see physics.py) a ball resting on an empty floor with no +# obstacles nearby counts too. A stuck ball stays put for that long +# (so visitors can see it) and is then removed with a small pop, +# freeing its slot, same as the lifetime limit above. 0 = never +# remove stuck balls (BALL_MAX_LIFETIME_SECONDS still applies). +BALL_STUCK_DISTANCE = 30.0 +BALL_STUCK_TIMEOUT_SECONDS = 8.0 + + +# ============================================================ +# White obstacle detection +# ============================================================ + +# The camera image is warped to screen space at this fraction of +# the real screen resolution before looking for stickers. 0.5 makes +# detection ~4x cheaper with no meaningful loss of precision (a +# sticker is still tens of pixels wide). All the pixel values below +# stay in real screen pixels - the code converts. +DETECTION_SCALE = 0.5 + +# Real sticker size limits, in screen pixels squared. +MIN_OBSTACLE_AREA = 1500 +MAX_OBSTACLE_AREA = 200000 + +# --- Finding candidate blobs (deliberately loose) ------------- +# Pass 1 finds anything vaguely bright and unsaturated. It is meant +# to be generous; the strict checks below decide what is a sticker. +# +# Both "bright" and "unsaturated" are judged in CIELAB, not HSV (see +# imaging.py): L (lightness) and chroma (how colorful a pixel is) +# are far more independent of each other there than HSV's V and S, +# which is what makes the two tests below behave consistently +# whether a sticker sits in a shadowed corner or under a spotlight. +# +# "Bright" is no longer one fixed brightness floor for the whole +# frame, either. A single global number means a sticker in dimmer +# ambient light can fall below it and never even reach the per-blob +# refinement step below - which is exactly what made otherwise +# identical stickers seem to be "detected by force" some of the +# time. Instead, pass 1 asks a strictly LOCAL question - "is this +# pixel clearly brighter than the small neighborhood around it?" +# (cv2.adaptiveThreshold) - the same question a human eye asks when +# picking out a pale sticker on a dim part of a black screen, and it +# holds regardless of the overall exposure level that frame happens +# to have. OBSTACLE_ADAPTIVE_OFFSET is how much brighter than that +# local neighborhood a pixel must be; OBSTACLE_MIN_ABSOLUTE_LIGHTNESS +# is a sanity floor underneath it, so camera sensor noise sitting in +# a totally dark area can never be mistaken for "locally brighter". +OBSTACLE_ADAPTIVE_OFFSET = 14 +OBSTACLE_MIN_ABSOLUTE_LIGHTNESS = 55 + +# The neighborhood size (screen px) the adaptive threshold above +# looks at is tied to the sticker's own size - large enough that the +# neighborhood is mostly black screen around the sticker rather than +# the sticker's own bright pixels (which would otherwise drag the +# local average up and make the sticker fail to stand out against +# ITSELF). Used as a multiple of the short side once a size is +# known; OBSTACLE_ADAPTIVE_DEFAULT_SHORT_SIDE is only the starting +# guess used before any sticker has been seen yet. +OBSTACLE_ADAPTIVE_BLOCK_SPAN = 2.4 +OBSTACLE_ADAPTIVE_DEFAULT_SHORT_SIDE = 90 + +# Loose chroma ceiling for pass 1 (and for the per-blob edge +# refinement below) - just needs to exclude obviously colored +# objects (clothing, skin, colored lighting splashed on the floor), +# not pin an exact "how white" number (that is MAX_OBSTACLE_CHROMA +# below). +OBSTACLE_COARSE_CHROMA_MAX = 45 + +# Each blob's edge is then re-drawn at the "half-way" LIGHTNESS +# between the sticker's own paper level and the dark screen next to +# it. That makes a dim sticker in a shadowy corner and an identical +# sticker under a spotlight come out exactly the same size, which is +# what makes all stickers look and behave alike. +OBSTACLE_EDGE_LEVEL = 0.5 + +# --- Deciding whether a blob really is a sticker -------------- +# Average lightness/chroma INSIDE the blob (CIELAB - see above). +MIN_OBSTACLE_BRIGHTNESS = 130 +MAX_OBSTACLE_CHROMA = 28 + +# Fraction of the blob's own best-fit rotated rectangle it fills. +# A disc (a ball's glow, a bulb) scores 0.785, so anything above +# ~0.84 rejects round things outright. +MIN_OBSTACLE_RECTANGULARITY = 0.86 + +# Fraction of its convex hull the blob fills (rejects L-shapes, +# blobs with bites out of them). +MIN_OBSTACLE_SOLIDITY = 0.90 + +# Longest side / shortest side. Squares up to index-card shapes. +MAX_OBSTACLE_ASPECT_RATIO = 4.0 + +# Paper is a uniform, diffuse surface. Reflections and hot-spots +# vary a lot in brightness across their surface. +MAX_OBSTACLE_BRIGHTNESS_STD = 30.0 + +# A real sticker sits on a black screen: this fraction of the ring +# of pixels just outside it must be clearly darker than the sticker. +MIN_OBSTACLE_DARK_SURROUND_RATIO = 0.55 + +# Only look for stickers inside the calibration border, and never +# closer than this many extra screen pixels to it (a sticker +# touching the bright border would merge with it). +OBSTACLE_BORDER_MASK_MARGIN = 4 +OBSTACLE_EDGE_MARGIN = 5 + +# --- All stickers are identical: use that ------------------- +# Once a few stickers have been seen the system learns their size +# and rejects blobs that don't match (within this fraction) - this +# is what kills "forced" detections of things that merely look +# roughly rectangular. Set OBSTACLE_EXPECTED_SIZE = (long, short) +# in screen pixels to pin the size instead of learning it (the +# learned size is printed to the console once it has settled). +OBSTACLE_SIZE_TOLERANCE = 0.22 +OBSTACLE_EXPECTED_SIZE = None + +# Two or three identical stickers glued edge to edge form one +# bigger rectangle; accept whole multiples of the sticker size. +# (For physics a merged rectangle behaves exactly like the parts.) +OBSTACLE_ALLOW_MERGED_STICKERS = True + +# Draw every sticker at exactly the learned size when the measured +# size is within ~10% of it, so identical stickers give identical +# obstacles instead of wobbling by a pixel or two. +OBSTACLE_SNAP_TO_EXPECTED_SIZE = True + +# Forget the learned size after this many seconds with no stickers +# at all, so a different sticker size can be used later. +OBSTACLE_SIZE_FORGET_SECONDS = 30.0 + +# Tracks that are already established are kept with slightly +# relaxed rules (this fraction of the strict thresholds) so a real +# sticker never flickers in and out on borderline frames, while NEW +# stickers still have to pass the full strict tests. +OBSTACLE_RELAXED_FACTOR = 0.90 + +# --- Following stickers over time (each one tracked separately) --- +# Consecutive detections needed before a NEW sticker counts. +OBSTACLE_REQUIRED_STABLE_FRAMES = 2 + +# An established sticker survives this many passes without being +# seen (someone's hand in the way). At 0.25 s per pass, 12 = 3 s. +OBSTACLE_MAX_MISSING_FRAMES = 12 + +# After this many passes without seeing an established sticker, the +# game stops PAINTING its white patch (and, a moment later, stops +# blanking that area for the detector). Without this, a sticker that +# was moved would be hidden by its own old painted patch until it +# timed out. If it was moved, it is found at the new spot within a +# pass or two; if it was removed, it is dropped at +# OBSTACLE_MAX_MISSING_FRAMES as before. Physics keeps using it +# either way until then. +OBSTACLE_STALE_MISSES = 3 + +# Not-yet-confirmed candidates are dropped after this many misses. +OBSTACLE_PENDING_MAX_MISSING_FRAMES = 1 + +# Max distance (screen px) a sticker may move between two passes +# and still be "the same" sticker. +OBSTACLE_TEMPORAL_MATCH_DISTANCE = 80 + +# Kernel sizes (screen pixels) for cleaning up the blob mask. +OBSTACLE_MASK_OPEN_KERNEL = 3 +OBSTACLE_MASK_CLOSE_KERNEL = 5 diff --git a/detection.py b/detection.py new file mode 100644 index 0000000..13f3498 --- /dev/null +++ b/detection.py @@ -0,0 +1,1132 @@ +# detection.py +# +# Finds the white paper stickers stuck on the monitor, as ROTATED +# rectangles, in a stable, flicker-free way. +# +# Input: a BGR image of the monitor already warped into screen +# space (Calibration.transform_frame) at DETECTION_SCALE of the real +# screen resolution. Output: one dict per sticker, in real SCREEN +# pixels (see geometry.py for the fields/angle convention). +# +# How it decides, in order (each step exists because a real +# exhibition hall breaks the simpler version): +# +# 1. Find generous "maybe" blobs: anything LOCALLY brighter than +# its own immediate surroundings and not very colored, inside +# the calibration border (see imaging.py - brightness and color +# are measured in CIELAB, not HSV, specifically because that +# separates the two cleanly). "Locally brighter" rather than +# "brighter than one fixed number for the whole frame" is what +# lets a sticker sitting in a dimmer part of the hall still be +# found - a global floor would simply never see it, no matter +# how good every later step is. The area the game itself +# painted white for already-detected stickers is blanked out +# first, otherwise a removed sticker would be "seen" forever in +# its own painted white (see DETECTED_OBSTACLE_PAINT_*), and any +# small hole a glare reflection punches through a blob (or a +# thin glare streak that splits it in two) is patched before +# measuring it (see imaging.fill_holes). +# +# 2. Re-draw each blob's edge at the half-way lightness between +# that blob's own paper brightness and the dark screen beside +# it. A dim sticker in a shadowy corner and an identical one +# under a spotlight then come out the SAME size - a fixed +# brightness cutoff can't do that, and it is what made +# identical stickers seem to be "detected by force" some of the +# time and not others. +# +# 3. Measure the refined blob: best-fit rotated rectangle (true +# side lengths and angle), how well it fills that rectangle +# (rejects round glows), convexity, uniformity of brightness +# (rejects hot-spot reflections), average saturation (rejects +# skin/colored objects), and whether the ring of pixels just +# outside it is dark (a sticker sits on a black screen). +# +# 4. All stickers are identical, so once a couple have been seen +# the size is learned and a blob of a different size is +# rejected. Two or three stickers glued edge to edge form one +# bigger rectangle and are accepted as whole multiples. +# +# 5. Every sticker is tracked separately over time. NEW stickers +# must pass the strict tests on consecutive passes; stickers +# already established are kept under slightly relaxed tests, so +# a real sticker doesn't flicker on borderline frames, and they +# survive a hand in front of them for a few seconds. Motion is +# smoothed with a dead-band (no jitter when nothing moves) and +# catches up fast when a sticker really is moved. + +import math +import time +from collections import deque + +import cv2 +import numpy as np + +import imaging + +from geometry import ( + normalize_angle, + blend_angle, + obb_corners, + extract_obb, + paint_dimensions, +) + +from config import ( + SCREEN_WIDTH, + SCREEN_HEIGHT, + CALIBRATION_OUTER_BORDER_SIZE, + CALIBRATION_INNER_BORDER_SIZE, + DETECTION_SCALE, + MIN_OBSTACLE_AREA, + MAX_OBSTACLE_AREA, + OBSTACLE_ADAPTIVE_OFFSET, + OBSTACLE_MIN_ABSOLUTE_LIGHTNESS, + OBSTACLE_ADAPTIVE_BLOCK_SPAN, + OBSTACLE_ADAPTIVE_DEFAULT_SHORT_SIDE, + OBSTACLE_COARSE_CHROMA_MAX, + OBSTACLE_EDGE_LEVEL, + MIN_OBSTACLE_BRIGHTNESS, + MAX_OBSTACLE_CHROMA, + MIN_OBSTACLE_RECTANGULARITY, + MIN_OBSTACLE_SOLIDITY, + MAX_OBSTACLE_ASPECT_RATIO, + MAX_OBSTACLE_BRIGHTNESS_STD, + MIN_OBSTACLE_DARK_SURROUND_RATIO, + OBSTACLE_BORDER_MASK_MARGIN, + OBSTACLE_EDGE_MARGIN, + OBSTACLE_SIZE_TOLERANCE, + OBSTACLE_EXPECTED_SIZE, + OBSTACLE_ALLOW_MERGED_STICKERS, + OBSTACLE_SNAP_TO_EXPECTED_SIZE, + OBSTACLE_SIZE_FORGET_SECONDS, + OBSTACLE_RELAXED_FACTOR, + OBSTACLE_REQUIRED_STABLE_FRAMES, + OBSTACLE_MAX_MISSING_FRAMES, + OBSTACLE_PENDING_MAX_MISSING_FRAMES, + OBSTACLE_STALE_MISSES, + OBSTACLE_TEMPORAL_MATCH_DISTANCE, + OBSTACLE_MASK_OPEN_KERNEL, + OBSTACLE_MASK_CLOSE_KERNEL, + SHOW_DETECTED_OBSTACLES, + DETECTED_OBSTACLE_PAINT_INSET_RATIO, +) + + +# Extra margin (screen px) added around the painted patch when it is +# blanked out, to absorb the small misalignment between the pixels +# we painted and where the camera sees them. +_BLANK_MARGIN = 5.0 + +# Below this long/short ratio a sticker is "square enough" that its +# orientation only matters modulo 90 degrees. +_SQUARE_ASPECT = 1.15 + + +class _Track: + """One physical sticker followed over time.""" + + __slots__ = ( + "id", + "cx", + "cy", + "angle", + "long", + "short", + "hits", + "misses", + "confirmed", + ) + + def __init__(self, track_id, candidate): + self.id = track_id + self.cx = candidate["cx"] + self.cy = candidate["cy"] + self.angle = candidate["angle"] + self.long = candidate["long"] + self.short = candidate["short"] + self.hits = 1 + self.misses = 0 + self.confirmed = False + + +class Detector: + def __init__(self): + self.scale = float(DETECTION_SCALE) + + self.det_width = int(round(SCREEN_WIDTH * self.scale)) + self.det_height = int(round(SCREEN_HEIGHT * self.scale)) + + self._s2 = self.scale * self.scale + + # Kernels, sized in screen pixels then scaled (always odd). + self._open_kernel = self._kernel(OBSTACLE_MASK_OPEN_KERNEL) + self._close_kernel = self._kernel(OBSTACLE_MASK_CLOSE_KERNEL) + self._small_kernel = np.ones((3, 3), np.uint8) + + # Region where stickers are searched: everything inside the + # calibration border (which is bright by design and would + # merge with a sticker touching it). + border = int( + round( + ( + CALIBRATION_OUTER_BORDER_SIZE + + CALIBRATION_INNER_BORDER_SIZE + + OBSTACLE_BORDER_MASK_MARGIN + ) + * self.scale + ) + ) + + self._roi = np.zeros( + (self.det_height, self.det_width), np.uint8 + ) + self._roi[ + border : self.det_height - border, + border : self.det_width - border, + ] = 255 + + margin = int(round(OBSTACLE_EDGE_MARGIN * self.scale)) + + self._roi_bounds = ( + border + margin, + border + margin, + self.det_width - border - margin, + self.det_height - border - margin, + ) + + self._pinned_size = None + + if OBSTACLE_EXPECTED_SIZE: + long_side, short_side = OBSTACLE_EXPECTED_SIZE + self._pinned_size = ( + float(max(long_side, short_side)), + float(min(long_side, short_side)), + ) + + # What "white" currently looks like to the camera, as a LAB + # (a*, b*) point - see Calibration.white_reference(). Starts + # at neutral (128, 128) and is kept updated by whoever drives + # this Detector (vision.py); NOT reset by reset() below, + # since it describes the room's current lighting, not + # anything about where the stickers are. + self._white_a = 128.0 + self._white_b = 128.0 + + self.reset() + + # ======================================================== + # Public API + # ======================================================== + + def set_white_reference(self, reference_ab): + """ + Update what "white" currently looks like to the camera (see + Calibration.white_reference()). Falls back to LAB neutral + (128, 128) if None (not measured yet, or adaptive hue/white + learning is disabled) - the same as the old fixed behavior. + """ + if reference_ab is None: + self._white_a = 128.0 + self._white_b = 128.0 + else: + self._white_a, self._white_b = reference_ab + + def reset(self): + self.tracks = [] + self._next_id = 1 + + self._samples = deque(maxlen=30) + self._expected = self._pinned_size + self._established = self._pinned_size is not None + self._last_size_log = None + + self._last_seen_any = time.monotonic() + + self.obstacles = [] + self._signature = None + + # Last pass's raw candidates, for debugging/tuning. + self.last_candidates = [] + + def detect(self, frame): + """ + Run one detection pass on a screen-space frame at + DETECTION_SCALE and return the current list of confirmed + stickers (screen pixels). The SAME list object is returned + while nothing changed, so consumers can cheaply notice + changes with an identity check. + """ + if ( + frame is None + or frame.ndim != 3 + or frame.shape[0] != self.det_height + or frame.shape[1] != self.det_width + ): + return self.obstacles + + candidates = self._find_candidates(frame) + + self.last_candidates = candidates + + self._update_tracks(candidates) + + self._forget_stale_size() + + self._publish() + + return self.obstacles + + # ======================================================== + # 1. Coarse blobs + # ======================================================== + + def _find_candidates(self, frame): + lightness, a_channel, b_channel = imaging.to_lab(frame) + + lightness = cv2.GaussianBlur(lightness, (3, 3), 0) + + chroma = np.clip( + imaging.chroma( + a_channel, + b_channel, + self._white_a, + self._white_b, + ), + 0, + 255, + ).astype(np.uint8) + + # "Locally brighter than its own neighborhood" (not "above + # one fixed brightness for the whole frame") - see the + # module docstring and config.py's OBSTACLE_ADAPTIVE_* for + # why. A plain floor is still kept underneath it purely as a + # noise guard: without it, sensor noise inside a totally + # dark, sticker-free patch of screen can register as + # "locally brighter than its (equally dark) surroundings" + # even though nothing is actually there. + local = cv2.adaptiveThreshold( + lightness, + 255, + cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + cv2.THRESH_BINARY, + self._adaptive_block_size(), + -OBSTACLE_ADAPTIVE_OFFSET, + ) + + mask = local & cv2.inRange( + lightness, + OBSTACLE_MIN_ABSOLUTE_LIGHTNESS, + 255, + ) + + mask &= cv2.inRange( + chroma, + 0, + OBSTACLE_COARSE_CHROMA_MAX, + ) + + mask &= self._roi + + blank = self._painted_blank_mask() + + if blank is not None: + mask[blank > 0] = 0 + + mask = cv2.morphologyEx( + mask, cv2.MORPH_OPEN, self._open_kernel + ) + mask = cv2.morphologyEx( + mask, cv2.MORPH_CLOSE, self._close_kernel + ) + + # Patches a glare highlight punched through the middle of a + # sticker, or a thin specular streak that splits one blob + # into two, without touching a real gap (e.g. a ball or a + # hand actually covering part of it), which stays open to + # the mask's edge and so is left alone. + mask = imaging.fill_holes(mask) + + contours, _ = cv2.findContours( + mask, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_SIMPLE, + ) + + candidates = [] + + for contour in contours: + candidate = self._analyze( + contour, + lightness, + chroma, + blank, + ) + + if candidate is not None: + candidates.append(candidate) + + return candidates + + def _adaptive_block_size(self): + """ + Neighborhood size (detection-scale px, always odd) for the + local brightness threshold above. Tied to the sticker's own + short side (once known) so the neighborhood sampled around a + sticker is mostly black screen rather than the sticker's own + bright pixels, which would otherwise drag the local average + up and make the sticker fail to stand out against itself. + """ + if self._expected is not None: + short_side = min(self._expected) + else: + short_side = OBSTACLE_ADAPTIVE_DEFAULT_SHORT_SIDE + + size = int( + round( + short_side + * self.scale + * OBSTACLE_ADAPTIVE_BLOCK_SPAN + ) + ) + + size = max(15, size) + + if size % 2 == 0: + size += 1 + + return size + + def _painted_blank_mask(self): + """ + Mask of the exact patches the game painted white for the + stickers it is currently tracking. Those pixels are ignored + so that a REMOVED sticker's leftover painted white can never + be mistaken for a sticker (the camera is looking at the same + screen the game paints on). + """ + if not SHOW_DETECTED_OBSTACLES: + return None + + # Keep blanking for two passes after painting stops, to + # cover the camera's own latency (it still shows the old + # painted patch for a moment). + confirmed = [ + t + for t in self.tracks + if t.confirmed + and t.misses < OBSTACLE_STALE_MISSES + 2 + ] + + if not confirmed: + return None + + blank = np.zeros( + (self.det_height, self.det_width), np.uint8 + ) + + for track in confirmed: + long_side, short_side = paint_dimensions( + track.long, + track.short, + DETECTED_OBSTACLE_PAINT_INSET_RATIO, + ) + + corners = obb_corners( + track.cx * self.scale, + track.cy * self.scale, + (long_side + 2 * _BLANK_MARGIN) * self.scale, + (short_side + 2 * _BLANK_MARGIN) * self.scale, + track.angle, + ) + + cv2.fillConvexPoly( + blank, + np.round(corners).astype(np.int32), + 255, + ) + + return blank + + # ======================================================== + # 2 + 3. Refine one blob and measure it + # ======================================================== + + def _analyze(self, contour, lightness, chroma, blank): + area_px = cv2.contourArea(contour) + + # Loose pre-gate on the coarse blob (the refined blob is + # gated exactly below). + loose_area = area_px / self._s2 + + if ( + loose_area < MIN_OBSTACLE_AREA * 0.5 + or loose_area > MAX_OBSTACLE_AREA * 1.5 + ): + return None + + x, y, w, h = cv2.boundingRect(contour) + + ring = max(3, int(round(0.15 * min(w, h)))) + pad = ring + 3 + + x0 = max(0, x - pad) + y0 = max(0, y - pad) + x1 = min(self.det_width, x + w + pad) + y1 = min(self.det_height, y + h + pad) + + crop_light = lightness[y0:y1, x0:x1] + crop_chroma = chroma[y0:y1, x0:x1] + crop_roi = self._roi[y0:y1, x0:x1] + + crop_blank = ( + blank[y0:y1, x0:x1] if blank is not None else None + ) + + shifted = contour - np.array([[[x0, y0]]], np.int32) + + coarse_fill = np.zeros(crop_light.shape, np.uint8) + + cv2.drawContours(coarse_fill, [shifted], -1, 255, -1) + + inside = crop_light[coarse_fill > 0] + outside = crop_light[coarse_fill == 0] + + if inside.size == 0 or outside.size < 8: + return None + + plateau = float(np.percentile(inside, 75)) + background = float(np.percentile(outside, 10)) + + # 2. Half-way brightness between this blob's own paper level + # and the dark screen beside it. + level = max( + background + + OBSTACLE_EDGE_LEVEL * (plateau - background), + float(OBSTACLE_MIN_ABSOLUTE_LIGHTNESS), + ) + + refined = cv2.inRange(crop_light, level, 255) + refined &= cv2.inRange( + crop_chroma, 0, OBSTACLE_COARSE_CHROMA_MAX + ) + refined &= crop_roi + + if crop_blank is not None: + refined[crop_blank > 0] = 0 + + refined = cv2.morphologyEx( + refined, cv2.MORPH_OPEN, self._small_kernel + ) + refined = cv2.morphologyEx( + refined, cv2.MORPH_CLOSE, self._small_kernel + ) + + found, _ = cv2.findContours( + refined, + cv2.RETR_EXTERNAL, + cv2.CHAIN_APPROX_SIMPLE, + ) + + if not found: + return None + + contour2 = max(found, key=cv2.contourArea) + + area2_px = cv2.contourArea(contour2) + area = area2_px / self._s2 + + if area < MIN_OBSTACLE_AREA or area > MAX_OBSTACLE_AREA: + return None + + # ---- 3. measurements --------------------------------- + rect = cv2.minAreaRect(contour2) + + (rect_cx, rect_cy), (rect_w, rect_h), _raw_angle = rect + + if rect_w <= 0 or rect_h <= 0: + return None + + rectangularity = area2_px / (rect_w * rect_h) + + # minAreaRect's own angle convention differs between + # OpenCV versions and jumps for near-square shapes, so the + # angle is measured from the box's actual long edge. + box = cv2.boxPoints(rect) + + edge_a = box[1] - box[0] + edge_b = box[2] - box[1] + + length_a = float(np.hypot(edge_a[0], edge_a[1])) + length_b = float(np.hypot(edge_b[0], edge_b[1])) + + if length_a <= 0 or length_b <= 0: + return None + + if length_a >= length_b: + long_edge, long_px, short_px = edge_a, length_a, length_b + else: + long_edge, long_px, short_px = edge_b, length_b, length_a + + aspect = long_px / short_px + + angle = math.degrees( + math.atan2(long_edge[1], long_edge[0]) + ) + + angle = normalize_angle( + angle, + 90.0 if aspect < _SQUARE_ASPECT else 180.0, + ) + + hull_area = cv2.contourArea(cv2.convexHull(contour2)) + + if hull_area <= 0: + return None + + solidity = area2_px / hull_area + + fill = np.zeros(crop_light.shape, np.uint8) + + cv2.drawContours(fill, [contour2], -1, 255, -1) + + inner = cv2.erode(fill, self._small_kernel) + + if not inner.any(): + inner = fill + + mean_light = float(cv2.mean(crop_light, mask=inner)[0]) + mean_chroma = float(cv2.mean(crop_chroma, mask=inner)[0]) + + _, std = cv2.meanStdDev(crop_light, mask=inner) + + light_std = float(std[0][0]) + + # Ring just outside the blob must be dark (black screen). + ring_kernel = np.ones( + (2 * ring + 1, 2 * ring + 1), np.uint8 + ) + + ring_mask = cv2.dilate(fill, ring_kernel) & ~fill + + ring_mask &= crop_roi + + if crop_blank is not None: + ring_mask[crop_blank > 0] = 0 + + ring_count = int(np.count_nonzero(ring_mask)) + + if ring_count >= 12: + dark = crop_light[ring_mask > 0] < (mean_light * 0.5) + + dark_ratio = float(np.count_nonzero(dark)) / ring_count + else: + dark_ratio = 1.0 + + # Touching the edge of the search area means the sticker is + # (or may be) cut off - its true size is unknown. + bx, by, bw, bh = cv2.boundingRect(contour2) + + left, top, right, bottom = self._roi_bounds + + clipped = ( + bx + x0 <= left + or by + y0 <= top + or bx + x0 + bw >= right + or by + y0 + bh >= bottom + ) + + # ---- back to real screen pixels ---------------------- + inverse = 1.0 / self.scale + + candidate = { + "cx": (rect_cx + x0) * inverse, + "cy": (rect_cy + y0) * inverse, + "angle": float(angle), + "long": long_px * inverse, + "short": short_px * inverse, + "area": float(area), + "aspect": float(aspect), + "rectangularity": float(rectangularity), + "solidity": float(solidity), + "brightness": mean_light, + "chroma": mean_chroma, + "std": light_std, + "dark_ratio": dark_ratio, + "clipped": clipped, + } + + candidate["strict_ok"], candidate["snap"] = self._passes( + candidate, 1.0 + ) + + candidate["relaxed_ok"], relaxed_snap = self._passes( + candidate, OBSTACLE_RELAXED_FACTOR + ) + + candidate["snap"] = candidate["snap"] or relaxed_snap + + return candidate + + # ======================================================== + # Sticker tests (strict for new stickers, relaxed to keep + # established ones) + # ======================================================== + + def _passes(self, c, factor): + if c["clipped"]: + return False, False + + if c["rectangularity"] < MIN_OBSTACLE_RECTANGULARITY * factor: + return False, False + + if c["solidity"] < MIN_OBSTACLE_SOLIDITY * factor: + return False, False + + if c["aspect"] > MAX_OBSTACLE_ASPECT_RATIO / factor: + return False, False + + if c["brightness"] < MIN_OBSTACLE_BRIGHTNESS * factor: + return False, False + + if c["chroma"] > MAX_OBSTACLE_CHROMA / factor: + return False, False + + if c["std"] > MAX_OBSTACLE_BRIGHTNESS_STD / factor: + return False, False + + if ( + c["dark_ratio"] + < MIN_OBSTACLE_DARK_SURROUND_RATIO * factor + ): + return False, False + + return self._size_matches( + c["long"], + c["short"], + OBSTACLE_SIZE_TOLERANCE / factor, + ) + + def _size_matches(self, long_side, short_side, tolerance): + """ + (matches, is_single_sticker_size). With no learned size yet + anything passes; with a size learned from just one sighting + the check is deliberately loose (that one sighting could + itself have been a false positive). + """ + if self._expected is None: + return True, False + + expected_long, expected_short = self._expected + + if not self._established: + tolerance = max(tolerance, 0.45) + + limit = 3 if OBSTACLE_ALLOW_MERGED_STICKERS else 1 + + for along in range(1, limit + 1): + for across in range(1, limit + 1): + if along * across > 6: + continue + + target_a = along * expected_long + target_b = across * expected_short + + target_long = max(target_a, target_b) + target_short = min(target_a, target_b) + + if ( + abs(long_side - target_long) / target_long + <= tolerance + and abs(short_side - target_short) + / target_short + <= tolerance + ): + single = along == 1 and across == 1 + + return True, single + + return False, False + + # ======================================================== + # Learned sticker size + # ======================================================== + + def _learn_size(self, long_side, short_side): + if self._pinned_size is not None: + return + + self._samples.append((long_side, short_side)) + + best = [] + + for a_long, a_short in self._samples: + members = [ + (b_long, b_short) + for b_long, b_short in self._samples + if abs(b_long - a_long) / a_long <= 0.15 + and abs(b_short - a_short) / a_short <= 0.15 + ] + + if len(members) > len(best): + best = members + + self._expected = ( + float(np.median([m[0] for m in best])), + float(np.median([m[1] for m in best])), + ) + + self._established = len(best) >= 2 + + if self._established: + rounded = ( + round(self._expected[0]), + round(self._expected[1]), + ) + + if ( + self._last_size_log is None + or abs(rounded[0] - self._last_size_log[0]) > 4 + or abs(rounded[1] - self._last_size_log[1]) > 4 + ): + self._last_size_log = rounded + + print( + "[DETECT] sticker size learned: about " + f"{rounded[0]} x {rounded[1]} px. To pin it, set " + f"OBSTACLE_EXPECTED_SIZE = ({rounded[0]}, " + f"{rounded[1]}) in config.py." + ) + + def _forget_stale_size(self): + if self._pinned_size is not None: + return + + now = time.monotonic() + + if any(t.confirmed for t in self.tracks): + self._last_seen_any = now + return + + if ( + self._expected is not None + and now - self._last_seen_any + > OBSTACLE_SIZE_FORGET_SECONDS + ): + self._samples.clear() + self._expected = None + self._established = False + self._last_size_log = None + + # ======================================================== + # 5. Tracking + # ======================================================== + + def _update_tracks(self, candidates): + pairs = [] + + for track_index, track in enumerate(self.tracks): + gate = min( + OBSTACLE_TEMPORAL_MATCH_DISTANCE, + max(25.0, 0.75 * track.short), + ) + + for cand_index, candidate in enumerate(candidates): + if not candidate["relaxed_ok"]: + continue + + distance = math.hypot( + candidate["cx"] - track.cx, + candidate["cy"] - track.cy, + ) + + if distance <= gate: + pairs.append((distance, track_index, cand_index)) + + pairs.sort() + + used_tracks = set() + used_candidates = set() + + for _distance, track_index, cand_index in pairs: + if ( + track_index in used_tracks + or cand_index in used_candidates + ): + continue + + used_tracks.add(track_index) + used_candidates.add(cand_index) + + self._apply_hit( + self.tracks[track_index], + candidates[cand_index], + ) + + survivors = [] + + for track_index, track in enumerate(self.tracks): + if track_index not in used_tracks: + track.misses += 1 + + limit = ( + OBSTACLE_MAX_MISSING_FRAMES + if track.confirmed + else OBSTACLE_PENDING_MAX_MISSING_FRAMES + ) + + if track.misses > limit: + continue + + survivors.append(track) + + self.tracks = survivors + + # New stickers: strict tests only, and not on top of a + # sticker we already follow. + for cand_index, candidate in enumerate(candidates): + if ( + cand_index in used_candidates + or not candidate["strict_ok"] + ): + continue + + overlaps = any( + math.hypot( + candidate["cx"] - t.cx, + candidate["cy"] - t.cy, + ) + < 0.5 * min(t.short, candidate["short"]) + for t in self.tracks + ) + + if overlaps: + continue + + track = _Track(self._next_id, candidate) + + self._next_id += 1 + + self.tracks.append(track) + + self._maybe_confirm(track, candidate) + + def _apply_hit(self, track, candidate): + track.misses = 0 + track.hits += 1 + + long_side = candidate["long"] + short_side = candidate["short"] + + if ( + OBSTACLE_SNAP_TO_EXPECTED_SIZE + and self._established + and self._expected is not None + and candidate["snap"] + ): + expected_long, expected_short = self._expected + + if ( + abs(long_side - expected_long) / expected_long <= 0.10 + and abs(short_side - expected_short) + / expected_short + <= 0.10 + ): + long_side, short_side = expected_long, expected_short + + # Position: dead-band while nothing moves (no jitter for a + # ball resting on the sticker), fast catch-up when it does. + distance = math.hypot( + candidate["cx"] - track.cx, + candidate["cy"] - track.cy, + ) + + if distance > 0.8: + alpha = min(1.0, max(0.25, distance / 8.0)) + + track.cx += (candidate["cx"] - track.cx) * alpha + track.cy += (candidate["cy"] - track.cy) * alpha + + period = ( + 90.0 + if track.long / max(track.short, 1e-6) < _SQUARE_ASPECT + else 180.0 + ) + + half = period / 2.0 + + angle_diff = abs( + ((candidate["angle"] - track.angle + half) % period) + - half + ) + + if angle_diff > 0.6: + alpha = min(1.0, max(0.25, angle_diff / 4.0)) + + track.angle = blend_angle( + track.angle, + candidate["angle"], + alpha, + period, + ) + + track.long += (long_side - track.long) * 0.35 + track.short += (short_side - track.short) * 0.35 + + self._maybe_confirm(track, candidate) + + def _maybe_confirm(self, track, candidate): + if ( + track.confirmed + or track.hits < OBSTACLE_REQUIRED_STABLE_FRAMES + ): + return + + track.confirmed = True + + # Only single-sticker sightings teach the size (a merged + # pair would otherwise be learned as "the" sticker size). + if candidate["snap"] or self._expected is None: + self._learn_size(candidate["long"], candidate["short"]) + + # ======================================================== + # Output + # ======================================================== + + def _publish(self): + confirmed = [t for t in self.tracks if t.confirmed] + + signature = tuple( + ( + t.id, + round(t.cx, 1), + round(t.cy, 1), + round(t.angle, 1), + round(t.long, 1), + round(t.short, 1), + t.misses < OBSTACLE_STALE_MISSES, + ) + for t in confirmed + ) + + if signature == self._signature: + return + + self._signature = signature + + obstacles = [] + + for track in confirmed: + corners = obb_corners( + track.cx, + track.cy, + track.long, + track.short, + track.angle, + ) + + xs = [p[0] for p in corners] + ys = [p[1] for p in corners] + + obstacles.append( + { + "id": track.id, + "center": (track.cx, track.cy), + "angle": track.angle, + "rect_long": track.long, + "rect_short": track.short, + # False once the sticker hasn't been seen for a + # few passes: rendering then stops painting it. + "painted": track.misses < OBSTACLE_STALE_MISSES, + "x": min(xs), + "y": min(ys), + "width": max(xs) - min(xs), + "height": max(ys) - min(ys), + } + ) + + self.obstacles = obstacles + + # ======================================================== + # Debug drawing + # ======================================================== + + def draw(self, frame, obstacles=None): + """ + Annotated copy of a screen-space detection frame: confirmed + stickers in green, rejected candidates in red (with the + first failing reason's numbers), for tuning on site. + """ + if obstacles is None: + obstacles = self.obstacles + + output = frame.copy() + + for candidate in self.last_candidates: + if candidate["relaxed_ok"]: + continue + + corners = obb_corners( + candidate["cx"], + candidate["cy"], + candidate["long"], + candidate["short"], + candidate["angle"], + ) + + points = ( + np.array(corners) * self.scale + ).astype(np.int32).reshape((-1, 1, 2)) + + cv2.polylines(output, [points], True, (0, 0, 255), 1) + + cv2.putText( + output, + f"r{candidate['rectangularity']:.2f} " + f"s{candidate['solidity']:.2f} " + f"d{candidate['dark_ratio']:.2f}", + (int(points[0][0][0]), int(points[0][0][1])), + cv2.FONT_HERSHEY_SIMPLEX, + 0.4, + (0, 0, 255), + 1, + cv2.LINE_AA, + ) + + for obstacle in obstacles: + obb = extract_obb(obstacle) + + if obb is None: + continue + + cx, cy, long_side, short_side, angle = obb + + corners = obb_corners(cx, cy, long_side, short_side, angle) + + points = ( + np.array(corners) * self.scale + ).astype(np.int32).reshape((-1, 1, 2)) + + cv2.polylines(output, [points], True, (0, 255, 0), 2) + + cv2.circle( + output, + (int(cx * self.scale), int(cy * self.scale)), + 4, + (0, 255, 0), + -1, + ) + + return output + + # ======================================================== + # Helpers + # ======================================================== + + def _kernel(self, size_screen_px): + size = max(1, int(round(size_screen_px * self.scale))) + + if size % 2 == 0: + size += 1 + + return np.ones((size, size), np.uint8) diff --git a/effects.py b/effects.py new file mode 100644 index 0000000..81dab97 --- /dev/null +++ b/effects.py @@ -0,0 +1,271 @@ +# effects.py +# +# Everything cosmetic that reacts to the simulation: comet tails, +# colour changes on bounce, squash-and-stretch, sparks and rings. +# Pure state + math (no pygame, no drawing) - rendering.py draws it. +# None of it ever feeds back into physics. +# +# Driven by two things only: each ball's own state (position, +# velocity) and the events physics.py emits (kind, x, y, normal_x, +# normal_y, speed, ball) with kind "obstacle", "ball" or "pop". +# +# Camera safety: rendering only ever draws these with saturated +# colours in small dim glows, so the camera looking at this same +# monitor never mistakes an effect for a white sticker. Set +# VISUAL_EFFECTS_ENABLED = False in config.py to switch it all off. + +import math +import random + +from config import ( + VISUAL_EFFECTS_ENABLED, + BALL_HUE_MIN_DEGREES, + BALL_HUE_MAX_DEGREES, + BALL_HUE_SHIFT_ON_BOUNCE, + BALL_TRAIL_LENGTH, + BALL_TRAIL_SAMPLE_SECONDS, + BALL_SQUASH_DURATION_SECONDS, + IMPACT_EFFECT_MIN_SPEED, + MAX_PARTICLES, +) + + +_PARTICLE_GRAVITY = 520.0 +_PARTICLE_DRAG = 2.2 +_MAX_RINGS = 24 + +# Impact speed (px/s) at which effects reach full strength. +_FULL_STRENGTH_SPEED = 1400.0 + + +class Effects: + def __init__(self): + # Sparks: [x, y, vx, vy, age, life, hue, size] + self.particles = [] + + # Expanding rings: [x, y, age, life, hue, max_radius] + self.rings = [] + + self._random = random.Random() + + def clear(self): + self.particles = [] + self.rings = [] + + # ======================================================== + # Update + # ======================================================== + + def update(self, delta_time, balls, events): + if not VISUAL_EFFECTS_ENABLED: + return + + delta_time = max(0.0, min(delta_time, 0.05)) + + for event in events: + self._on_event(event) + + for ball in balls: + self._update_ball(ball, delta_time) + + self._update_particles(delta_time) + self._update_rings(delta_time) + + # ======================================================== + # Per-ball state + # ======================================================== + + def _update_ball(self, ball, dt): + if ball.squash > 0.0: + ball.squash = max( + 0.0, + ball.squash + - dt / max(BALL_SQUASH_DURATION_SECONDS, 1e-3), + ) + + if BALL_TRAIL_LENGTH <= 0: + ball.trail = [] + return + + ball.trail_timer += dt + + # More than one sample can be due on a slow frame; keep the + # tail evenly spaced in TIME regardless of frame rate. + while ball.trail_timer >= BALL_TRAIL_SAMPLE_SECONDS: + ball.trail_timer -= BALL_TRAIL_SAMPLE_SECONDS + + if ball.dying: + # A popping ball's tail drains away. + if ball.trail: + ball.trail.pop(0) + else: + ball.trail.append((ball.x, ball.y)) + + excess = len(ball.trail) - BALL_TRAIL_LENGTH + + if excess > 0: + del ball.trail[:excess] + + @staticmethod + def _shift_hue(hue, amount): + span = BALL_HUE_MAX_DEGREES - BALL_HUE_MIN_DEGREES + + return ( + BALL_HUE_MIN_DEGREES + + (hue - BALL_HUE_MIN_DEGREES + amount) % span + ) + + # ======================================================== + # Events -> sparks, rings, squash, colour change + # ======================================================== + + def _on_event(self, event): + kind, x, y, nx, ny, speed, ball = event + + if kind == "pop": + self._burst(x, y, ball.hue, 16, 80.0, 260.0) + self._ring(x, y, ball.hue, ball.radius * 3.0, 0.45) + return + + if kind == "launch": + self._sparks(x, y, nx, ny, ball.hue, 12, 0.8) + self._ring(x, y, ball.hue, ball.radius * 2.2, 0.25) + return + + strength = min(1.0, speed / _FULL_STRENGTH_SPEED) + + ball.squash = 1.0 + ball.squash_nx = nx + ball.squash_ny = ny + + if kind == "obstacle": + ball.hue = self._shift_hue( + ball.hue, BALL_HUE_SHIFT_ON_BOUNCE + ) + + self._sparks( + x, + y, + nx, + ny, + ball.hue, + int(4 + 10 * strength), + strength, + ) + + self._ring( + x, + y, + ball.hue, + ball.radius * (1.6 + 1.6 * strength), + 0.32, + ) + else: + self._burst( + x, + y, + ball.hue, + int(3 + 6 * strength), + 60.0, + 140.0 + 200.0 * strength, + ) + + self._ring( + x, + y, + ball.hue, + ball.radius * (1.3 + 1.0 * strength), + 0.25, + ) + + def _sparks(self, x, y, nx, ny, hue, count, strength): + rng = self._random + + base = math.atan2(ny, nx) + + for _ in range(count): + angle = base + rng.uniform(-1.25, 1.25) + + speed = rng.uniform( + 140.0, + 260.0 + 420.0 * strength, + ) + + self._add_particle( + x, + y, + math.cos(angle) * speed, + math.sin(angle) * speed, + rng.uniform(0.35, 0.8), + hue + rng.uniform(-14.0, 14.0), + rng.uniform(2.5, 5.5), + ) + + def _burst(self, x, y, hue, count, speed_min, speed_max): + rng = self._random + + for _ in range(count): + angle = rng.uniform(0.0, 2.0 * math.pi) + speed = rng.uniform(speed_min, speed_max) + + self._add_particle( + x, + y, + math.cos(angle) * speed, + math.sin(angle) * speed, + rng.uniform(0.3, 0.7), + hue + rng.uniform(-18.0, 18.0), + rng.uniform(2.5, 5.0), + ) + + def _add_particle(self, x, y, vx, vy, life, hue, size): + if len(self.particles) >= MAX_PARTICLES: + # Drop the oldest so a busy moment never grows without + # bound. + del self.particles[0] + + self.particles.append( + [x, y, vx, vy, 0.0, life, hue, size] + ) + + def _ring(self, x, y, hue, max_radius, life): + if len(self.rings) >= _MAX_RINGS: + del self.rings[0] + + self.rings.append([x, y, 0.0, life, hue, max_radius]) + + # ======================================================== + # Ageing + # ======================================================== + + def _update_particles(self, dt): + alive = [] + + drag = max(0.0, 1.0 - _PARTICLE_DRAG * dt) + + for particle in self.particles: + particle[4] += dt + + if particle[4] >= particle[5]: + continue + + particle[3] += _PARTICLE_GRAVITY * dt + particle[2] *= drag + particle[3] *= drag + particle[0] += particle[2] * dt + particle[1] += particle[3] * dt + + alive.append(particle) + + self.particles = alive + + def _update_rings(self, dt): + alive = [] + + for ring in self.rings: + ring[2] += dt + + if ring[2] < ring[3]: + alive.append(ring) + + self.rings = alive diff --git a/geometry.py b/geometry.py new file mode 100644 index 0000000..39a139a --- /dev/null +++ b/geometry.py @@ -0,0 +1,255 @@ +# geometry.py +# +# Small, shared helpers for working with ROTATED rectangles (an +# "OBB" - oriented bounding box) in screen space: a rectangle +# stickers can be glued on crooked, so an obstacle's position is no +# longer just an axis-aligned (x, y, width, height) - it also has +# an angle. This module is the single place that knows how to: +# +# - turn a raw cv2.minAreaRect() result into a STABLE, canonical +# (center, long side, short side, angle) representation that +# doesn't jitter or jump between frames (see normalize_angle +# and detection.py for why that's needed), +# - blend/interpolate that angle smoothly across frames +# (blend_angle), +# - convert between world space and the rectangle's own local, +# axis-aligned frame (world_to_local / local_to_world) - this +# is what lets both collision detection AND collision response +# be written as plain axis-aligned math that "just works" for +# any rotation, by doing the rotation once at the boundary +# instead of re-deriving rotated formulas everywhere, +# - compute the 4 corners of a rectangle for drawing +# (obb_corners), +# - pull all of the above out of a detected-obstacle dict in one +# place, with one shared definition of what counts as a valid +# obstacle (extract_obb), so detection/physics/rendering can +# never disagree about it. +# +# Angle convention: an un-rotated rectangle (long side horizontal) +# has angle 0. Positive angles rotate the long side counter- +# clockwise in standard math coordinates - visually CLOCKWISE on +# screen, since screen y grows downward. Angles are always kept in +# the canonical range (-90, 90], because a rectangle looks +# identical after a 180-degree rotation (it has no "arrow" printed +# on it), so anything outside that range is a redundant duplicate +# of an angle already inside it. Collapsing to one canonical range +# is what keeps blend_angle's shortest-path interpolation well +# defined instead of it occasionally spinning the "wrong" way. + +import math + + +# ============================================================ +# Angle canonicalization +# ============================================================ + +def normalize_angle(angle_degrees, period=180.0): + """ + Fold an orientation angle into the canonical (-period/2, + period/2] range. + + A rectangle has 180-degree rotational symmetry, so any angle is + equivalent to one inside a 180-degree-wide window (period=180). + A (nearly) SQUARE sticker is even more symmetric - it looks the + same after every 90 degrees - so for those callers pass + period=90. Always folding into the SAME window is what makes + two angles describing the same physical orientation compare and + blend as equal instead of looking like they are far apart. + """ + half = period / 2.0 + + angle = angle_degrees % period + + if angle > half: + angle -= period + + return angle + + +def blend_angle(old_angle, new_angle, alpha, period=180.0): + """ + Move old_angle toward new_angle by fraction alpha along the + SHORTEST path, for orientations that repeat every `period` + degrees. + + A naive `old + alpha * (new - old)` breaks near the wrap-around + point: 89 degrees and -89 degrees are really only 2 degrees + apart for a rectangle, but the naive subtraction sees 178 and + blends the wrong way (and for a near-square sticker, averaging + across its 90-degree ambiguity would even produce a completely + wrong in-between angle). Wrapping the raw difference into + (-period/2, period/2] first avoids both. + """ + half = period / 2.0 + + diff = ((new_angle - old_angle + half) % period) - half + + return normalize_angle(old_angle + alpha * diff, period) + + +def paint_dimensions(rect_long, rect_short, inset_ratio): + """ + Size of the white patch painted on the monitor for a detected + sticker: the sticker trimmed by inset_ratio * short side on every + edge. Shared by rendering (to draw it) and detection (to blank + exactly that area out before looking for stickers) so the two can + never disagree about what was painted. + """ + inset = inset_ratio * rect_short + + return ( + max(1.0, rect_long - 2.0 * inset), + max(1.0, rect_short - 2.0 * inset), + ) + + +# ============================================================ +# World <-> local (rectangle-aligned) frame conversion +# ============================================================ + +def world_to_local(x, y, angle_degrees): + """ + Rotate a world-space vector (x, y) by -angle_degrees, i.e. into + the rectangle's own local frame, where its long side lies along + the local x-axis and its short side along the local y-axis. + + This takes a VECTOR, not a point - to transform a point, first + subtract the rectangle's center, call this, and the result is + the point's position relative to the rectangle in its own + frame. The same function (with no subtraction) also correctly + rotates direction-only quantities like velocity, since rotation + doesn't care about position. + """ + theta = math.radians(angle_degrees) + + cos_t = math.cos(theta) + sin_t = math.sin(theta) + + return ( + x * cos_t + y * sin_t, + -x * sin_t + y * cos_t, + ) + + +def local_to_world(x, y, angle_degrees): + """ + The exact inverse of world_to_local: rotate a local-frame + vector back by +angle_degrees into world space. + """ + theta = math.radians(angle_degrees) + + cos_t = math.cos(theta) + sin_t = math.sin(theta) + + return ( + x * cos_t - y * sin_t, + x * sin_t + y * cos_t, + ) + + +# ============================================================ +# Corners (for drawing / debugging) +# ============================================================ + +def obb_corners(center_x, center_y, rect_long, rect_short, angle_degrees): + """ + Return the 4 corners of a rotated rectangle, in order around + its perimeter, as world-space (x, y) tuples. Safe to hand + straight to pygame.draw.polygon or cv2.polylines. + """ + half_long = rect_long / 2.0 + half_short = rect_short / 2.0 + + local_corners = ( + (half_long, half_short), + (half_long, -half_short), + (-half_long, -half_short), + (-half_long, half_short), + ) + + corners = [] + + for local_x, local_y in local_corners: + world_dx, world_dy = local_to_world( + local_x, + local_y, + angle_degrees, + ) + + corners.append( + ( + center_x + world_dx, + center_y + world_dy, + ) + ) + + return corners + + +# ============================================================ +# Obstacle dict -> OBB extraction +# ============================================================ + +def extract_obb(obstacle): + """ + Pull (center_x, center_y, rect_long, rect_short, angle) out of + a detected-obstacle dict, or return None if the obstacle is + missing/malformed/degenerate. This is the ONE place that + decides what counts as a usable oriented rectangle, so physics, + rendering, and validity-checking can never disagree with each + other about it. + + Falls back to treating the obstacle as an axis-aligned + (angle=0) rectangle built from x/y/width/height if it has no + "center"/"rect_long"/"rect_short"/"angle" fields, so any code + that still hands in a plain axis-aligned obstacle dict keeps + working exactly as before. + """ + if not isinstance(obstacle, dict): + return None + + try: + if "center" in obstacle: + center_x, center_y = obstacle["center"] + else: + center_x = ( + obstacle["x"] + obstacle["width"] / 2.0 + ) + center_y = ( + obstacle["y"] + obstacle["height"] / 2.0 + ) + + rect_long = obstacle.get( + "rect_long", + obstacle.get("width"), + ) + rect_short = obstacle.get( + "rect_short", + obstacle.get("height"), + ) + angle = obstacle.get("angle", 0.0) + + center_x = float(center_x) + center_y = float(center_y) + rect_long = float(rect_long) + rect_short = float(rect_short) + angle = float(angle) + except (KeyError, TypeError, ValueError): + return None + + if not all( + math.isfinite(value) + for value in ( + center_x, + center_y, + rect_long, + rect_short, + angle, + ) + ): + return None + + if rect_long <= 0 or rect_short <= 0: + return None + + return center_x, center_y, rect_long, rect_short, angle diff --git a/imaging.py b/imaging.py new file mode 100644 index 0000000..37b9910 --- /dev/null +++ b/imaging.py @@ -0,0 +1,124 @@ +# imaging.py +# +# Small, shared OpenCV helpers for illumination-robust color work, +# used by both calibration.py (finding the monitor) and +# detection.py (finding the white stickers). +# +# Both work in CIELAB rather than HSV, for one concrete reason: LAB +# separates brightness (L) from true color (a, b) far more cleanly +# than HSV does. A glare highlight, a shadow, a dimmer corner of the +# hall, or the camera's own auto white-balance drifting under +# exhibition lighting - all of these move L a lot while barely +# moving a/b. That is exactly the property needed to answer "is +# this green?" or "is this a neutral white/gray?" in a way that +# stays true regardless of how bright, dim, or unevenly lit that +# particular pixel happens to be right now - which is the real +# source of "the camera doesn't quite see what the monitor is +# actually showing": most of that gap is a brightness/exposure +# effect, not a pure color-chemistry one, and LAB is built to shrug +# that off. + +import cv2 +import numpy as np + + +def to_lab(frame_bgr): + """ + (L, a, b), each a 2D uint8 array in OpenCV's 0-255 LAB + encoding: L is lightness (0=black, 255=white), a is the + green(low)-red(high) axis centered at 128, b is the + blue(low)-yellow(high) axis centered at 128. + """ + lab = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2LAB) + + return lab[:, :, 0], lab[:, :, 1], lab[:, :, 2] + + +def chroma(a_channel, b_channel, reference_a=128.0, reference_b=128.0): + """ + How colorful a pixel is, independent of how bright or dark it + is - near 0 at the reference point, larger for anything further + away from it in LAB's roughly perceptually-even a/b plane. + Unlike HSV saturation (a ratio that gets noisy and unstable for + dark pixels, since it is normalized by brightness itself), this + stays meaningful whether a pixel is in a bright spot or a + shadowed corner. + + The reference defaults to (128, 128) - LAB's own neutral gray - + but callers that have a measured "what does a KNOWN-white + surface look like to this camera right now" point (see + Calibration.white_reference) should pass it in instead. Under + colored ambient light even genuinely white paper is NOT neutral + gray to the camera - that is unavoidable physics, not a flaw in + the camera - so testing distance from true neutral would wrongly + reject it. Testing distance from "whatever white looks like + right now" sidesteps that entirely, the same way a photographer + custom white-balances off a gray card instead of trusting a + fixed assumption about what "white" means. + """ + da = a_channel.astype(np.float32) - reference_a + db = b_channel.astype(np.float32) - reference_b + + return np.sqrt(da ** 2 + db ** 2) + + +def green_dominance(a_channel): + """ + How far into "green" territory a pixel sits on LAB's red-green + axis, clipped to [0, 255]. Green is the LOW end of the a + channel (a < 128), so this is just how far below 128 the pixel + is. Using this one, roughly illumination-independent channel + instead of comparing raw R/G/B brightness is what lets the + calibration pattern still be found once the camera's own white + balance has shifted its recorded color away from what the + monitor is actually displaying. + """ + return np.clip( + 128 - a_channel.astype(np.int16), + 0, + 255, + ).astype(np.uint8) + + +def fill_holes(mask): + """ + Fills any fully-enclosed hole in a binary mask (0/255) - e.g. a + glare highlight punching a small gap in the middle of an + otherwise solid colored region, or a reflection splitting a + sticker's blob in two. Standard flood-fill-from-the-border + trick: anything NOT reachable from the mask's own border by + walking through zero pixels must be an enclosed hole, so it + gets filled in. A real gap that reaches all the way out to the + mask's edge (e.g. a person actually standing in front of part + of the pattern) stays untouched, since it IS reachable from the + border. + """ + height, width = mask.shape[:2] + + # Padding with a guaranteed-zero 1px border first means the + # flood-fill seed point is always background, even if the real + # mask happens to already be foreground right at image corner + # (0, 0) - otherwise floodFill would treat that corner's own + # color as the "background" to flood, filling nothing at all. + padded = cv2.copyMakeBorder( + mask, + 1, + 1, + 1, + 1, + cv2.BORDER_CONSTANT, + value=0, + ) + + flood = padded.copy() + + flood_fill_mask = np.zeros( + (height + 4, width + 4), + np.uint8, + ) + + cv2.floodFill(flood, flood_fill_mask, (0, 0), 255) + + flood = flood[1:-1, 1:-1] + + return mask | cv2.bitwise_not(flood) diff --git a/launcher.py b/launcher.py new file mode 100644 index 0000000..ae0fb19 --- /dev/null +++ b/launcher.py @@ -0,0 +1,138 @@ +# launcher.py +# +# Represents the top-center launcher tube. +# +# Features: +# - Pivot point attached at top-center of the screen (x = width / 2, y = -15), +# extending above the screen top so it is clipped by the screen boundary. +# - Consists ONLY of a short, elongated solid tube/barrel (no base/tank/turret). +# - Continuous smooth rotation between LAUNCHER_MIN_ANGLE and LAUNCHER_MAX_ANGLE. +# - Provides collision parameters for PhysicsWorld (_Solid OBB). +# - Provides ball spawn transform (position + velocity) outward from nozzle exit. + +import math + +from config import ( + LAUNCHER_MIN_ANGLE, + LAUNCHER_MAX_ANGLE, + LAUNCHER_LENGTH, + LAUNCHER_WIDTH, + LAUNCHER_ROTATION_SPEED, + LAUNCHER_BALL_SPAWN_SPEED, + BALL_RADIUS, + BALL_RADIUS_VARIATION, +) + + +class Launcher: + def __init__(self, screen_width, screen_height): + self.screen_width = float(screen_width) + self.screen_height = float(screen_height) + + # Pivot point at top center. Placed slightly above y=0 (-15px) so the upper part + # extends beyond the top of the screen and is clipped by the screen boundary. + self.pivot_x = self.screen_width / 2.0 + self.pivot_y = -15.0 + + self.min_angle = float(LAUNCHER_MIN_ANGLE) + self.max_angle = float(LAUNCHER_MAX_ANGLE) + self.length = float(LAUNCHER_LENGTH) + self.tube_width = float(LAUNCHER_WIDTH) + self.rotation_speed = float(LAUNCHER_ROTATION_SPEED) + self.spawn_speed = float(LAUNCHER_BALL_SPAWN_SPEED) + + self._time = 0.0 + self.angle = 0.0 # degrees from straight down (0=down, negative=left, positive=right) + self._update_angle() + + def update_screen_size(self, width, height): + self.screen_width = float(width) + self.screen_height = float(height) + self.pivot_x = self.screen_width / 2.0 + + def update(self, delta_time): + self._time += delta_time + self._update_angle() + + def _update_angle(self): + mid_angle = (self.min_angle + self.max_angle) / 2.0 + amplitude = (self.max_angle - self.min_angle) / 2.0 + + if amplitude <= 0: + self.angle = mid_angle + return + + # Smooth harmonic oscillation within [min_angle, max_angle] + period = max(0.1, (4.0 * amplitude) / max(1.0, self.rotation_speed)) + omega = (2.0 * math.pi) / period + + sine_val = math.sin(omega * self._time) + subtle_wave = 0.85 * sine_val + 0.15 * math.sin(omega * 0.5 * self._time) + + self.angle = mid_angle + amplitude * max(-1.0, min(1.0, subtle_wave)) + + def get_solid_params(self): + """ + Returns (cx, cy, rect_long, rect_short, angle) for creating a _Solid in physics. + """ + rad = math.radians(self.angle) + dir_x = math.sin(rad) + dir_y = math.cos(rad) + + cx = self.pivot_x + dir_x * (self.length / 2.0) + cy = self.pivot_y + dir_y * (self.length / 2.0) + + # OBB angle in geometry convention: 90 degrees minus launcher angle + obb_angle = 90.0 - self.angle + + return cx, cy, self.length, self.tube_width, obb_angle + + def get_spawn_transform(self): + """ + Returns (spawn_x, spawn_y, dir_x, dir_y) for spawning a ball outward + along the nozzle direction, positioned slightly outside the nozzle tip. + """ + rad = math.radians(self.angle) + dir_x = math.sin(rad) + dir_y = math.cos(rad) + + max_ball_radius = BALL_RADIUS * (1.0 + BALL_RADIUS_VARIATION) + safe_dist = self.length + max_ball_radius + 3.0 + + spawn_x = self.pivot_x + dir_x * safe_dist + spawn_y = self.pivot_y + dir_y * safe_dist + + return spawn_x, spawn_y, dir_x, dir_y + + def get_points(self): + """ + Returns the 4 corner points of the launcher tube rectangle. + """ + rad = math.radians(self.angle) + u_x = math.sin(rad) + u_y = math.cos(rad) + + v_x = -u_y + v_y = u_x + + half_w = self.tube_width / 2.0 + length = self.length + + p_top_left = ( + self.pivot_x - v_x * half_w, + self.pivot_y - v_y * half_w, + ) + p_top_right = ( + self.pivot_x + v_x * half_w, + self.pivot_y + v_y * half_w, + ) + p_bot_right = ( + self.pivot_x + u_x * length + v_x * half_w, + self.pivot_y + u_y * length + v_y * half_w, + ) + p_bot_left = ( + self.pivot_x + u_x * length - v_x * half_w, + self.pivot_y + u_y * length - v_y * half_w, + ) + + return [p_top_left, p_top_right, p_bot_right, p_bot_left] diff --git a/main.py b/main.py new file mode 100644 index 0000000..c16bcbe --- /dev/null +++ b/main.py @@ -0,0 +1,274 @@ +# main.py + +import time + +import cv2 +import pygame + +from camera import Camera +from calibration import Calibration +from detection import Detector +from vision import Vision +from physics import PhysicsWorld +from effects import Effects +from rendering import Renderer + +from config import ( + CAMERA_URL, + SCREEN_WIDTH, + SCREEN_HEIGHT, + BALL_SPAWN_INTERVAL_SECONDS, +) + + +# If a spawn attempt fails (too many balls in play, or another ball +# is sitting right at the drop point), try again after this short +# delay instead of waiting a whole spawn interval. +_SPAWN_RETRY_SECONDS = 0.25 + +_WARNING_INTERVAL_SECONDS = 5.0 + + +class Game: + # ======================================================== + # Initialization + # ======================================================== + + def __init__(self): + # Connecting happens on the camera's own thread, so the + # window below comes up (and stays responsive to the + # keyboard) even if the phone isn't streaming yet. + self.camera = Camera(CAMERA_URL) + + self.calibration = Calibration() + self.detector = Detector() + + # Camera -> calibration -> sticker detection, all on a + # background thread; the loop below only ever reads its + # latest published result. + self.vision = Vision( + self.camera, + self.calibration, + self.detector, + ) + + self.physics = PhysicsWorld( + SCREEN_WIDTH, + SCREEN_HEIGHT, + ) + + self.effects = Effects() + + self.renderer = Renderer() + + self.running = True + + self.was_calibrated = False + + # Very negative so the first ball drops the moment + # calibration locks on. + self.last_ball_spawn = -1e9 + + self.last_warning = 0.0 + + self.vision.start() + + # ======================================================== + # Main loop + # ======================================================== + + def run(self): + try: + while self.running: + self._handle_events() + + if not self.running: + break + + # The per-frame pipeline is wrapped defensively: + # this runs unattended in a busy exhibition hall, + # and one unexpected error must never take the + # installation down. Worst case we skip a frame. + try: + self._run_one_frame() + except Exception as error: + print( + "[ERROR] unexpected failure in main " + f"loop, continuing: {error!r}" + ) + + time.sleep(0.05) + + finally: + self.close() + + def _run_one_frame(self): + ( + calibrated, + frozen, + obstacles, + message, + ) = self.vision.snapshot() + + current_time = time.monotonic() + + self._maybe_warn(current_time, frozen, message) + + if not calibrated: + if self.was_calibrated: + print( + "[CALIBRATION] lost/reset - " + "waiting for the screen to be recognized..." + ) + + self.was_calibrated = False + + self.renderer.render_calibration() + + # Nothing moves while calibrating; no need to spin. + self.renderer.clock.tick(30) + + return + + if not self.was_calibrated: + print("[CALIBRATION] locked on - starting game.") + + self.was_calibrated = True + + delta_time = self.renderer.get_delta_time() + + self.physics.update(delta_time, obstacles) + + self._spawn_ball(current_time) + + self.effects.update( + delta_time, + self.physics.balls, + self.physics.events, + ) + + self.renderer.render( + self.physics.balls, + obstacles, + self.effects, + self.physics.launcher, + ) + + # ======================================================== + # Warnings + # ======================================================== + + def _maybe_warn(self, current_time, frozen, message): + if ( + current_time - self.last_warning + < _WARNING_INTERVAL_SECONDS + ): + return + + if frozen: + print( + "[WARNING] Camera feed looks FROZEN " + "(no motion detected for a while) - " + "check the phone/app/connection. " + "Calibration and obstacle detection are " + "paused until a live frame is seen again." + ) + elif message: + print(f"[WARNING] {message}") + else: + return + + self.last_warning = current_time + + # ======================================================== + # Ball spawning + # ======================================================== + + def _spawn_ball(self, current_time): + # Balls arrive on their OWN clock, one every + # BALL_SPAWN_INTERVAL_SECONDS, whether or not earlier balls + # have left. A ball wedged between stickers just stays + # (until the physics removes it as stuck) and the next one + # still comes on schedule. + if ( + current_time - self.last_ball_spawn + < BALL_SPAWN_INTERVAL_SECONDS + ): + return + + if self.physics.spawn_ball() is not None: + self.last_ball_spawn = current_time + else: + self.last_ball_spawn = ( + current_time + - BALL_SPAWN_INTERVAL_SECONDS + + _SPAWN_RETRY_SECONDS + ) + + # ======================================================== + # Events + # ======================================================== + + def _handle_events(self): + for event in pygame.event.get(): + if event.type == pygame.QUIT: + self.running = False + + elif event.type == pygame.KEYDOWN: + if event.key == pygame.K_q: + self.running = False + + elif event.key in ( + pygame.K_c, + pygame.K_ESCAPE, + ): + self._reset_calibration() + + # ======================================================== + # Calibration reset + # ======================================================== + + def _reset_calibration(self): + self.vision.request_reset() + + self.physics.clear() + + self.effects.clear() + + self.renderer.force_full_repaint() + + self.last_ball_spawn = -1e9 + + # ======================================================== + # Shutdown + # ======================================================== + + def close(self): + try: + self.vision.stop() + except Exception: + pass + + try: + self.camera.release() + except Exception: + pass + + try: + cv2.destroyAllWindows() + except Exception: + pass + + try: + self.renderer.close() + except Exception: + pass + + +def main(): + game = Game() + game.run() + + +if __name__ == "__main__": + main() diff --git a/physics.py b/physics.py new file mode 100644 index 0000000..d51bf91 --- /dev/null +++ b/physics.py @@ -0,0 +1,747 @@ +# physics.py +# +# Pure simulation - no pygame, no OpenCV, no drawing. Everything the +# viewer sees (trails, glow, sparks, squash) is derived from this +# module's state and the events it emits, in effects.py. +# +# What is simulated: +# - balls falling under gravity, with slightly different sizes +# (bigger = heavier), +# - bouncing off detected stickers, which are ROTATED rectangles: +# the bounce uses the real surface normal (flat edge or corner) +# of the tilted sticker, with restitution (how bouncy) and +# friction (so balls roll along slopes instead of skating), +# - balls bouncing off each other, +# - ALL FOUR screen edges as solid walls - a ball never leaves the +# screen, it bounces off the top/bottom/left/right forever, +# - by default none of the above bounces lose any energy +# (BALL_RESTITUTION = BALL_BALL_RESTITUTION = 1.0, BALL_FRICTION +# = 0 in config.py), so with gravity a ball just keeps bouncing +# indefinitely like an ideal rubber ball, instead of gradually +# settling - lower these if a more "real rubber ball" feel that +# loses a little height each bounce is wanted instead, +# - a ball is only ever removed in one of two ways, each +# independently configurable in config.py and each with its own +# little pop effect: it reaches BALL_MAX_LIFETIME_SECONDS after +# being spawned, or it is "stuck" (parked in one spot, usually a +# pocket between stickers) for BALL_STUCK_TIMEOUT_SECONDS. +# +# Performance notes (this runs 144 times a second): +# - obstacle dicts are converted ONCE per detection update into +# lightweight _Solid records with cos/sin already computed, +# - each ball first does a cheap circle-vs-circle reject against +# each solid's bounding circle before any real collision math, +# - time is split into equal slices no longer than +# PHYSICS_MAX_STEP_SECONDS (and short enough that a fast ball +# can't skip over a sticker), so motion stays smooth and +# proportional to real elapsed time. + +import math +import random + +from geometry import extract_obb +from launcher import Launcher + +from config import ( + BALL_RADIUS, + BALL_RADIUS_VARIATION, + BALL_GRAVITY, + BALL_RESTITUTION, + BALL_REST_SPEED, + BALL_FRICTION, + BALL_MAX_SPEED, + BALL_BALL_COLLISION_ENABLED, + BALL_BALL_RESTITUTION, + PHYSICS_MAX_STEP_SECONDS, + MAX_SIMULTANEOUS_BALLS, + BALL_STUCK_DISTANCE, + BALL_STUCK_TIMEOUT_SECONDS, + BALL_MAX_LIFETIME_SECONDS, + BALL_HUE_MIN_DEGREES, + BALL_HUE_MAX_DEGREES, + IMPACT_EFFECT_MIN_SPEED, +) + + +# Continuous slow drag (per second) applied to the sliding speed of +# a ball that is resting/rolling ON a surface, so it doesn't glide +# forever on a slope. Impact friction (BALL_FRICTION) is separate. +_ROLLING_DRAG = 0.6 + +# How long a removed ("popped") ball takes to shrink away. +_DYING_SECONDS = 0.45 + +_MAX_EVENTS_PER_FRAME = 64 + + +class Ball: + def __init__( + self, + x, + y, + radius=15, + velocity_x=0.0, + velocity_y=0.0, + gravity=900.0, + hue=200.0, + ): + self.x = float(x) + self.y = float(y) + + self.radius = float(radius) + + # Mass grows with area, so a big ball shoves a small one + # around instead of bouncing off it like an equal. + self.mass = self.radius * self.radius + + self.velocity_x = float(velocity_x) + self.velocity_y = float(velocity_y) + + self.gravity = float(gravity) + + self.alive = True + + # True once the ball has fallen down far enough to actually + # be on screen (y - radius >= 0). Until then the top wall is + # not enforced, so a ball can still drop in from just above + # the visible screen the way it always has; from then on, + # the top wall is a solid ceiling like every other edge. + self.entered = False + + # Dying balls are on their way out (shrinking away): they + # no longer collide with anything. + self.dying = False + self.fade = 1.0 + + self.age = 0.0 + + # Purely cosmetic state, owned/updated by effects.py. + self.hue = float(hue) + self.trail = [] + self.trail_timer = 0.0 + self.squash = 0.0 + self.squash_nx = 0.0 + self.squash_ny = -1.0 + + # Stuck detection. + self._anchor_x = self.x + self._anchor_y = self.y + self._stuck_time = 0.0 + + @property + def speed(self): + return math.hypot(self.velocity_x, self.velocity_y) + + +class _Solid: + """One detected sticker, pre-digested for fast collision tests.""" + + __slots__ = ( + "cx", + "cy", + "half_long", + "half_short", + "cos", + "sin", + "bound", + ) + + def __init__(self, cx, cy, rect_long, rect_short, angle): + theta = math.radians(angle) + + self.cx = cx + self.cy = cy + self.half_long = rect_long / 2.0 + self.half_short = rect_short / 2.0 + self.cos = math.cos(theta) + self.sin = math.sin(theta) + + # Radius of the smallest circle containing the whole + # rectangle: anything farther away than this (plus a + # ball's radius) can't possibly touch it. + self.bound = math.hypot( + self.half_long, + self.half_short, + ) + + +class PhysicsWorld: + def __init__(self, width, height): + self.width = float(width) + self.height = float(height) + + self.launcher = Launcher(width, height) + + self.balls = [] + + # Impact/pop/launch events produced by the last update(), consumed + # by effects.py: (kind, x, y, normal_x, normal_y, speed, + # ball). kind is "obstacle", "ball", "pop" or "launch". + self.events = [] + + self._solids = [] + self._sticker_solids = [] + self._solid_source = None + + self._random = random.Random() + self._hue_cursor = self._random.uniform( + BALL_HUE_MIN_DEGREES, + BALL_HUE_MAX_DEGREES, + ) + + # ======================================================== + # Balls + # ======================================================== + + def add_ball(self, ball): + if ball is not None: + self.balls.append(ball) + + def live_ball_count(self): + return sum( + 1 + for ball in self.balls + if ball.alive and not ball.dying + ) + + def _next_hue(self): + # Golden-ratio hop around the allowed hue range: successive + # balls always get clearly different colors. + span = BALL_HUE_MAX_DEGREES - BALL_HUE_MIN_DEGREES + + self._hue_cursor = ( + BALL_HUE_MIN_DEGREES + + ( + self._hue_cursor + - BALL_HUE_MIN_DEGREES + + span * 0.618034 + ) + % span + ) + + return self._hue_cursor + + def spawn_ball(self): + """ + Spawns a new ball emerging from the launcher nozzle. + Returns the ball, or None if blocked at nozzle exit or max balls reached. + """ + if self.live_ball_count() >= MAX_SIMULTANEOUS_BALLS: + return None + + spawn_x, spawn_y, dir_x, dir_y = self.launcher.get_spawn_transform() + + rng = self._random + + radius = BALL_RADIUS * ( + 1.0 + + rng.uniform( + -BALL_RADIUS_VARIATION, + BALL_RADIUS_VARIATION, + ) + ) + + min_distance = (radius + BALL_RADIUS * (1.0 + BALL_RADIUS_VARIATION)) * 1.05 + + for other in self.balls: + if not other.alive or other.dying: + continue + + if math.hypot(spawn_x - other.x, spawn_y - other.y) < min_distance: + return None + + vel_x = dir_x * self.launcher.spawn_speed + vel_y = dir_y * self.launcher.spawn_speed + + ball = Ball( + x=spawn_x, + y=spawn_y, + radius=radius, + velocity_x=vel_x, + velocity_y=vel_y, + gravity=BALL_GRAVITY, + hue=self._next_hue(), + ) + ball.entered = True + + self.balls.append(ball) + + if len(self.events) < _MAX_EVENTS_PER_FRAME: + self.events.append( + ( + "launch", + spawn_x, + spawn_y, + dir_x, + dir_y, + self.launcher.spawn_speed, + ball, + ) + ) + + return ball + + # ======================================================== + # Obstacles + # ======================================================== + + def set_obstacles(self, obstacles): + # The vision thread publishes a NEW list object whenever + # something changed; rebuild sticker solids only then. + if obstacles is not self._solid_source: + self._solid_source = obstacles + + solids = [] + + for obstacle in obstacles or (): + obb = extract_obb(obstacle) + + if obb is None: + continue + + solids.append(_Solid(*obb)) + + self._sticker_solids = solids + + # Combine sticker solids + launcher tube solid + self._solids = list(self._sticker_solids) + if self.launcher is not None: + self._solids.append(_Solid(*self.launcher.get_solid_params())) + + # ======================================================== + # Update + # ======================================================== + + def update(self, delta_time, obstacles=None): + self.events = [] + + if delta_time <= 0: + return + + delta_time = min(delta_time, 0.05) + + if self.launcher is not None: + self.launcher.update(delta_time) + + self.set_obstacles(obstacles) + + movers = [ + ball + for ball in self.balls + if ball.alive and not ball.dying + ] + + if movers: + fastest = max(ball.speed for ball in movers) + + smallest = min(ball.radius for ball in movers) + + # Enough slices that (a) none is longer than the + # configured maximum and (b) no ball travels more than + # half its own radius in one slice - which is what + # guarantees it can't skip through a thin sticker. + steps = max( + math.ceil( + delta_time / PHYSICS_MAX_STEP_SECONDS + ), + math.ceil( + fastest * delta_time + / (0.5 * smallest) + ), + 1, + ) + + steps = min(steps, 10) + + step = delta_time / steps + + for _ in range(steps): + self._step(movers, step) + + self._housekeeping(delta_time) + + def _step(self, movers, h): + for ball in movers: + ball.velocity_y += ball.gravity * h + + speed = ball.speed + + if speed > BALL_MAX_SPEED: + scale = BALL_MAX_SPEED / speed + ball.velocity_x *= scale + ball.velocity_y *= scale + + ball.x += ball.velocity_x * h + ball.y += ball.velocity_y * h + + if BALL_BALL_COLLISION_ENABLED and len(movers) > 1: + self._collide_balls(movers) + + for ball in movers: + # Up to 3 passes so a ball squeezed between two + # stickers (or a sticker and another ball's push) ends + # up somewhere consistent instead of half inside one. + for _ in range(3): + touched = False + + for solid in self._solids: + if self._collide_solid(ball, solid, h): + touched = True + + if not touched: + break + + self._keep_inside_walls(ball) + + # ======================================================== + # Ball vs. sticker (oriented rectangle) + # ======================================================== + + def _collide_solid(self, ball, solid, h): + radius = ball.radius + + dx = ball.x - solid.cx + dy = ball.y - solid.cy + + reach = solid.bound + radius + + # Cheap reject: outside the sticker's bounding circle. + if dx * dx + dy * dy > reach * reach: + return False + + cos_t = solid.cos + sin_t = solid.sin + + # Ball center in the sticker's own un-rotated frame, where + # this is plain circle-vs-axis-aligned-box. + local_x = dx * cos_t + dy * sin_t + local_y = -dx * sin_t + dy * cos_t + + half_long = solid.half_long + half_short = solid.half_short + + closest_x = max(-half_long, min(local_x, half_long)) + closest_y = max(-half_short, min(local_y, half_short)) + + diff_x = local_x - closest_x + diff_y = local_y - closest_y + + distance_sq = diff_x * diff_x + diff_y * diff_y + + if distance_sq > radius * radius: + return False + + if distance_sq > 1e-9: + # Center outside the box: (diff) already points from the + # nearest surface point to the ball - the true surface + # normal for both a flat face hit and a corner hit. + distance = math.sqrt(distance_sq) + + normal_x = diff_x / distance + normal_y = diff_y / distance + + penetration = radius - distance + + local_x += normal_x * penetration + local_y += normal_y * penetration + else: + # Center already inside the box (very fast ball / thin + # sticker): leave through whichever side is nearest. + overlap_x = half_long + radius - abs(local_x) + overlap_y = half_short + radius - abs(local_y) + + if overlap_x < overlap_y: + normal_x = 1.0 if local_x >= 0 else -1.0 + normal_y = 0.0 + local_x = normal_x * (half_long + radius) + else: + normal_x = 0.0 + normal_y = 1.0 if local_y >= 0 else -1.0 + local_y = normal_y * (half_short + radius) + + ball.x = solid.cx + local_x * cos_t - local_y * sin_t + ball.y = solid.cy + local_x * sin_t + local_y * cos_t + + local_vx = ( + ball.velocity_x * cos_t + + ball.velocity_y * sin_t + ) + local_vy = ( + -ball.velocity_x * sin_t + + ball.velocity_y * cos_t + ) + + normal_speed = local_vx * normal_x + local_vy * normal_y + + # Only react if the ball is actually moving INTO the + # surface. (Reflecting a ball that is already leaving is + # what makes bounces flip-flop and jitter.) + if normal_speed < 0: + impact = -normal_speed + + tangent_x = -normal_y + tangent_y = normal_x + + tangent_speed = ( + local_vx * tangent_x + local_vy * tangent_y + ) + + if impact >= BALL_REST_SPEED: + bounce = BALL_RESTITUTION + tangent_speed *= 1.0 - BALL_FRICTION + else: + # Gentle contact = resting/rolling: no bounce, just + # a little rolling resistance. + bounce = 0.0 + tangent_speed *= max( + 0.0, + 1.0 - _ROLLING_DRAG * h, + ) + + new_normal_speed = -normal_speed * bounce + + local_vx = ( + new_normal_speed * normal_x + + tangent_speed * tangent_x + ) + local_vy = ( + new_normal_speed * normal_y + + tangent_speed * tangent_y + ) + + ball.velocity_x = local_vx * cos_t - local_vy * sin_t + ball.velocity_y = local_vx * sin_t + local_vy * cos_t + + if ( + impact >= IMPACT_EFFECT_MIN_SPEED + and len(self.events) < _MAX_EVENTS_PER_FRAME + ): + world_nx = normal_x * cos_t - normal_y * sin_t + world_ny = normal_x * sin_t + normal_y * cos_t + + self.events.append( + ( + "obstacle", + ball.x - world_nx * ball.radius, + ball.y - world_ny * ball.radius, + world_nx, + world_ny, + impact, + ball, + ) + ) + + return True + + # ======================================================== + # Ball vs. ball + # ======================================================== + + def _collide_balls(self, movers): + count = len(movers) + + for i in range(count - 1): + first = movers[i] + + for j in range(i + 1, count): + second = movers[j] + + reach = first.radius + second.radius + + dx = second.x - first.x + dy = second.y - first.y + + if abs(dx) >= reach or abs(dy) >= reach: + continue + + distance_sq = dx * dx + dy * dy + + if distance_sq >= reach * reach: + continue + + distance = math.sqrt(distance_sq) + + if distance > 1e-6: + normal_x = dx / distance + normal_y = dy / distance + else: + normal_x = 0.0 + normal_y = 1.0 + + total_mass = first.mass + second.mass + + # Push apart, the lighter ball moving more. + penetration = reach - distance + + first_share = penetration * (second.mass / total_mass) + second_share = penetration * (first.mass / total_mass) + + first.x -= normal_x * first_share + first.y -= normal_y * first_share + second.x += normal_x * second_share + second.y += normal_y * second_share + + closing = ( + (second.velocity_x - first.velocity_x) * normal_x + + (second.velocity_y - first.velocity_y) * normal_y + ) + + if closing >= 0: + continue + + impact = -closing + + bounce = ( + BALL_BALL_RESTITUTION + if impact >= BALL_REST_SPEED + else 0.0 + ) + + impulse = ( + -(1.0 + bounce) + * closing + / (1.0 / first.mass + 1.0 / second.mass) + ) + + first.velocity_x -= impulse / first.mass * normal_x + first.velocity_y -= impulse / first.mass * normal_y + second.velocity_x += impulse / second.mass * normal_x + second.velocity_y += impulse / second.mass * normal_y + + if ( + impact >= IMPACT_EFFECT_MIN_SPEED + and len(self.events) < _MAX_EVENTS_PER_FRAME + ): + self.events.append( + ( + "ball", + first.x + normal_x * first.radius, + first.y + normal_y * first.radius, + normal_x, + normal_y, + impact, + first, + ) + ) + + # ======================================================== + # Walls + # ======================================================== + # + # All four screen edges are solid: a ball bounces off every one + # of them forever and never leaves the screen (the only ways a + # ball ever disappears are the lifetime/stuck-timeout removal in + # _housekeeping below, each with its own little pop effect). + # With BALL_RESTITUTION left at its lossless default (1.0) these + # bounces never bleed off energy, so together with gravity a + # ball just keeps bouncing indefinitely, exactly like a perfect + # rubber ball in a box, until its time is up. + + def _keep_inside_walls(self, ball): + radius = ball.radius + + if ball.x - radius < 0: + ball.x = radius + + if ball.velocity_x < 0: + ball.velocity_x *= -BALL_RESTITUTION + + elif ball.x + radius > self.width: + ball.x = self.width - radius + + if ball.velocity_x > 0: + ball.velocity_x *= -BALL_RESTITUTION + + if not ball.entered: + if ball.y - radius < 0: + # Still on its way in from above the visible + # screen - leave the top wall unenforced for now. + return + + ball.entered = True + + if ball.y - radius < 0: + ball.y = radius + + if ball.velocity_y < 0: + ball.velocity_y *= -BALL_RESTITUTION + + elif ball.y + radius > self.height: + ball.y = self.height - radius + + if ball.velocity_y > 0: + ball.velocity_y *= -BALL_RESTITUTION + + # ======================================================== + # Lifetime, stuck balls, removal + # ======================================================== + + def _housekeeping(self, delta_time): + for ball in self.balls: + if not ball.alive: + continue + + if ball.dying: + ball.fade -= delta_time / _DYING_SECONDS + + if ball.fade <= 0.0: + ball.alive = False + + continue + + ball.age += delta_time + + if BALL_STUCK_TIMEOUT_SECONDS > 0: + moved = math.hypot( + ball.x - ball._anchor_x, + ball.y - ball._anchor_y, + ) + + if moved > BALL_STUCK_DISTANCE: + ball._anchor_x = ball.x + ball._anchor_y = ball.y + ball._stuck_time = 0.0 + else: + ball._stuck_time += delta_time + + if ball._stuck_time >= BALL_STUCK_TIMEOUT_SECONDS: + self._start_dying(ball) + continue + + if ( + BALL_MAX_LIFETIME_SECONDS > 0 + and ball.age >= BALL_MAX_LIFETIME_SECONDS + ): + self._start_dying(ball) + + self.balls = [ + ball for ball in self.balls if ball.alive + ] + + def _start_dying(self, ball): + ball.dying = True + ball.fade = 1.0 + + if len(self.events) < _MAX_EVENTS_PER_FRAME: + self.events.append( + ( + "pop", + ball.x, + ball.y, + 0.0, + -1.0, + 0.0, + ball, + ) + ) + + # ======================================================== + # Reset + # ======================================================== + + def clear(self): + self.balls = [] + self.events = [] + + def reset(self): + self.clear() diff --git a/rendering.py b/rendering.py new file mode 100644 index 0000000..40901ed --- /dev/null +++ b/rendering.py @@ -0,0 +1,965 @@ +# rendering.py +# +# Draws the game onto the monitor. +# +# Speed: with USE_DIRTY_RECTS the screen is split into a grid of +# small cells. Only cells where something was drawn last frame or is +# drawn now get repainted (background, then the sticker patches in +# them, then the effects in them) and only those cells are pushed to +# the display. A few balls touch a few dozen cells instead of all +# 2 million pixels. The calibration border and black interior are +# pre-rendered ONCE into a background surface and just copied in. +# +# Camera safety: the camera watches this very monitor for WHITE +# stickers. So everything here except the required sticker patch is +# drawn in saturated colour, with dim additive glows (never near +# white) and only tiny pastel highlights. The sticker patch is drawn +# slightly smaller than the sticker (see geometry.paint_dimensions) +# and the detector blanks it out, so a removed sticker can't be +# "seen" in its own leftover paint. + +import colorsys +import math +import os +import time + +import pygame + +from geometry import ( + obb_corners, + extract_obb, + paint_dimensions, +) + +from config import ( + SCREEN_WIDTH, + SCREEN_HEIGHT, + BACKGROUND_COLOR, + CALIBRATION_PATTERN_OUTER_COLOR, + CALIBRATION_PATTERN_INNER_COLOR, + CALIBRATION_OUTER_BORDER_SIZE, + CALIBRATION_INNER_BORDER_SIZE, + TARGET_FPS, + USE_DIRTY_RECTS, + FULL_REPAINT_SECONDS, + VISUAL_EFFECTS_ENABLED, + BALL_GLOW_ENABLED, + BALL_SQUASH_STRETCH_ENABLED, + SHOW_DETECTED_OBSTACLES, + DETECTED_OBSTACLE_COLOR, + DETECTED_OBSTACLE_BORDER_WIDTH, + DETECTED_OBSTACLE_PAINT_INSET_RATIO, +) + + +# Side length (px) of one dirty-repaint cell. +_CELL = 96 + +# Additive glow strengths (fraction of the full hue colour at the +# glow's centre). Deliberately dim: see "camera safety" above. +_GLOW_LEVELS = (0.16, 0.30, 0.48, 0.66) + +# Layers, drawn in this order inside every repainted cell. +_LAYER_GLOW = 0 +_LAYER_RING = 1 +_LAYER_BODY = 2 + + +class Renderer: + # ======================================================== + # Initialization + # ======================================================== + + def __init__(self): + # Ask SDL to place the (borderless) window at the top-left + # corner so it lines up exactly with the screen. + os.environ.setdefault("SDL_VIDEO_WINDOW_POS", "0,0") + + pygame.init() + + pygame.display.set_caption("Monitor Ball") + + # A borderless window sized to the full screen ("fake + # fullscreen"/kiosk mode) instead of pygame.FULLSCREEN. + # Exclusive SDL fullscreen needs a real videofullscreen-mode switch, + # which is unreliable on several X11/Wayland/XWayland driver + # combinations (blank frames, crashes on exit). + self.screen = pygame.display.set_mode( + (SCREEN_WIDTH, SCREEN_HEIGHT), + pygame.NOFRAME, + ) + + # pygame.mouse.set_visible(False) + + # try: + # pygame.event.set_grab(True) + # except Exception: + # pass + + # SDL2 turns "text input mode" on by default, which on + # systems with an active IME can swallow certain keys + # before they reach us as a normal KEYDOWN. + try: + pygame.key.stop_text_input() + except Exception: + pass + + self.clock = pygame.time.Clock() + + self._screen_rect = pygame.Rect( + 0, 0, SCREEN_WIDTH, SCREEN_HEIGHT + ) + + self._cols = (SCREEN_WIDTH + _CELL - 1) // _CELL + self._rows = (SCREEN_HEIGHT + _CELL - 1) // _CELL + + self._background = self._build_background() + + self._mode = None + self._need_full = True + self._last_full = 0.0 + + # Sticker patches, rebuilt only when the detector publishes + # a new list. + self._obstacle_source = None + self._obstacle_polys = [] + self._obstacle_cells = {} + self._obstacle_dirty = set() + + self._prev_cells = set() + + self._glow_cache = {} + self._color_cache = {} + + def _build_background(self): + surface = pygame.Surface((SCREEN_WIDTH, SCREEN_HEIGHT)) + surface.fill(BACKGROUND_COLOR) + return surface.convert() + + # ======================================================== + # Calibration screens + # ======================================================== + + def render_calibration(self): + """ + Solid pink over the whole screen: the simplest possible + target for the camera's first lock-on. It never changes, so + it is painted once (and again every FULL_REPAINT_SECONDS as + a safety net) instead of 144 times a second. + """ + now = time.monotonic() + + if ( + self._mode != "calibration" + or now - self._last_full >= FULL_REPAINT_SECONDS + ): + self.draw_full_calibration_screen() + + pygame.display.flip() + + self._mode = "calibration" + self._last_full = now + + self._need_full = True + self._prev_cells = set() + + def draw_full_calibration_screen(self): + self.screen.set_clip(None) + + self.screen.fill(CALIBRATION_PATTERN_OUTER_COLOR) + + def draw_calibration_border(self): + # Border ring + black interior (pre-rendered). + self.screen.set_clip(None) + + self.screen.blit(self._background, (0, 0)) + + def force_full_repaint(self): + self._need_full = True + + # ======================================================== + # Main rendering + # ======================================================== + + def render(self, balls, obstacles, effects=None, launcher=None): + now = time.monotonic() + + self._sync_obstacles(obstacles) + + items = self._collect_items(balls, effects, launcher) + + cell_items = {} + + for layer, bounds, draw in items: + self._add_to_cells( + cell_items, + bounds, + (layer, draw), + ) + + full = ( + not USE_DIRTY_RECTS + or self._need_full + or self._mode != "game" + or now - self._last_full >= FULL_REPAINT_SECONDS + ) + + if full: + self._paint_full(cell_items, now) + else: + self._paint_dirty(cell_items) + + self._prev_cells = set(cell_items) + + def _paint_full(self, cell_items, now): + screen = self.screen + + screen.set_clip(None) + + screen.blit(self._background, (0, 0)) + + for poly in self._obstacle_polys: + self._draw_obstacle(poly) + + everything = [] + + for entries in cell_items.values(): + everything.extend(entries) + + # An item spanning several cells appears once per cell; + # drawing it once per cell is wasteful when nothing is + # clipped, so de-duplicate by identity. + seen = set() + ordered = [] + + for entry in everything: + key = id(entry[1]) + + if key in seen: + continue + + seen.add(key) + ordered.append(entry) + + ordered.sort(key=lambda entry: entry[0]) + + for _layer, draw in ordered: + self._draw_item(draw) + + pygame.display.flip() + + self._need_full = False + self._mode = "game" + self._last_full = now + self._obstacle_dirty = set() + + def _paint_dirty(self, cell_items): + screen = self.screen + background = self._background + + dirty = ( + set(cell_items) + | self._prev_cells + | self._obstacle_dirty + ) + + self._obstacle_dirty = set() + + rects = [] + + for cell in dirty: + rect = self._cell_rect(cell) + + if rect.width <= 0 or rect.height <= 0: + continue + + rects.append(rect) + + screen.set_clip(rect) + + screen.blit(background, rect.topleft, rect) + + for index in self._obstacle_cells.get(cell, ()): + self._draw_obstacle(self._obstacle_polys[index]) + + entries = cell_items.get(cell) + + if entries: + entries.sort(key=lambda entry: entry[0]) + + for _layer, draw in entries: + self._draw_item(draw) + + screen.set_clip(None) + + if rects: + pygame.display.update(rects) + + # ======================================================== + # Grid helpers + # ======================================================== + + def _cell_rect(self, cell): + column, row = cell + + return pygame.Rect( + column * _CELL, + row * _CELL, + _CELL, + _CELL, + ).clip(self._screen_rect) + + def _cells_for(self, rect): + left = max(0, rect.left // _CELL) + top = max(0, rect.top // _CELL) + + right = min(self._cols - 1, (rect.right - 1) // _CELL) + bottom = min(self._rows - 1, (rect.bottom - 1) // _CELL) + + if rect.right <= 0 or rect.bottom <= 0: + return () + + if rect.left >= SCREEN_WIDTH or rect.top >= SCREEN_HEIGHT: + return () + + return [ + (column, row) + for row in range(top, bottom + 1) + for column in range(left, right + 1) + ] + + def _add_to_cells(self, cell_items, bounds, entry): + for cell in self._cells_for(bounds): + cell_items.setdefault(cell, []).append(entry) + + # ======================================================== + # Sticker patches + # ======================================================== + + def _sync_obstacles(self, obstacles): + if obstacles is self._obstacle_source: + return + + self._obstacle_source = obstacles + + # Cells that used to show a patch must be repainted, as + # must cells that now do. + self._obstacle_dirty |= set(self._obstacle_cells) + + polys = [] + cells = {} + + if SHOW_DETECTED_OBSTACLES: + for obstacle in obstacles or (): + if not obstacle.get("painted", True): + continue + + obb = extract_obb(obstacle) + + if obb is None: + continue + + cx, cy, long_side, short_side, angle = obb + + long_side, short_side = paint_dimensions( + long_side, + short_side, + DETECTED_OBSTACLE_PAINT_INSET_RATIO, + ) + + if long_side <= 0 or short_side <= 0: + continue + + corners = obb_corners( + cx, + cy, + long_side, + short_side, + angle, + ) + + points = [ + (int(round(px)), int(round(py))) + for px, py in corners + ] + + xs = [p[0] for p in points] + ys = [p[1] for p in points] + + bounds = pygame.Rect( + min(xs) - 2, + min(ys) - 2, + max(xs) - min(xs) + 4, + max(ys) - min(ys) + 4, + ) + + index = len(polys) + + polys.append(points) + + for cell in self._cells_for(bounds): + cells.setdefault(cell, []).append(index) + + self._obstacle_polys = polys + self._obstacle_cells = cells + + self._obstacle_dirty |= set(cells) + + def _draw_obstacle(self, points): + if DETECTED_OBSTACLE_BORDER_WIDTH <= 0: + pygame.draw.polygon( + self.screen, + DETECTED_OBSTACLE_COLOR, + points, + ) + else: + pygame.draw.polygon( + self.screen, + DETECTED_OBSTACLE_COLOR, + points, + int(DETECTED_OBSTACLE_BORDER_WIDTH), + ) + + # ======================================================== + # Collecting what to draw this frame + # ======================================================== + # + # Everything is turned into a flat list of + # (layer, bounding_rect, draw_tuple) ONCE per frame; the + # per-cell painting then only replays those tuples. + + def _collect_items(self, balls, effects, launcher=None): + items = [] + + if launcher is not None: + self._collect_launcher(items, launcher) + + effects_on = ( + VISUAL_EFFECTS_ENABLED and effects is not None + ) + + if effects_on: + for particle in effects.particles: + self._collect_particle(items, particle) + + for ring in effects.rings: + self._collect_ring(items, ring) + + for ball in balls or (): + if ball.alive: + self._collect_ball(items, ball, effects_on) + + return items + + def _collect_launcher(self, items, launcher): + points = launcher.get_points() + + xs = [p[0] for p in points] + ys = [p[1] for p in points] + + bounds = pygame.Rect( + int(min(xs)) - 3, + int(min(ys)) - 3, + int(max(xs) - min(xs)) + 6, + int(max(ys) - min(ys)) + 6, + ) + + items.append((_LAYER_BODY, bounds, ("launcher", launcher))) + + # ---- particles / rings ------------------------------------- + + def _collect_particle(self, items, particle): + x, y, _vx, _vy, age, life, hue, size = particle + + fraction = 1.0 - age / life + + level = min( + len(_GLOW_LEVELS) - 1, + int(fraction * len(_GLOW_LEVELS)), + ) + + radius = max(2, int(round(size * (0.6 + 0.6 * fraction))) + 2) + + sprite = self._glow(hue, radius, level) + + position = (int(x) - radius, int(y) - radius) + + bounds = pygame.Rect( + position[0], + position[1], + radius * 2, + radius * 2, + ) + + items.append( + ( + _LAYER_GLOW, + bounds, + ("sprite", sprite, position), + ) + ) + + def _collect_ring(self, items, ring): + x, y, age, life, hue, max_radius = ring + + fraction = age / life + + radius = max(2, int(max_radius * (0.25 + 0.75 * fraction))) + + fade = (1.0 - fraction) ** 1.5 + + color = self._hue_rgb(hue, 0.85, 0.9 * fade) + + width = max(1, int(3 * (1.0 - fraction)) + 1) + + bounds = pygame.Rect( + int(x) - radius - 2, + int(y) - radius - 2, + radius * 2 + 4, + radius * 2 + 4, + ) + + items.append( + ( + _LAYER_RING, + bounds, + ( + "ring", + color, + (int(x), int(y)), + radius, + width, + ), + ) + ) + + # ---- balls -------------------------------------------------- + + def _collect_ball(self, items, ball, effects_on): + fade = max(0.0, ball.fade) + + radius = ball.radius * fade + + if radius < 1.0: + return + + if effects_on: + self._collect_trail(items, ball, radius) + + if BALL_GLOW_ENABLED: + self._collect_glow(items, ball, radius, fade) + + points, angle_bounds = self._ball_polygon( + ball, + radius, + effects_on, + ) + + body = self._hue_rgb(ball.hue, 0.85, 1.0) + rim = self._hue_rgb(ball.hue, 0.95, 0.72) + + # A small pastel highlight (saturated enough that the + # camera can never count it as "white"). + highlight_color = self._hue_rgb(ball.hue, 0.42, 1.0) + + highlight_radius = max(1, int(round(radius * 0.24))) + + highlight_position = ( + int(round(ball.x - radius * 0.32)), + int(round(ball.y - radius * 0.34)), + ) + + items.append( + ( + _LAYER_BODY, + angle_bounds, + ( + "body", + points, + body, + rim, + highlight_position, + highlight_radius, + highlight_color, + ), + ) + ) + + def _ball_polygon(self, ball, radius, effects_on): + # Ellipse approximated by a polygon: stretched along the + # direction of travel with speed, flattened against the + # surface for a moment after a hit. + semi_a = radius + semi_b = radius + angle = 0.0 + + if effects_on and BALL_SQUASH_STRETCH_ENABLED: + speed = ball.speed + + if speed > 30.0: + stretch = min(0.35, speed / 4000.0) + + semi_a = radius * (1.0 + stretch) + semi_b = radius / (1.0 + stretch * 0.9) + + angle = math.atan2( + ball.velocity_y, + ball.velocity_x, + ) + + weight = ball.squash * ball.squash + + if weight > 0.12: + k = weight * 0.38 + + # First axis lies along the surface normal. + semi_a = radius * (1.0 - k) + semi_b = radius * (1.0 + k * 0.7) + + angle = math.atan2( + ball.squash_ny, + ball.squash_nx, + ) + + count = 18 if radius < 20 else 26 + + cos_a = math.cos(angle) + sin_a = math.sin(angle) + + points = [] + + for i in range(count): + theta = 2.0 * math.pi * i / count + + ex = semi_a * math.cos(theta) + ey = semi_b * math.sin(theta) + + points.append( + ( + ball.x + ex * cos_a - ey * sin_a, + ball.y + ex * sin_a + ey * cos_a, + ) + ) + + xs = [p[0] for p in points] + ys = [p[1] for p in points] + + bounds = pygame.Rect( + int(min(xs)) - 2, + int(min(ys)) - 2, + int(max(xs) - min(xs)) + 5, + int(max(ys) - min(ys)) + 5, + ) + + return points, bounds + + def _collect_glow(self, items, ball, radius, fade): + glow_radius = int(radius * 2.4) + + level = min( + len(_GLOW_LEVELS) - 1, + int(ball.speed / 450.0), + ) + + if fade < 0.6: + level = max(0, level - 1) + + sprite = self._glow(ball.hue, glow_radius, level) + + position = ( + int(round(ball.x)) - glow_radius, + int(round(ball.y)) - glow_radius, + ) + + bounds = pygame.Rect( + position[0], + position[1], + glow_radius * 2, + glow_radius * 2, + ) + + items.append( + ( + _LAYER_GLOW, + bounds, + ("sprite", sprite, position), + ) + ) + + def _collect_trail(self, items, ball, radius): + trail = ball.trail + + count = len(trail) + + if count < 2: + return + + sprites = [] + + min_x = min_y = 10 ** 9 + max_x = max_y = -(10 ** 9) + + biggest = 0 + + for index, (tx, ty) in enumerate(trail): + fraction = (index + 1) / count + + sprite_radius = max( + 2, + int(radius * (0.35 + 0.6 * fraction)), + ) + + level = min( + len(_GLOW_LEVELS) - 1, + int(fraction * len(_GLOW_LEVELS)), + ) + + position = ( + int(round(tx)) - sprite_radius, + int(round(ty)) - sprite_radius, + ) + + sprites.append( + ( + self._glow(ball.hue, sprite_radius, level), + position, + ) + ) + + biggest = max(biggest, sprite_radius) + + min_x = min(min_x, tx) + max_x = max(max_x, tx) + min_y = min(min_y, ty) + max_y = max(max_y, ty) + + bounds = pygame.Rect( + int(min_x) - biggest - 1, + int(min_y) - biggest - 1, + int(max_x - min_x) + biggest * 2 + 3, + int(max_y - min_y) + biggest * 2 + 3, + ) + + items.append( + ( + _LAYER_GLOW, + bounds, + ("trail", sprites), + ) + ) + + # ======================================================== + # Drawing one collected item + # ======================================================== + + def _draw_item(self, draw): + kind = draw[0] + + screen = self.screen + + if kind == "sprite": + screen.blit( + draw[1], + draw[2], + special_flags=pygame.BLEND_RGB_ADD, + ) + + elif kind == "trail": + for sprite, position in draw[1]: + screen.blit( + sprite, + position, + special_flags=pygame.BLEND_RGB_ADD, + ) + + elif kind == "ring": + _kind, color, center, radius, width = draw + + pygame.draw.circle( + screen, + color, + center, + radius, + width, + ) + + elif kind == "body": + ( + _kind, + points, + body, + rim, + highlight_position, + highlight_radius, + highlight_color, + ) = draw + + pygame.draw.polygon(screen, body, points) + + # Anti-aliased outline in the rim colour smooths the + # jagged polygon fill edge. + pygame.draw.aalines(screen, rim, True, points) + + pygame.draw.circle( + screen, + highlight_color, + highlight_position, + highlight_radius, + ) + + elif kind == "launcher": + launcher = draw[1] + points = launcher.get_points() + + # Solid metallic tube fill + pygame.draw.polygon(screen, (48, 55, 66), points) + + # Highlight & Shadow lines along the tube + rad = math.radians(launcher.angle) + u_x = math.sin(rad) + u_y = math.cos(rad) + v_x = -u_y + v_y = u_x + + half_w = launcher.tube_width / 2.0 + length = launcher.length + + # Highlight line along left inner edge + hl_start = ( + launcher.pivot_x - v_x * (half_w * 0.4), + launcher.pivot_y - v_y * (half_w * 0.4), + ) + hl_end = ( + launcher.pivot_x + u_x * length - v_x * (half_w * 0.4), + launcher.pivot_y + u_y * length - v_y * (half_w * 0.4), + ) + pygame.draw.line( + screen, + (115, 126, 142), + hl_start, + hl_end, + width=max(2, int(half_w * 0.45)), + ) + + # Shadow line along right inner edge + sh_start = ( + launcher.pivot_x + v_x * (half_w * 0.5), + launcher.pivot_y + v_y * (half_w * 0.5), + ) + sh_end = ( + launcher.pivot_x + u_x * length + v_x * (half_w * 0.5), + launcher.pivot_y + u_y * length + v_y * (half_w * 0.5), + ) + pygame.draw.line( + screen, + (26, 30, 38), + sh_start, + sh_end, + width=max(2, int(half_w * 0.35)), + ) + + # Dark nozzle rim at tip + p_bot_right = points[2] + p_bot_left = points[3] + rim_points = [ + ( + launcher.pivot_x + u_x * (length - 6.0) - v_x * half_w, + launcher.pivot_y + u_y * (length - 6.0) - v_y * half_w, + ), + ( + launcher.pivot_x + u_x * (length - 6.0) + v_x * half_w, + launcher.pivot_y + u_y * (length - 6.0) + v_y * half_w, + ), + p_bot_right, + p_bot_left, + ] + pygame.draw.polygon(screen, (32, 36, 44), rim_points) + + # Crisp metallic outline + pygame.draw.polygon(screen, (78, 88, 102), points, width=2) + pygame.draw.aalines(screen, (95, 108, 124), True, points) + + # ======================================================== + # Colours and glow sprites + # ======================================================== + + def _hue_rgb(self, hue, saturation, value): + key = ( + int(hue) % 360, + int(saturation * 50), + int(value * 50), + ) + + color = self._color_cache.get(key) + + if color is None: + r, g, b = colorsys.hsv_to_rgb( + (key[0] % 360) / 360.0, + saturation, + max(0.0, min(1.0, value)), + ) + + color = ( + int(r * 255), + int(g * 255), + int(b * 255), + ) + + self._color_cache[key] = color + + return color + + def _glow(self, hue, radius, level): + # Soft radial gradient, cached by (hue bucket, size, level). + radius = max(2, min(int(radius), 96)) + + bucket = int(hue // 6) % 60 + + key = (bucket, radius, level) + + sprite = self._glow_cache.get(key) + + if sprite is None: + color = self._hue_rgb(bucket * 6 + 3, 0.9, 1.0) + + gain = _GLOW_LEVELS[level] + + sprite = pygame.Surface((radius * 2, radius * 2)) + + sprite.fill((0, 0, 0)) + + for i in range(radius, 0, -1): + t = 1.0 - i / radius + + k = (t ** 1.8) * gain + + pygame.draw.circle( + sprite, + ( + int(color[0] * k), + int(color[1] * k), + int(color[2] * k), + ), + (radius, radius), + i, + ) + + sprite = sprite.convert() + + self._glow_cache[key] = sprite + + return sprite + + # ======================================================== + # Timing and shutdown + # ======================================================== + + def get_delta_time(self): + delta_time = self.clock.tick(TARGET_FPS) / 1000.0 + + return max(0.0, min(delta_time, 0.05)) + + def close(self): + pygame.mouse.set_visible(True) + + pygame.quit() diff --git a/vision.py b/vision.py new file mode 100644 index 0000000..1c2097a --- /dev/null +++ b/vision.py @@ -0,0 +1,228 @@ +# vision.py +# +# Runs everything that looks at the camera - screen calibration and +# sticker detection - on its own background thread, so the render / +# physics loop never waits on OpenCV. (Before this, a detection pass +# ran inside the game loop and every pass was a small visible hitch.) +# OpenCV releases the GIL while it works, so the two really do run +# side by side. +# +# The game loop only ever calls snapshot(), which is a cheap lock + +# a few attribute reads. The obstacle list it returns is the SAME +# list object until something actually changed (Detector publishes a +# new list only on change), so physics/rendering can detect changes +# with a plain identity check. + +import threading +import time + +from config import ( + DETECTION_SCALE, + MONITOR_DETECTION_INTERVAL, + OBSTACLE_DETECTION_INTERVAL, +) + + +class Vision: + def __init__(self, camera, calibration, detector): + self.camera = camera + self.calibration = calibration + self.detector = detector + + self._lock = threading.Lock() + + self._calibrated = False + self._frozen = False + self._obstacles = [] + self._message = "Camera not connected yet." + + # Bumped by request_reset(); a result computed against an + # older generation is discarded instead of being published. + self._generation = 0 + self._reset_pending = False + + self._stop_event = threading.Event() + + self._thread = threading.Thread( + target=self._run, + name="vision", + daemon=True, + ) + + # ======================================================== + # Public API (called from the game loop) + # ======================================================== + + def start(self): + self._thread.start() + + def stop(self): + self._stop_event.set() + + def request_reset(self): + """Forget calibration and stickers; start over.""" + with self._lock: + self._generation += 1 + self._reset_pending = True + self._calibrated = False + self._obstacles = [] + + def snapshot(self): + """(calibrated, camera_frozen, obstacles, camera_message).""" + with self._lock: + return ( + self._calibrated, + self._frozen, + self._obstacles, + self._message, + ) + + # ======================================================== + # Thread + # ======================================================== + + def _run(self): + last_frame_id = None + last_monitor_pass = 0.0 + last_obstacle_pass = 0.0 + last_error_print = 0.0 + + while not self._stop_event.is_set(): + try: + with self._lock: + reset = self._reset_pending + self._reset_pending = False + generation = self._generation + + if reset: + self.calibration.reset() + self.detector.reset() + + last_frame_id = None + last_monitor_pass = 0.0 + last_obstacle_pass = 0.0 + + try: + frame, frame_id = self.camera.read_with_id() + except RuntimeError as error: + self._publish( + generation, + message=str(error), + ) + + time.sleep(0.05) + continue + + frozen = self.camera.is_frozen() + + if frame_id == last_frame_id: + # Same frame as last time: nothing new to + # analyse (the camera delivers ~30 fps, the + # loop spins faster). Still keep the frozen + # flag fresh in case frames stopped entirely. + self._publish(generation, frozen=frozen) + + time.sleep(0.004) + continue + + last_frame_id = frame_id + + if frozen: + # A stuck feed must not be trusted for + # calibration or stickers. + self._publish( + generation, + frozen=True, + message="", + ) + continue + + now = time.monotonic() + + calibrated = self.calibration.is_calibrated() + + # Until locked on, look at every new frame; once + # locked, only re-verify the monitor occasionally. + if ( + not calibrated + or now - last_monitor_pass + >= MONITOR_DETECTION_INTERVAL + ): + self.calibration.update(frame) + + last_monitor_pass = now + + calibrated = self.calibration.is_calibrated() + + obstacles = None + + if ( + calibrated + and now - last_obstacle_pass + >= OBSTACLE_DETECTION_INTERVAL + ): + small = self.calibration.transform_frame( + frame, + DETECTION_SCALE, + ) + + # Keep the sticker detector's idea of "what does + # white look like right now" in sync with what + # calibration is continuously learning off the + # ring - see Calibration.white_reference(). + self.detector.set_white_reference( + self.calibration.white_reference() + ) + + obstacles = self.detector.detect(small) + + last_obstacle_pass = time.monotonic() + + self._publish( + generation, + calibrated=calibrated, + frozen=False, + message="", + obstacles=obstacles, + ) + + except Exception as error: + # Never let one bad frame kill the thread (and with + # it, all vision until the next restart). + now = time.monotonic() + + if now - last_error_print > 5.0: + print( + "[VISION] unexpected error, " + f"continuing: {error!r}" + ) + + last_error_print = now + + time.sleep(0.1) + + def _publish( + self, + generation, + calibrated=None, + frozen=None, + message=None, + obstacles=None, + ): + with self._lock: + if generation != self._generation: + # A reset happened while this pass was running; its + # results describe the old state. + return + + if calibrated is not None: + self._calibrated = calibrated + + if frozen is not None: + self._frozen = frozen + + if message is not None: + self._message = message + + if obstacles is not None: + self._obstacles = obstacles