Inconsistent DOM class when changing class dynamically

Question

I have this component which handles scroll by dragging:

import { useSignal } from '@builder.io/qwik';
import { $, component$, Slot, useStore } from '@builder.io/qwik';

export const DragToScrollContainer = component$(() => {
  const ref = useSignal<HTMLDivElement>(undefined!);
  const isDownSig = useSignal(false);
  const state = useStore({
    startX: 0,
    scrollLeft: 0,
  });

  const handleMouseDown$ = $((e: any) => {
    isDownSig.value = true;
    state.startX = e.pageX - ref.value.offsetLeft;
    state.scrollLeft = ref.value.scrollLeft;
  });

  const handleMouseLeave$ = $(() => {
    isDownSig.value = false;
  });

  const handleMouseUp$ = $(() => {
    isDownSig.value = false;
  });

  const handleMouseMove$ = $((e: MouseEvent) => {
    if (!isDownSig.value) return;
    e.preventDefault();
    const x = e.pageX - ref.value.offsetLeft;
    const walk = (x - state.startX) * 2;
    ref.value.scrollLeft = state.scrollLeft - walk;
  });

  return (
    <div
      ref={ref}
      class={{
        'scrollable-container grabbable custom-grid hidden-scrollbar': true,
        'cursor-grabbing': isDownSig.value,
        'cursor-grab': !isDownSig.value,
      }}
      onMouseDown$={handleMouseDown$}
      onMouseLeave$={handleMouseLeave$}
      onMouseUp$={handleMouseUp$}
      onMouseMove$={handleMouseMove$}
    >
      <Slot />
    </div>
  );
});

Whenever I access the page which contains this component directly (Server rendered), trying to drag the items makes the class broken with empty state. While having another page opened then navigating to the page which contains this component makes everything working well. Please check the attached video.

BTW, the issue only appears when building the application but serving locally the project is working as expected.