Wrong data when reading pixels in mode 13 (386 Assembly - DOS)

Question

For a school project, I'm writing a paint program in assembly for DOSBOX. In my program, the user uses the left mouse button to paint pixels in a certain color. I use direct writes in mode 13 for that. The user can change that color by right-clicking, which reads the color of the pixel that the mouse is pointing at.

This works perfectly fine, until the user initiates a certain subroutine that is intended to wipe the screen, and show a palette of colors to pick from (with the same right-click functionality). Then, the user right-clicks on a color in the palette, and int 10h,0dh is used to read the color. No matter where the user right-clicks in the palette screen, the color that is later written to the screen by the user is always white.

tl;dr right clicking anywhere changes the brush color properly, but if done in palette-mode the subsequent color is always white. Here is my code:

GRAPHICS equ 13h
H_HOTSPOT equ 7
V_HOTSPOT equ 7
H_RES equ 320
V_RES equ 200
PIXELCOUNT equ H_RES*V_RES
DISPLAY_SEG equ 0A000h

EXIT_KEY equ 1071h          ; q key
COLORPICKER_KEY equ 1177h   ; w key

PALETTE_SIZE equ 16
SQUARE_SIZE equ 10
SQUARE_PADDING equ 2
SQUARE_ROW_JUMP equ H_RES - SQUARE_SIZE
PALETTE_ROW_JUMP equ H_RES - (SQUARE_PADDING + SQUARE_SIZE)*PALETTE_SIZE

sseg SEGMENT
    db 256 dup(?)
sseg ENDS

dseg SEGMENT

stdBrushMask    dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 0000000100000000b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b
                dw 1111111011111111b

                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 1111111011111111b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b
                dw 0000000100000000b

stdBrushHotSpots dw 7
                 dw 7

pickerToolMask  dw 1111100001000001b
                dw 1111100000000000b
                dw 1111100000000000b
                dw 1111100000000000b
                dw 1111100000000000b
                dw 1111000000000000b
                dw 1110000000000000b
                dw 1100000000000000b
                dw 1000000000000000b
                dw 1000000000000000b
                dw 1000000000000000b
                dw 1000000000011111b
                dw 0000000000111111b
                dw 0000000001111111b
                dw 0000000011111111b
                dw 0000111111111111b

                dw 0000000000000000b
                dw 0000001100011100b
                dw 0000001111111110b
                dw 0000000111111110b
                dw 0000000111111110b
                dw 0000001111111100b
                dw 0000011111111100b
                dw 0000111111111100b
                dw 0001111111111110b
                dw 0011111111100110b
                dw 0001111111000000b
                dw 0001111110000000b
                dw 0011111100000000b
                dw 0111001000000000b
                dw 0110000000000000b
                dw 0000000000000000b

pickerToolHotSpots dw 1
                   dw 14

foreground_color dw 000fh
background_color dw 0000h
colorpicker_flag db 00

prev_position    dw H_RES   ;   Cache of the last drawn pixel's coordinates, to avoid duplicate writes.
                 dw V_RES

pos_backup       dw H_RES/2
                 dw V_RES/2 ;   Cache of the mouse position before switching to palette mode or back.

video_mode_not_available db "Mode 13h is not supported on this computer. To use this program, get a VGA or MCGA graphics card / monitor.$"

display_backup db PIXELCOUNT dup(?)
dseg ENDS

cseg SEGMENT
assume cs:cseg, ds:dseg, ss:sseg

;---------------Mouse Procs-------------
stdBrush PROC
    push bx cx ax dx
    mov ax, dseg
    mov es, ax
    mov bx, stdBrushHotSpots
    mov cx, stdBrushHotSpots + 2
    mov ax, 9
    mov dx, offset stdBrushMask
    int 33h
    mov ax, DISPLAY_SEG
    mov es, ax
    pop dx ax cx bx
    ret
stdBrush ENDP

pickerTool PROC
    push bx cx ax dx
    mov ax, dseg
    mov es, ax
    mov bx, pickerToolHotSpots
    mov cx, pickerToolHotSpots + 2
    mov ax, 9
    mov dx, offset pickerToolMask
    int 33h
    mov ax, DISPLAY_SEG
    mov es, ax
    pop dx ax cx bx
    ret
pickerTool ENDP

mouseReset PROC
    push ax
    mov ax, 0
    int 33h
    pop ax
    ret
mouseReset ENDP

showCursor PROC
    push ax
    mov ax, 1
    int 33h
    pop ax
    ret
showCursor ENDP

hideCursor PROC
    push ax
    mov ax, 2
    int 33h
    pop ax
    ret
hideCursor ENDP

getCursorStat PROC
    push ax
    mov ax, 3
    int 33h
    pop ax
    ret
getCursorStat ENDP

initCursor PROC
    mov ax, dseg
    mov es, ax
    call mouseReset
    call stdBrush
    call showCursor
    mov ax, DISPLAY_SEG
    mov es, ax
    ret
initCursor ENDP

;--------------------Graphics Procs-------------
graphicsCompat PROC
    ; Checks if mode 13h is supported on this machine.
    mov ax, 1a00h
    int 10h ; Get display combination
    cmp al, 1ah
    je mode_13_supported
    mov ah, 9
    mov dx, offset video_mode_not_available
    int 21h ; Display error message
    inc sp  ; Discard the return address
    inc sp
    push offset exit    ; Replace return address with exit procedure address
    ret                 ; Return to exit procedure.

    mode_13_supported:
        ;No error, mode 13 is supported
    ret
graphicsCompat ENDP

graphicsMode PROC
    mov ah, 0
    mov al, GRAPHICS
    int 10h
    ret
graphicsMode ENDP

pixel PROC
    push bp
    mov bp, sp
    push ax bx cx dx
    xor bx, bx
    mov cx, ss:[bp+6]   ; x coord
    mov dx, ss:[bp+4]   ; y coord
    cmp cx, H_RES
    jnc pixel_outofbounds
    cmp dx, V_RES
    jnc pixel_outofbounds
    mov ax, H_RES
    mul dx
    add ax, cx
    mov di, ax      ; Puts the pixel's offset in di
    mov ax, ss:[bp+8]   ; read color argument
    stosb
  pixel_outofbounds:
    pop dx cx bx ax bp
    ret 6
pixel ENDP

backupScreen PROC
    push ax di si
    mov ax, DISPLAY_SEG
    mov ds, ax
    mov ax, dseg
    mov es, ax
    mov cx, PIXELCOUNT
    mov si, 0
    mov di, offset display_backup
    call hideCursor
    rep movsb
    call showCursor
    mov ds, ax
    mov ax, DISPLAY_SEG
    mov es, ax
    pop si di ax
    ret
backupScreen ENDP

restoreScreen PROC
    push di si
    mov cx, PIXELCOUNT
    mov si, offset display_backup
    mov di, 0
    rep movsb
    pop si di
    ret
restoreScreen ENDP

;-------------------Colorpicker Procs----------
paletteModeToggle PROC
    push ax
    call hideCursor
    mov pos_backup, cx
    mov pos_backup+2, dx
    mov al, colorpicker_flag
    not al
    mov colorpicker_flag, al
    test al, al
    jz palette_mode_off
    palette_mode_on:
        call backupScreen
        call graphicsMode
        call paletteDraw
        call showCursor
        call pickerTool
        pop ax
        jmp input_loop
    palette_mode_off:
        call graphicsMode
        call restoreScreen
        call showCursor
        call stdBrush
    pop ax
    jmp input_loop
paletteModeToggle ENDP

paletteSquareDraw PROC
    push bp
    mov bp, sp
    push ax cx dx

    mov dx, ss:[bp+4]   ; retrieves ypos
    mov cx, ss:[bp+6]   ; retrieves xpos
    mov ax, H_RES
    mul dx
    add ax, cx
    mov di, ax  ; Puts the real pixel address in di.
    mov ax, ss:[bp+8]   ;retrieves color

    xor ch, ch
    square_loop:
        xor cl, cl
        square_row_loop:
            stosb
            inc cl  ; increment horizontal counter
            cmp cl, SQUARE_SIZE
            jb square_row_loop
        inc ch                      ; increment the vertical counter
        add di, SQUARE_ROW_JUMP     ; move di to the start of the next row
        cmp ch, SQUARE_SIZE
        jb square_loop

    pop dx cx ax bp
    ret 6
paletteSquareDraw ENDP

paletteDraw PROC

    xor cx, cx  ; xpos = 0
    xor dx, dx  ; ypos = 0
    xor ax, ax  ; color = 0
    xor bx, bx  ; squarecounter = 0

    palette_loop:   ; for i in paletterows:
        xor bl, bl
        add dx, SQUARE_PADDING
        palette_row_loop:   ;   for j in palettecols:
            add cx, SQUARE_PADDING  ; xpos += padding
            push ax
            push cx
            push dx
            call paletteSquareDraw  ; DrawSquare()

            add cx, SQUARE_SIZE
            inc ax  ; next color
            inc bl  ; squarecounter++
            cmp bl, PALETTE_SIZE
            jb palette_row_loop
        ; prep for new row
        xor bl, bl
        xor cx, cx
        add dx, SQUARE_SIZE
        inc bh
        cmp bh, PALETTE_SIZE
        jb palette_loop
        ret

paletteDraw ENDP

pickColor PROC
    push bp
    mov bp, sp
    push ax dx si
    mov dx, ss:[bp+4]
    mov cx, ss:[bp+6]
    xor bh, bh
    mov ah, 0dh
    int 10h
    mov foreground_color, ax
    pop si dx ax bp
    ret 4
pickColor ENDP

;--------------------------Program Entry-------------------------
main:
    mov ax, dseg
    mov ds, ax
    mov ax, sseg
    mov ss, ax
    mov ax, DISPLAY_SEG
    mov es, ax

    call graphicsCompat
    call graphicsMode
    call mouseReset
    call stdBrush
    call showCursor
    mov ax, DISPLAY_SEG
    mov es, ax

input_loop:

    ;------------Mouse Input Detection------------------------------------
    call getCursorStat
    shr bx, 1
    jnc no_left_click
    ; Left click detected
    mov al, colorpicker_flag
    test al, al
    jnz no_left_click   ; no left clicks in palette mode.
    shr cx, 1
    cmp cx, prev_position
    jnz draw_pixel
    cmp dx, prev_position + 2
    jz no_left_click
  draw_pixel:
    call hideCursor
    mov ax, foreground_color
    mov prev_position, cx
    mov prev_position + 2, dx
    push ax cx dx
    call pixel
    call showCursor


  no_left_click:
    mov bx, 1
    mov ax, 6
    int 33h
    test bx, bx
    jz keyboard_detection
    ; Right click detected
    shr cx, 1
    push cx dx
    call pickColor
  keyboard_detection:
    ;------------Keyboard Input Detection----------------------------------
    mov ah, 1
    int 16h
    jz no_keystroke
    xor ah, ah
    int 16h
    cmp ax, EXIT_KEY
    je exit
    cmp ax, COLORPICKER_KEY
    jne no_palette_toggle
    jmp paletteModeToggle
  no_palette_toggle:
  no_keystroke:
    jmp input_loop

exit:
    mov al, 3
    mov ah, 0
    int 10h
    mov ax, 4c00h
    int 21h

cseg ENDS
end main

screenshots for clarity: Using the blue brush in drawing mode. After switching to palette mode, I right clicked the bright green square. After switching back to drawing mode, the brush was drawing white pixels. Right clicking a blue pixel in drawing mode. Brush color properly switches to blue.

In this version I used int 10h,0dh to read the pixel, but the same problem occurred when I tried to directly read from segment A000 (which, during debugging, appeared to be completely filled with zeroes when the screen was clearly not empty). I switched to using int 10h to see if direct reading was the problem.

I did use direct writes though.

The bottom line question: What am I doing wrong with reading these pixels? Why does my program read pixel colors properly initially, but after essentially clearing the screen, changing a cursor and drawing a bunch of colored squares, it only reads white? Why does it work again after clearing those squares and restoring the screen to its previous state?

Answer 1

Your problem is that the mouse driver uses a software cursor. VGA cards don't support a hardware cursor, which means in order to display the cursor the mouse driver needs to paint it to the VGA frame buffer. That means when when the picker cursor is displayed and you read the pixel at the current mouse position, you're actually reading the colour of the picker cursor's hot spot, which is white. The problem doesn't happen when you're using the brush cursor because the pixel at its hot spot is transparent.