Samsung TV Tizen Grabber App for AmbientLight

Question

I am currently working on a project that involves capturing the screen content of a Samsung TV running Tizen OS in real-time and transmitting the color information to a connected Hyperion server for ambient lighting purposes. Here is a summary of what I have achieved so far and the challenges I am facing: What I've Achieved:

1. Capturing Video Content: I successfully implemented a web app that captures the content of a video playing within the app and transmits it to the Hyperion server with minimal latency. This ensures smooth and real-time color capture from the video content.
2. Real-time Transmission: The captured video frames are processed and sent to the Hyperion server efficiently, providing a seamless ambient lighting experience based on the video content.

Also, if someone wants the code from what I achieved sending screenshots from video to Hyperion here is the code (run it on Tizen Studio) main.js:

// Main
document.addEventListener('DOMContentLoaded', function () {
    log('Document loaded and DOM fully parsed');
    document.getElementById('startCapture').addEventListener('click', startCapture);
    document.getElementById('remoteButton').addEventListener('click', startCapture);
    document.getElementById('testImageButton').addEventListener('click', sendTestImage);
    document.getElementById('sendColorButton').addEventListener('click', sendRandomColor);
    document.getElementById('startVideoCaptureButton').addEventListener('click', startVideoCapture);
    document.getElementById('stopVideoCaptureButton').addEventListener('click', stopVideoCapture);

    document.addEventListener('keydown', function(event) {
        switch (event.key) {
            case 'Enter':
                log('Remote control button pressed');
                document.activeElement.click();
                break;
            case 'ArrowUp':
                focusPreviousElement();
                break;
            case 'ArrowDown':
                focusNextElement();
                break;
        }
    });

    initializeWebSocket();
});

function focusPreviousElement() {
    let focusable = document.querySelectorAll('[tabindex]');
    let index = Array.prototype.indexOf.call(focusable, document.activeElement);
    if (index > 0) {
        focusable[index - 1].focus();
    }
}

function focusNextElement() {
    let focusable = document.querySelectorAll('[tabindex]');
    let index = Array.prototype.indexOf.call(focusable, document.activeElement);
    if (index < focusable.length - 1) {
        focusable[index + 1].focus();
    }
}

function startCapture() {
    log('Start Capture button clicked');
    captureScreen();
}

function captureScreen() {
    log('Capturing screen');
    const captureArea = document.getElementById('captureArea');
    const canvas = document.createElement('canvas');
    const context = canvas.getContext('2d');

    const fixedWidth = 120;
    const fixedHeight = 120;

    canvas.width = fixedWidth;
    canvas.height = fixedHeight;

    try {
        // Colores alternativos para asegurar que la imagen no sea negra
        const colors = ['red', 'green', 'blue', 'yellow', 'purple', 'orange'];
        const randomColor = colors[Math.floor(Math.random() * colors.length)];

        context.fillStyle = randomColor;
        context.fillRect(0, 0, canvas.width, canvas.height);

        const text = captureArea.innerText || captureArea.textContent;
        context.font = '30px Arial';
        context.fillStyle = 'white';
        context.textAlign = 'center';
        context.textBaseline = 'middle';
        context.fillText(text, canvas.width / 2, canvas.height / 2);

        const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03

        log('Screenshot captured, Base64 length: ' + imageData.length);

        sendToHyperion(imageData);
    } catch (error) {
        log('Error capturing screen: ' + error);
    }
}

function sendTestImage() {
    log('Sending test image to Hyperion');
    const testImageUrl = 'https://eligeeducar.cl/content/uploads/2019/10/Imagen-que-dice-rojo-sobre-un-fondo-azul-y-azul-sobre-un-fondo-rojo--1920x550.jpg';

    fetch(testImageUrl)
        .then(response => response.blob())
        .then(blob => compressImage(blob))
        .then(imageData => sendToHyperion(imageData))
        .catch(error => {
            log('Error fetching test image: ' + error);
        });
}

function compressImage(blob) {
    return new Promise((resolve, reject) => {
        const reader = new FileReader();
        reader.onload = function () {
            const img = new Image();
            img.onload = function () {
                const canvas = document.createElement('canvas');
                const ctx = canvas.getContext('2d');
                const maxWidth = 120;
                const maxHeight = 120;
                let width = img.width;
                let height = img.height;

                if (width > height) {
                    if (width > maxWidth) {
                        height *= maxWidth / width;
                        width = maxWidth;
                    }
                } else {
                    if (height > maxHeight) {
                        width *= maxHeight / height;
                        height = maxHeight;
                    }
                }
                canvas.width = width;
                canvas.height = height;
                ctx.drawImage(img, 0, 0, width, height);
                const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03
                log('Compressed image captured, Base64 length: ' + imageData.length);
                resolve(imageData);
            };
            img.onerror = reject;
            img.src = reader.result;
        };
        reader.onerror = reject;
        reader.readAsDataURL(blob);
    });
}

let ws;
let messageQueue = [];

function sendToHyperion(imageData) {
    const jsonData = {
        command: "image",
        imagedata: imageData,
        name: "TizenScreenCapture",
        format: "auto",
        origin: "TizenApp",
        priority: 50,
        duration: 30000 // Duración de 30 segundos
    };

    log('Sending screenshot to Hyperion');

    if (ws.readyState === WebSocket.OPEN) {
        ws.send(JSON.stringify(jsonData));
    } else {
        messageQueue.push(JSON.stringify(jsonData));
    }
}

function initializeWebSocket() {
    const ipHyperion = '192.168.88.101';
    const puertoHyperion = 8090;

    ws = new WebSocket(`ws://${ipHyperion}:${puertoHyperion}/jsonrpc`);

    ws.onopen = function () {
        log('WebSocket connection opened');
        while (messageQueue.length > 0) {
            ws.send(messageQueue.shift());
        }
    };

    ws.onmessage = function (event) {
        log('Message from server: ' + event.data);
    };

    ws.onerror = function (error) {
        log('WebSocket error: ' + error.message);
    };

    ws.onclose = function () {
        log('WebSocket connection closed');
        // Intentar reconectar después de 1 segundo
        setTimeout(initializeWebSocket, 1000);
    };
}

function sendRandomColor() {
    const ipHyperion = '192.168.88.101';
    const puertoHyperion = 8090;

    const colors = [
        [255, 0, 0],    // Red
        [0, 255, 0],    // Green
        [0, 0, 255],    // Blue
        [255, 255, 0],  // Yellow
        [255, 0, 255],  // Magenta
        [0, 255, 255]   // Cyan
    ];

    const randomColor = colors[Math.floor(Math.random() * colors.length)];

    const jsonData = {
        command: "color",
        color: randomColor,
        duration: 120000,  // 2 minutes
        priority: 20,
        origin: "TizenApp"
    };

    log('Sending color to Hyperion: ' + JSON.stringify(jsonData));

    fetch(`http://${ipHyperion}:${puertoHyperion}/json-rpc`, {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json'
        },
        body: JSON.stringify(jsonData)
    })
    .then(response => response.json())
    .then(data => log('Response from Hyperion: ' + JSON.stringify(data)))
    .catch(error => log('Error sending color to Hyperion: ' + error));
}

// Configuración del reproductor de YouTube
let player;
function onYouTubeIframeAPIReady() {
    player = new YT.Player('player', {
        height: '390',
        width: '640',
        videoId: 'Gt6wKDnG0xA',
        events: {
            'onReady': onPlayerReady,
            'onStateChange': onPlayerStateChange
        }
    });
}

let captureInterval;
function startVideoCapture() {
    if (!player || !player.getIframe) {
        log('YouTube Player not ready');
        return;
    }

    log('Starting video capture');
    const iframe = player.getIframe();
    const videoElement = iframe.contentWindow.document.querySelector('video');
    if (!videoElement) {
        log('Video element not found');
        return;
    }

    const canvas = document.createElement('canvas');
    const context = canvas.getContext('2d');

    canvas.width = 120;
    canvas.height = 120;

    captureInterval = setInterval(() => {
        try {
            context.drawImage(videoElement, 0, 0, canvas.width, canvas.height);
            const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03
            sendToHyperion(imageData);
        } catch (error) {
            log('Error capturing video frame: ' + error);
            clearInterval(captureInterval);
        }
    }, 100); // Capturar cada 100 ms
}

function stopVideoCapture() {
    log('Stopping video capture');
    clearInterval(captureInterval);
}

function onPlayerReady(event) {
    event.target.playVideo();
}

function onPlayerStateChange(event) {
    if (event.data == YT.PlayerState.PLAYING) {
        log('YouTube Player playing');
        startVideoCapture();
    } else {
        log('YouTube Player paused or ended');
        stopVideoCapture();
    }
}

function log(message) {
    const logs = document.getElementById('logs');
    const logMessage = document.createElement('p');
    logMessage.textContent = message;
    logs.appendChild(logMessage);
    console.log(message);
}

index.html:

<!DOCTYPE html>
<html>
<head>
    <title>SamyGrabber</title>
    <style>
        body {
            font-family: Arial, sans-serif;
            background-color: #121212;
            color: white;
            text-align: center;
            padding: 20px;
        }
        #captureArea {
            width: 100%;
            height: 300px;
            border: 1px solid #ccc;
            margin-bottom: 20px;
            display: flex;
            justify-content: center;
            align-items: center;
        }
        #logs {
            background-color: #222;
            padding: 10px;
            border-radius: 5px;
            margin-top: 20px;
            max-height: 200px;
            overflow-y: scroll;
            text-align: left;
        }
        .button-container {
            display: flex;
            justify-content: center;
            gap: 20px;
        }
        button {
            padding: 10px 20px;
            font-size: 16px;
            border: none;
            border-radius: 5px;
            background-color: #6200ee;
            color: white;
            cursor: pointer;
        }
        button:hover {
            background-color: #3700b3;
        }
    </style>
</head>
<body>
    <h1>SamyGrabber</h1>
    <div id="captureArea" contenteditable="true">
        <span>Texto</span>
    </div>
    <div class="button-container">
        <button id="startCapture" tabindex="1">Iniciar Captura</button>
        <button id="remoteButton" tabindex="2">Captura con Control Remoto</button>
        <button id="testImageButton" tabindex="3">Enviar Imagen de Prueba</button>
        <button id="sendColorButton" tabindex="4">Enviar Color Aleatorio</button>
        <button id="startVideoCaptureButton" tabindex="5">Iniciar Captura de Video</button>
        <button id="stopVideoCaptureButton" tabindex="6">Detener Captura de Video</button>
    </div>
    <div id="logs"></div>
    <div id="player"></div>
    <script src="https://www.youtube.com/iframe_api"></script>
    <script src="main.js"></script>
</body>
</html>

Challenges:

1. Web App Limitation: The current solution is limited to capturing content within the web app. My goal is to capture the entire screen content of the TV, including when the web app is not in focus or running.
2. Permission Issues: When attempting to use certain APIs (e.g., captureScreen), I encounter permission errors. This is despite setting the necessary privileges in the config.xml file.
3. Alternative Solutions: I am exploring alternative methods such as using low-level graphics APIs (OpenGL, Vulkan) or other native capabilities to achieve screen capture. However, guidance on the best approach to start with would be highly appreciated. Goals: • I aim to capture the color information from the borders of the TV screen in real-time, regardless of the running application or TV state. • The solution should work continuously and efficiently without requiring constant user intervention. Questions:
4. Best Approach for Screen Capture: What is the recommended method for capturing the entire screen content on a Tizen TV? Are there specific APIs or permissions I need to be aware of?
5. Using Low-level Graphics APIs: How feasible is it to use OpenGL or Vulkan for capturing screen content? Are there any examples or documentation that could help me get started?
6. Handling Permissions: How can I overcome permission issues when trying to access screen content? Are there specific settings or privileges that need to be configured?

Any guidance, documentation, or examples would be immensely helpful. Thank you in advance for your assistance!

Answer 1

1. TV didn't allow capture for other app because of 3rd party copyright issue.
2. We don't support captureScreen API in TV. where is it?
3. For 3rd pary webapp, TV didn't allow background operation. So if you get the APi(current not exist), you can't get it.
4. There was similar case which is Philips Hue Sync. It's not web app and need contact region service PM.
5. Not Exist.
6. Not Allow

Answer 2

1. In general, third-party companies are not allowed to use JavaScript while in the background. Allowing JavaScript behavior in the background requires very high privileges, and if this feature is absolutely necessary, you will need to contact your app manager for assistance.

2. It appears that you are trying to capture a canvas that exists within an app. From your question, it seems that you would also like to capture content that is outside of your app. However, apps are unable to capture anything external to themselves due to policy restrictions. This function is not supported.