How can i get live market data without minimal delay

Question

I need live market data that includes close, low, volume, ... on 5 min timeframe. After each ending, i need this data immediately.

I noticed, that yfinance has ~90s delay before it can providing the latest update.

Answer 1

I am now having a working solution for crypto with python-binance using a custom timing method.

simple_time.py (my timing method)

from datetime import datetime, timedelta


class SimpleTime:
    def __init__(self, hour=0, minute=0, second=0, millisecond=0, microsecond=0):
        self.hour = hour
        self.minute = minute
        self.second = second
        self.millisecond = millisecond
        self.microsecond = microsecond

    def __str__(self):
        return f'{self.hour}:{self.minute}:{self.second}'

    @staticmethod
    def from_datetime(_datetime):
        r_millisecond = _datetime.microsecond % 1000
        millisecond = int((_datetime.microsecond - r_millisecond) / 1000)
        return SimpleTime(_datetime.hour, _datetime.minute, _datetime.second, millisecond, r_millisecond)

    @staticmethod
    def from_microseconds(microseconds):
        assert microseconds <= 8.64e+10
        if not microseconds:
            return SimpleTime()
        r_hour = microseconds % 3.6e+9
        hour = int((microseconds - r_hour) / 3.6e+9)
        r_minute = r_hour % 6e+7
        minute = int((r_hour - r_minute) / 6e+7)
        r_second = r_minute % 1e+6
        second = int((r_minute - r_second) / 1e+6)
        r_millisecond = r_second % 1000
        millisecond = int((r_second - r_millisecond) / 1000)
        return SimpleTime(hour, minute, second, millisecond, int(r_millisecond))

    def to_seconds(self):
        return self.hour * 3600 + self.minute * 60 + self.second + \
               self.millisecond / 1000 + self.microsecond / 1e+6

    def to_microseconds(self):
        return self.hour * 3.6e+9 + self.minute * 6e+7 + self.second * 1e+6 + \
               self.millisecond * 1000 + self.microsecond

    # calculates the simpletime based on the multiply on a given period
    def get_next_period(self, period):
        ms = self.to_microseconds()
        p_ms = period.to_microseconds()
        assert 8.64e+10 % p_ms == 0
        if ms < p_ms:
            # time before period, therefore the period is also the next period time
            return period
        # calculate next period time based on
        # x times the period fits fully into the current time plus the period
        i = int((ms - (ms % p_ms)) / p_ms)
        next_p_ms = i * p_ms + p_ms
        if next_p_ms == 8.64e+10:
            # end of day, return 00:00 instead of 24:00
            return SimpleTime()
        return SimpleTime.from_microseconds(next_p_ms)

    # difference between two times, supports overnight
    def calc_difference(self, simple_time):
        a, b = self.to_microseconds(), simple_time.to_microseconds()
        if a <= b:
            # same day
            diff = b - a
            is_overnight = False
        else:
            # overnight
            diff = 8.64e+10 - a + b
            is_overnight = True
        return SimpleTime.from_microseconds(diff), is_overnight


# this method calculates the next time based on a given period
# e.g. you want the next 5min close datetime
# currently it is 15:1:0
# this method will calculate 15:5:0 as return
def get_next_end_dt(period=SimpleTime(minute=5)):
    dt = datetime.utcnow()
    st = SimpleTime.from_datetime(dt)
    next_p = st.get_next_period(period)
    _, is_overnight = st.calc_difference(next_p)
    if is_overnight:
        dt += timedelta(days=1)
    return datetime(year=dt.year, month=dt.month, day=dt.day,
                    hour=next_p.hour, minute=next_p.minute, second=next_p.second)

klines_client (uses python-finance to fetch data and parses it into DataFrame)

import asyncio
import time
from datetime import datetime
import pandas as pd

from binance import AsyncClient


# based on: https://binance-docs.github.io/apidocs/spot/en/#compressed-aggregate-trades-list
def parse_klines(klines):
    data = [[
        float(kline[1]),
        float(kline[2]),
        float(kline[3]),
        float(kline[4]),
        float(kline[5]),
        datetime.utcfromtimestamp(kline[0] / 1000)
    ] for kline in klines]
    df = pd.DataFrame(data, columns=['Open', 'High', 'Low', 'Close', 'Volume', 'Datetime'])
    df.set_index('Datetime', inplace=True)
    return df


class KlinesClient:
    def __init__(self):
        self.client = None

    async def create_client(self):
        self.client = await AsyncClient.create()

    # this method fetches klines with a target datetime
    # @return is a DataFrame with ['Open', 'High', 'Low', 'Close', 'Volume', 'Datetime'] columns
    # e.g. you want the next 5min close
    # currently it is 15:1:0
    # your end datetime would be 15:5:0
    # as soon as there is an entry that starts at 15:5:0 from the api,
    # this method will return every entries before that
    # therefore is last entries from the returned klines will be the latest 5min close
    async def fetch_klines_with_target_dt(self, end_dt, symbol='BTCUSDT', timeout_s=60,
                                          interval=AsyncClient.KLINE_INTERVAL_5MINUTE, start_str='1 day ago UTC'):
        s = (end_dt - datetime.utcnow()).total_seconds()
        if s > 0:
            await asyncio.sleep(s)
        t_start = time.time()
        while time.time() - t_start < timeout_s:
            klines = await self.client.get_historical_klines(symbol, interval, start_str)
            for i in range(len(klines) - 1, -1, -1):
                dt = datetime.utcfromtimestamp(klines[i][0] / 1000)
                if dt == end_dt:
                    if i == 0:
                        break
                    return parse_klines(klines[:i])
        raise TimeoutError

    async def close_connection(self):
        await self.client.close_connection()

wait_connection.py (method to wait for internet connection)

import socket
import time


def wait_for_internet_connection(cooldown_s=1):
    while True:
        try:
            socket.create_connection(("1.1.1.1", 53))
            return
        except OSError:
            time.sleep(cooldown_s)

main.py (fetching latest close data (5min, 15min, 1h, ...))

import asyncio
import logging

from klines_client import KlinesClient
from simple_time import *
from wait_connection import *


async def main():
    # init
    klines_client = KlinesClient()
    await klines_client.create_client()

    while True:
        # fetch next period
        end_dt = get_next_end_dt()
        logging.info(f'Next close: {end_dt}')
        try:
            df = await klines_client.fetch_klines_with_target_dt(end_dt)
        except Exception as e:
            # error -> await connection and skip this interval
            logging.error(e)
            logging.info('Waiting for connection...')
            wait_for_internet_connection()
            logging.info('Connection ok, skipping this interval due to error')
            continue

        # now u can do some stuff with it
        logging.debug(df.iloc[-1])

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    asyncio.run(main())

Answer 2

There are several ways to get live market data without minimal delay. The most common methods are to use a direct market access (DMA) provider, use a third-party data provider, or use an API.

A direct market access (DMA) provider is the most reliable way to get live market data with minimal delay. These providers are connected to the exchanges’ servers, which allows them to get direct access to the data. This means that the data is very fast and reliable.

Third-party data providers are another option for getting live market data without minimal delay. These providers collect data from multiple sources, including exchanges, and then aggregate it into a single feed. This allows you to get real-time data without having to connect to each exchange directly.

Finally, you can use an API to get live market data without minimal delay. APIs are a great way to get real-time data, as they allow you to access the data from multiple sources, including exchanges, in a single request.