Build the EFK system used for simulating logging server on Docker

Question

I want to simulate laravel logging to EFK system server
Base on this, I build up two container. One of laravel project's container. The ohter is EFK system container

but EFK's fluentd does not catch any data or event

---

my container's compose:

version: '3'
services:
  nginx:
    image: nginx:latest
    ports:
      - 8010:80
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:delegated
      - ./server:/var/www/:delegated
    depends_on:
      - php
      - fluentd
    logging:
      driver: "fluentd"
      options:
        fluentd-address: fluentd:24225
        fluentd-async-connect: 'true'
        fluentd-retry-wait: '1s'
        fluentd-max-retries: '30'
        tag: fubo.logger

  php:
    container_name: php-laravel
    build: ./php
    volumes:
      - ./server:/var/www/:delegated

  db:
    build: ./mysql
    volumes:
      - ./mysql/data/:/var/lib/mysql
    ports:
      - 3306:3306

  phpmyadmin:
    image: phpmyadmin/phpmyadmin:latest
    ports:
      - 8811:80
    depends_on:
      - db

  fluentd:
    build: ./fluentd
    volumes:
      - ./fluentd/conf:/fluentd/etc
    ports:
      - "24225:24224"
      - "24225:24224/udp"
    networks:
      - docker-efk_efk_network
networks:
  docker-efk_efk_network:
    external: true

my container's fluent.conf:

<source>
  @type tail
  path /etc/logs/laravel.log
  pos_file /etc/logs/laravel.log.pos
  tag docker.space
  <parse>
    @type json
  </parse>
</source>

<match *.**>
  @type forward
  send_timeout 60s
  recover_wait 10s
  hard_timeout 60s

  <server>
    name dockerSpace
    host docker-efk-fluentd-1
    port 24224
    weight 60
  </server>
</match>

---

EFK's container compose:

version: '3'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1
    container_name: elasticsearch
    restart: unless-stopped
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
    ports:
      - 9200:9200

  kibana:
    image: docker.elastic.co/kibana/kibana:7.15.1
    container_name: kibana
    restart: unless-stopped
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      - I18N_LOCALE=zh-tw
    ports:
      - 5601:5601
    links:
      - elasticsearch

  fluentd:
    build: ./fluentd
    volumes:
      - ./fluentd/conf/:/fluentd/etc/
    links:
      - elasticsearch
    ports:
      - "24224:24224"
      - "24224:24224/udp"
    networks:
      - efk_network
networks:
  efk_network:
    driver: bridge

EFK's container fluent.conf:

<source>
  @type forward
  port 24225
  bind docker-space_fluentd_1
</source>

<match *.**>
  @type copy

  <store>
    @type elasticsearch
    host elasticsearch
    port 9200
    logstash_format true
    logstash_prefix fluentd
    logstash_dateformat %Y%m%d
    include_tag_key true
    type_name access_log
    tag_key @log_name
    flush_interval 1s
  </store>

  <store>
    @type stdout
  </store>
</match>

This is my container networks list:

name	driver	scope
docker-efk_default	bridge	local
docker-efk_efk_network	bridge	local
docker-space_default	bridge	local

What's wrong my understanding?

Answer 1

There are two step to do:

First, ensurce both of container has connected each other. More detail can see this.
How to link multiple docker-compose services via network

Second, modify EFK container's fluentd configuare:

<source>
  @type forward
  bind 0.0.0.0
  port 24224
</source>

<match *.**>
  @type copy

  <store>
    @type elasticsearch
    host elasticsearch
    port 9200
    logstash_format true
    logstash_prefix fluentd
    logstash_dateformat %Y%m%d
    include_tag_key true
    type_name access_log
    tag_key @log_name
    flush_interval 1s
  </store>

  <store>
    @type stdout
  </store>
</match>

And ... it's work.