Logstash Got response code '401' contacting Elasticsearch at URL 'http://elasticsearch:9200/

Question

I'm trying to setup a ELK stack in docker environment, but I have no luck so far with Logstash. I've got Kibana + Elastic running and connection with one another, but can't get logstash to work. I've tried different configurations and github repos to start with, but nothing seems to work.

I get the following error:

[logstash.licensechecker.licensereader] Attempted to resurrect connection to dead ES instance, but got an error {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError, :message=>"Got response code '401' contacting Elasticsearch at URL 'http://elasticsearch:9200/'"}

Another thing I get is:

[2022-09-12T13:16:18,558][ERROR][logstash.monitoring.internalpipelinesource] Failed to fetch X-Pack information from Elasticsearch. This is likely due to failure to reach a live Elasticsearch cluster.

My docker-compose is:

version: '3.8'
services:
  server:
    build:
      context: ./
      target: dev
    volumes:
      - .:/src
    command: npm run dev
    container_name: server
    ports:
      - "${NODE_PORT}:${NODE_PORT}"
    environment:
      NODE_ENV: development
      DEBUG: nodejs-docker-express:*
    networks:
      - network-name
  kibana:
    build:
      context: ./docker/kibana/
      args:
        ELASTIC_PORT: ${ELASTIC_PORT}
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    container_name: kibana
    ports:
      - "${KIBANA_PORT}:${KIBANA_PORT}"
    volumes:
      - ./docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
    environment:
      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD}
      ELASTIC_PORT: ${ELASTIC_PORT}
    networks:
      - network-name
  elasticsearch:
    build:
      context: ./docker/elasticsearch/
      args: 
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
      - ./docker/elasticsearch/data:/usr/share/elasticsearch/data:z
    container_name: elasticsearch
    ports:
      - "${ELASTIC_PORT}:${ELASTIC_PORT}"
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx512m
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
      discovery.type: single-node
    networks:
      - network-name
  logstash:
    build:
      context: ./docker/logstash/
      args:
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
    container_name: logstash
    environment:
      ELASTIC_VERSION: ${ELASTIC_VERSION}
      ELASTIC_PORT: ${ELASTIC_PORT}
      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD}
    ports:
      - "5044:5044"
      - "50000:50000/tcp"
      - "50000:50000/udp"
      - "9600:9600"
    networks:
      - network-name
networks:
  network-name:
    name: "network-name"
    driver_opts:
      icc: "true"
    driver: bridge

logstash.conf

input {
  tcp {
    port => 5000
    type => syslog
  }
}
## Add your filters / logstash plugins configuration here

output {
    elasticsearch { 
      hosts => ["elasticsearch:9200"]
      user => "elastic"
      password => "dxAq9YLjR23VqcA4oS1I"
      #ssl => true
      ssl_certificate_verification => false
      timeout => 240
      #cacert => '${LS_HOME}/config/ssl/ca.pem'
    }
    stdout { codec => rubydebug }
}

logstash.yml

http.host: "0.0.0.0"
config.reload.automatic: true
log.level: debug

queue.type: persisted
queue.max_bytes: 1gb
queue.checkpoint.writes: 1

ENV file

#ELK CONFIG
ELASTIC_PORT=9200
ELASTIC_PASSWORD=dxAq9YLjR23VqcA4oS1I
ELASTIC_VERSION=8.4.0
#KIBANA
KIBANA_PORT=5601
KIBANA_SYSTEM_PASSWORD=1cDUbBrO8XRosNoayRpy
#BEATS

#LOGSTASH
LOGSTASH_INTERNAL_PASSWORD=C9i0SgSq3loKYbdooKTV

I can't figure out what am I doing wrong. I've generated new password for all the users several times, rebuilded containers and nothing seems to work for logstash :(

Answer 1

Tldr;

The Elasticsearch service is not on the same network.

elasticsearch is on network-name. kibana and logstash are on tower-defense

Then in the environment of elasticsearch service, you refer to ${ELASTIC_PASSWORD:-} for the password of elastic ... isn't it a typo ?

Solution

version: '3.8'
services:
  server:
    build:
      context: ./
      target: dev
    volumes:
      - .:/src
    command: npm run dev
    container_name: server
    ports:
      - "${NODE_PORT}:${NODE_PORT}"
    environment:
      NODE_ENV: development
      DEBUG: nodejs-docker-express:*
    networks:
      - network-name
  kibana:
    build:
      context: ./docker/kibana/
      args:
        ELASTIC_PORT: ${ELASTIC_PORT}
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    container_name: kibana
    ports:
      - "${KIBANA_PORT}:${KIBANA_PORT}"
    volumes:
      - ./docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
    environment:
      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD}
      ELASTIC_PORT: ${ELASTIC_PORT}
    networks:
      - tower-defense
  elasticsearch:
    build:
      context: ./docker/elasticsearch/
      args: 
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
      - ./docker/elasticsearch/data:/usr/share/elasticsearch/data:z
    container_name: elasticsearch
    ports:
      - "${ELASTIC_PORT}:${ELASTIC_PORT}"
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx512m
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
      discovery.type: single-node
    networks:
      - tower-defense
  logstash:
    build:
      context: ./docker/logstash/
      args:
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
    container_name: logstash
    environment:
      ELASTIC_VERSION: ${ELASTIC_VERSION}
      ELASTIC_PORT: ${ELASTIC_PORT}
      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD}
    ports:
      - "5044:5044"
      - "50000:50000/tcp"
      - "50000:50000/udp"
      - "9600:9600"
    networks:
      - tower-defense
networks:
  tower-defense:
    name: "network-name"
    driver_opts:
      icc: "true"
    driver: bridge