OpenSearch Logstash - host install

---

OpenSearch Logstash - host install

 

 

 

 

 

java heap 메모리 설정 (java 설치 선행)

vim /etc/elasticsearch/jvm.options
-Xms4g
-Xmx4g

 

 

logstash 설치

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.10.2-x86_64.rpm
rpm --install logstash-7.10.2-x86_64.rpm

 

 

plugin 설치

cd /usr/share/logstash
bin/logstash-plugin install logstash-filter-json_encode

 

 

설정 샘플 파일

input {
  beats {
    host => "0.0.0.0"
    port => "5044"
  }
}

filter {
  if [log][file][path] {
    mutate{
      copy => {"[log][file][path]" => "srcPath"}
    }
  }

  grok {
    match => ["srcPath","%{GREEDYDATA}/%{GREEDYDATA}/%{GREEDYDATA:filename}\.*"]
    remove_field => [ "srcPath" ]
  }

  mutate {
    split => {"filename" => "."}
    add_field => { "server_type" => "%{[filename][0]}" }
    remove_field => [ "filename" ]
  }

  if [fields][worldId] {
    mutate {
      add_field => {
        "[@metadata][index_name]" => "%{[fields][worldId]}_%{[server_type]}"
      }
      remove_field => [ "[fields][worldId]" ]
    }
  }
  
  else {
    mutate {
      add_field => {
        "[@metadata][index_name]" => "unhandled_files"
      }
    }
  }

  mutate {
    lowercase => "[@metadata][index_name]"
    gsub => [
      "[@metadata][index_name]", "-", ""
    ]
    remove_field => [ "[server_type]" ]
  }

  json{
    source => "[message]"
    target=> "[json]"
  }

  mutate {
    remove_field => ["@version", "@timestamp","host","agent","ecs","input","message"]
  }

  mutate{
    copy => {"[json][level]" => "level"}
    copy => {"[json][timestamp]" => "timestamp"}
    remove_field => [ "[json][level]","[json][timestamp]" ]
  }

  if [json][userId] {
    mutate{
      copy => {"[json][userId]" => "userId"}
      remove_field => [ "[json][userId]" ]
    }
  }
  if [json][typeStr] {
    mutate{
      copy => {"[json][typeStr]" => "typeStr"}
      remove_field => [ "[json][typeStr]" ]
    }
  }
  if [json][mcode] {
    mutate{
      copy => {"[json][mcode]" => "mcode"}
      remove_field => [ "[json][mcode]" ]
    }
  }

  if [json][packetType] {
    mutate{
      copy => {"[json][packetType]" => "packetType"}
      remove_field => [ "[json][packetType]" ]
    }
  }
  if [json][disconnectReason] {
    mutate{
      copy => {"[json][disconnectReason]" => "disconnectReason"}
      remove_field => [ "[json][disconnectReason]" ]
    }
  }

  if [json][origin] and [json][origin][userId]  {
    mutate{
      copy => {"[json][origin][userId]" => "originUserId"}
    }
  }

  if [json][origin] and [json][origin][userIds]  {
    mutate{
      copy => {"[json][origin][userIds]" => "originUserIds"}
    }
  }

  if [json][message] {
    ruby { code => 'case event.get("[json][message]")
                    when String
                      event.tag("messageStr")
                    end'
    }
  }
  if "messageStr" in [tags] {
    mutate {
      copy => {"[json][message]" => "message"}
      remove_field => [ "[json][message]","tags" ]
    }
  }
  if [json][message] {
     mutate{
      copy => {"[json][message]" => "messageObj"}
      remove_field => [ "[json][message]" ]
    }
  }

  if [json][error] {
    ruby { code => 'case event.get("[json][error]")
                    when String
                      event.tag("errorStr")
                    end'
    }
  }
  if "errorStr" in [tags] {
    mutate {
      copy => {"[json][error]" => "throwError"}
      remove_field => [ "[json][error]","tags" ]
    }
  }
  if [json][error] {
    mutate{
      copy => {"[json][error]" => "throwErrorObj"}
      remove_field => [ "[json][error]" ]
    }
  }
  json_encode {
    source => "[json]"
  }

}

output {
  elasticsearch {
    hosts => ["domain:9200"]
    index => "q_%{[@metadata][index_name]}"
  }
}

 

 

 

 

---

by mkdir-chandler

---