{"type":"doc","content":[{"type":"layoutSection","marks":[{"type":"breakout","attrs":{"mode":"full-width"}}],"content":[{"type":"layoutColumn","attrs":{"width":66.66},"content":[{"type":"paragraph","content":[{"text":"Elasticsearch, Fluentd, and Kibana (EFK) is a set of logging solution. It helps you view the application logs. This section guides you on installing EFK. Before you deploy the EFK, make sure that you have met the following prerequisites.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Prerequisites","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kubernetes 1.16+","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Helm 3+","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Elasticsearch","type":"text"}]},{"type":"paragraph","content":[{"text":"Elasticsearch maintains the index of all the logs for each microservice pushed to it by Fluentd.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Prerequisites ","type":"text"}]},{"type":"paragraph","content":[{"text":"Minimum cluster requirements include the following to run the Elasticsearch helm chart with default settings. All of these settings are configurable.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Three Kubernetes nodes to respect the default \"hard\" affinity settings.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"1GB of RAM for the JVM heap.","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Installation","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"To deploy Elasticsearch, you need to follow the steps as given below.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the following command to add the Elasticsearch helm chart from the Elasticsearch helm repository.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo add elastic https://helm.elastic.co/","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update the Helm repository by running the following command.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo update","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the helm install command as shown below to deploy Elasticsearch.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm install elasticsearch elastic/elasticsearch -n ","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Fluentd","type":"text"}]},{"type":"paragraph","content":[{"text":"Fluentd collects the Adeptia application logs from Kubernetes and pushes them into Elasticsearch. ","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Installation","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"To deploy Fluentd, you need to follow the steps as given below.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the following command to add the Fluentd helm chart from the Fluentd helm repository.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo add fluent https://fluent.github.io/helm-charts","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update the Helm repository by running the following command.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo update","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the helm install command as shown below to deploy Fluentd .","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm install fluentd fluent/fluentd -n ","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Once you have installed Fluentd, update your Fluentd ConfigMap using the Fluentd ConfigMap configuration given in the file below. This defines how to parse application logs and how to store the logs into Elasticsearch.","type":"text"}]},{"type":"heading","attrs":{"level":5},"content":[{"text":"Fluentd ConfigMap","type":"text","marks":[{"type":"strong"}]}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: fluentd-forwarder-cm\n namespace: fluentd\ndata:\n fluentd.conf: |-\n # Ignore fluentd own events\n \n @type null\n \n\n\n @include source.conf\n @include elastic-output.conf\n\n source.conf: |-\n # HTTP input for the liveness and readiness probes\n \n @type http\n port 9880\n \n \n @include webapp-gateway-input.conf\n @include webrunner-input.conf\n @include portal-input.conf\n @include event-input.conf\n @include runtime-input.conf\n \n webapp-gateway-input.conf: |-\n \n @type tail\n tag service.webapp-gateway\n path /var/log/containers/*webapp-gateway*.log\n pos_file /var/log/webapp-gateway-containers.log.pos\n read_from_head true\n @include source-parser.conf\n \n \n @id service.webapp-gateway\n @include exception-detector.conf\n \n @include concat-filter.conf\n \n @id webapp-gateway_kubernetes_metadata-filter\n @include kubernetes_metadata-filter.conf\n \n \n @id webapp-gateway_log-field-parser\n @include log-field-parser.conf\n \n \n portal-input.conf: |-\n \n @type tail\n tag service.portal\n path /var/log/containers/*portal*.log\n pos_file /var/log/portal-containers.log.pos\n read_from_head true\n @include source-parser.conf\n \n \n @id service.portal\n @include exception-detector.conf\n \n @include concat-filter.conf\n \n @id portal_kubernetes_metadata-filter\n @include kubernetes_metadata-filter.conf\n \n \n @id portal_log-field-parser\n @include log-field-parser.conf\n \n \n webrunner-input.conf: |-\n \n @type tail\n tag service.webrunner\n path /var/log/containers/*web-runner*.log\n pos_file /var/log/webrunner-containers.log.pos\n read_from_head true\n @include source-parser.conf\n \n @include concat-filter.conf\n \n @id webrunner_kubernetes_metadata-filter\n @include kubernetes_metadata-filter.conf\n \n \n @id webrunner_log-field-parser\n @include log-field-parser.conf\n \n\n event-input.conf: |-\n \n @type tail\n tag service.event\n path /var/log/containers/*event*.log\n pos_file /var/log/event-containers.log.pos\n read_from_head true\n @include source-parser.conf\n \n \n @id service.event\n @include exception-detector.conf\n \n @include concat-filter.conf\n \n @id event_kubernetes_metadata-filter\n @include kubernetes_metadata-filter.conf\n \n \n @id event_log-field-parser\n @include log-field-parser.conf\n \n\n runtime-input.conf: |-\n \n @type tail\n tag service.runtime\n path /var/log/containers/*runtime*.log\n pos_file /var/log/runtime-containers.log.pos\n read_from_head true\n @include source-parser.conf\n \n \n @id service.runtime\n @include exception-detector.conf\n \n @include concat-filter.conf\n \n @id runtime_kubernetes_metadata-filter\n @include kubernetes_metadata-filter.conf\n \n \n @id runtime_log-field-parser\n @include log-field-parser.conf\n \n \n \n exception-detector.conf: |-\n # Detect exceptions in the log output and forward them as one log entry.\n @type detect_exceptions\n remove_tag_prefix service\n message log\n stream stream\n multiline_flush_interval 5\n max_bytes 500000\n max_lines 1000\n\n concat-filter.conf: |-\n \n # @id filter_concat\n @type concat\n key log\n use_first_timestamp true\n multiline_end_regexp /\\n$/\n separator \"\"\n \n\n kubernetes_metadata-filter.conf: |-\n # Enriches records with Kubernetes metadata\n @type kubernetes_metadata\n # skip_namespace_metadata true\n # skip_master_url true\n # skip_labels false\n # skip_container_metadata false\n\n log-field-parser.conf: |-\n @type parser\n key_name log\n reserve_time true\n reserve_data true\n remove_key_name_field true\n \n @type multi_format\n \n format json\n \n \n format none\n \n \n \n source-parser.conf: |-\n \n @type multi_format\n \n format json\n time_key time\n #time_format %Y-%m-%dT%H:%M:%S.%NZ\n \n \n format /^(?.+) (?stdout|stderr) [^ ]* (?.*)$/\n time_format %Y-%m-%dT%H:%M:%S.%N%:z\n \n \n elastic-output.conf: |-\n \n @include elastic-search.conf\n index_name Webapp Gateway\n \n \n @include elastic-search.conf\n index_name Portal\n \n \n @include elastic-search.conf\n index_name WebRunner\n \n \n @include elastic-search.conf\n index_name Event\n \n \n @include elastic-search.conf\n index_name Runtime\n \n \n elastic-search.conf: |-\n @type elasticsearch\n host \"#{ENV['FLUENT_ELASTICSEARCH_HOST'] || 'elasticsearch-master.logs.svc.cluster.local'}\"\n port \"#{ENV['FLUENT_ELASTICSEARCH_PORT'] || '9200'}\"\n include_tag_key true","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Kibana","type":"text"}]},{"type":"paragraph","content":[{"text":"Kibana is the graphical interface where you can view the microservices logs available in Elasticsearch.","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Installation","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"To deploy Kibana, you need to follow the steps as given below.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the following command to add the Kibana helm chart from the Kibana helm repository.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo add elastic https://helm.elastic.co","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Update the Helm repository by running the following command.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm repo update","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Run the helm install command as shown below to deploy Kibana.","type":"text"}]},{"type":"codeBlock","attrs":{"language":"css"},"content":[{"text":"helm install kibana elastic/kibana -n ","type":"text"}]}]}]}]},{"type":"layoutColumn","attrs":{"width":33.33},"content":[{"type":"bodiedExtension","attrs":{"extensionType":"com.atlassian.confluence.macro.core","extensionKey":"panel","parameters":{"macroParams":{"borderStyle":{"value":"solid"},"title":{"value":"You may be interested in..."}},"macroMetadata":{"macroId":{"value":"ff4edf13-15d8-41c0-bfb3-926e670f91c8"},"schemaVersion":{"value":"1"},"title":"Panel"}}},"content":[{"type":"heading","attrs":{"level":5},"content":[{"text":"What is new","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/AC41/pages/15994943"}}]},{"type":"hardBreak"},{"text":"Application architecture","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/AC41/pages/15994937"}}]},{"type":"hardBreak"},{"text":"Key terms","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/AC41/pages/15994930"}}]},{"type":"hardBreak"},{"text":"Install","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/AC41/pages/15994976"}}]},{"type":"hardBreak"},{"text":"Usage scenarios","type":"text","marks":[{"type":"link","attrs":{"href":"/wiki/spaces/AC41/pages/15994946"}}]}]}]}]}]}],"version":1}

Browser not supported