playbooks/roles/filebeat/templates/filebeat-quick-template.yml

@@ -0,0 +1,62 @@
+path.data: {{ work_path }}/data3
+filebeat.registry.path: {{ work_path }}/data3/registry/quick-{{ instance_name }}
+
+queue.mem:
+  events: 512
+  flush.min_events: 128
+  flush.timeout: 5s
+  
+filebeat.inputs:
+{% for log in quicklog %}
+- type: log
+  paths:
+    - {{ log }}
+  fields:
+    source: quick
+  harvester_buffer_size: 1638400
+  tail_files: {{ tail_files | default(true) | string | lower }}
+{% endfor %}
+
+max_procs: {{ max_procs | default(2) }}
+
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+setup.template.name: "quick"
+setup.template.pattern: "quick-*"
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  hosts: {{ quick_es_hosts }}
+  index: "quick-{{ instance_name }}-%{[fields.source]}-%{+yyyy.MM.dd}"
+  pipeline: "quicklog"
+  username: {{ quick_es_user }}
+  password: {{ quick_es_passwd }}
+  bulk_max_size: 1500
+  compression_level: {{ compression_level | default(0) }}
+  worker: 1
+
+
+# ================================= Processors =================================
+processors:
+  - add_host_metadata:
+      when.not.contains.tags: forwarded
+  - add_cloud_metadata: ~
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+  - dissect:
+      tokenizer: "/%{}/%{}/%{}/%{}/%{}/%{zone_id}_%{server_id}_%{log_t}.txt"
+      field: "log.file.path"
+      target_prefix: ""
+  - dissect:
+      tokenizer: "%{date} %{+date->} %{log_id->} %{account_id->} %{role_id->} %{role_name->} %{spid->} %{channel->} %{col1->} %{col2->} %{col3->} %{col4->} %{col5->} %{col6->} %{col7->} %{col8->} %{col9->} %{col10->} %{col11->} %{col12->} %{col13->}"
+      field: "message"
+      target_prefix: ""

playbooks/roles/filebeat/vars/main.yml

@@ -14,6 +14,8 @@ gwlogs:
   - "81"
 mgrlogs: 
   - "/data/game/htdocs/center/manager/storage/cp/logs/*.log"
+quicklog:
+  - "/data/game/sync/quick/filebeat/*.txt"
 config_files:
   - "gateway-nginx"
   - "manager"