Differences

This shows you the differences between two versions of the page.

--- giganet:sflow [2021/08/02 13:18]
rb
+++ giganet:sflow [2021/08/03 13:28] (current)
rb
@@ Line 1: / Line 1: @@
-====== Схема мониторинга sflow статистики ======
-===== Задействованные сервера =====
-  * **goflow **- сервер на котором крутится kafka и goflow2 приложение по сбору статистики
-  * **clickhouse **- сервер с базой данных clickhouse, хранение и обработка статистики
-===== Задействованные приложения =====
-  * **goflow2**
-  * **kafka**
-  * **clickhouse**
 ===== Общая схема мониторинга =====
@@ Line 28: / Line 17: @@
 примеры конфигов для свитчей
+<code>
+sampling.1G = 8192
+sampling.10G = 8192
+sampling.40G = 16384
+sampling.100G = 32768
+</code>
 ==== goflow2 ====
-пример настройки goflow2
+file /etc/default/goflow2
+<code>
+GOFLOW2_ARGS=-format pb -format.protobuf.fixedlen=true -listen=sflow://172.17.172.111:6343 -reuseport=true -transport=kafka -transport.kafka.brokers=172.17.172.111:9092 -transport.kafka.hashing=true -transport.kafka.log.err=true -transport.kafka.topic=sflow -reuseport=true -workers=2
+</code>
 ==== kafka ====
-пример настройки kafka
+== Установка ==
+<code>
+useradd kafka -m
+passwd kafka
+su -l kafka
+cd
+curl https://archive.apache.org/dist/kafka/2.8.0/kafka_2.13-2.8.0.tgz -O kafka_2.13-2.8.0.tgz
+tar -xvzf kafka_2.13-2.8.0.tgz
+ln -s  /home/kafka/kafka_2.13-2.8.0 /home/kafka/kafka
+</code>
+/home/kafka/kafka/config/server.properties
+<code>
+delete.topic.enable=true
+</code>
+/etc/systemd/system/zookeeper.service
+<code>
+[Unit]
+Requires=network.target remote-fs.target
+After=network.target remote-fs.target
+[Service]
+Type=simple
+User=kafka
+ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties
+ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh
+Restart=on-abnormal
+[Install]
+WantedBy=multi-user.target
+</code>
+/etc/systemd/system/kafka.service
+<code>
+[Unit]
+Requires=zookeeper.service
+After=zookeeper.service
+[Service]
+Type=simple
+User=kafka
+ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties> /home/kafka/kafka/kafka.log 2>&1'
+ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh
+Restart=on-abnormal
+[Install]
+WantedBy=multi-user.target
+</code>
+Запускаем сервисы
+<code>
+systemctl daemon-reload
+systemctl start kafka
+systemctl enable kafka
+</code>
+Работаем с kafka
+<code>
+/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --list
+/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --describe "test-1"
+/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic test-1
+/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --config compression.type=producer --config cleanup.policy=delete --topic sflow --partitions 10 --replication-factor 1
+</code>
 ==== clickhouse ====
-=== таблица kafka ===
+=== дополнительная информация ===
+[[https://github.com/netsampler/goflow2/blob/main/pb/flow.proto|https://github.com/netsampler/goflow2/blob/main/pb/flow.proto]] \\
+[[https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/|https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/]] \\
+[[https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip/|https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip/]] \\
+[[https://gist.github.com/sanchezzzhak/511fd140e8809857f8f1d84ddb937015|https://gist.github.com/sanchezzzhak/511fd140e8809857f8f1d84ddb937015]]
+=== таблица kafka ===
 <code>
 CREATE TABLE traffic.kafka
 (
@@ Line 203: / Line 287: @@
 PARTITION BY toYYYYMMDD(Date)
 ORDER BY Datetime
-TTL Date + toIntervalDay(7)
+TTL Date   toIntervalDay(7)
 SETTINGS index_granularity = 8192
@@ Line 351: / Line 435: @@
 </code>
 ==== Словари ====
@@ Line 365: / Line 450: @@
 LIFETIME(MIN 0 MAX 3600)
 LAYOUT(FLAT())
-</code>asnv4 - конвертация ipv4 к ASN
+</code>
+asnv4 - конвертация ipv4 к ASN
 <code>

R.B. wiki

User Tools

Site Tools

Differences

Page Tools