R.B. wiki

User Tools

Site Tools

Trace:
giganet:sflow

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revision Previous revision
Next revision 		Previous revision
giganet:sflow [2021/08/02 13:10]
rb 		giganet:sflow [2021/08/03 13:28] (current)
rb
Line 1: Line 1:
-====== Схема мониторинга sflow статистики ====== 
- 
-===== Задействованные сервера ===== 
- 
-  * **goflow **- сервер на котором крутится kafka и goflow2 приложение по сбору статистики 
-  * **clickhouse **- сервер с базой данных clickhouse, хранение и обработка статистики 
-===== Задействованные приложения ===== 
- 
-  * **goflow2** 
-  * **kafka** 
-  * **clickhouse** 
 ===== Общая схема мониторинга ===== ===== Общая схема мониторинга =====
  
Line 28: Line 17:
  
 примеры конфигов для свитчей примеры конфигов для свитчей
 +
 +<code>
 +sampling.1G = 8192
 +sampling.10G = 8192
 +sampling.40G = 16384
 +sampling.100G = 32768
 +
 +</code>
  
 ==== goflow2 ==== ==== goflow2 ====
  
-пример настройки goflow2+file /etc/default/goflow2 
 + 
 +<code> 
 +GOFLOW2_ARGS=-format pb -format.protobuf.fixedlen=true -listen=sflow://172.17.172.111:6343 -reuseport=true -transport=kafka -transport.kafka.brokers=172.17.172.111:9092 -transport.kafka.hashing=true -transport.kafka.log.err=true -transport.kafka.topic=sflow -reuseport=true -workers=2 
 + 
 +</code>
  
 ==== kafka ==== ==== kafka ====
  
-пример настройки kafka+== Установка == 
 + 
 +<code> 
 +useradd kafka -m 
 +passwd kafka 
 +su -l kafka 
 +cd 
 +curl https://archive.apache.org/dist/kafka/2.8.0/kafka_2.13-2.8.0.tgz -O kafka_2.13-2.8.0.tgz 
 +tar -xvzf kafka_2.13-2.8.0.tgz 
 +ln -s  /home/kafka/kafka_2.13-2.8.0 /home/kafka/kafka 
 + 
 +</code> 
 + 
 +/home/kafka/kafka/config/server.properties 
 + 
 +<code> 
 +delete.topic.enable=true 
 + 
 +</code> 
 + 
 +/etc/systemd/system/zookeeper.service 
 + 
 +<code> 
 +[Unit] 
 +Requires=network.target remote-fs.target 
 +After=network.target remote-fs.target 
 + 
 +[Service] 
 +Type=simple 
 +User=kafka 
 +ExecStart=/home/kafka/kafka/bin/zookeeper-server-start.sh /home/kafka/kafka/config/zookeeper.properties 
 +ExecStop=/home/kafka/kafka/bin/zookeeper-server-stop.sh 
 +Restart=on-abnormal 
 + 
 +[Install] 
 +WantedBy=multi-user.target 
 + 
 +</code> 
 + 
 +/etc/systemd/system/kafka.service 
 + 
 +<code> 
 +[Unit] 
 +Requires=zookeeper.service 
 +After=zookeeper.service 
 + 
 +[Service] 
 +Type=simple 
 +User=kafka 
 +ExecStart=/bin/sh -c '/home/kafka/kafka/bin/kafka-server-start.sh /home/kafka/kafka/config/server.properties> /home/kafka/kafka/kafka.log 2>&1' 
 +ExecStop=/home/kafka/kafka/bin/kafka-server-stop.sh 
 +Restart=on-abnormal 
 + 
 +[Install] 
 +WantedBy=multi-user.target 
 + 
 +</code> 
 + 
 +Запускаем сервисы 
 + 
 +<code> 
 +systemctl daemon-reload 
 +systemctl start kafka 
 +systemctl enable kafka 
 + 
 +</code> 
 + 
 +Работаем с kafka 
 + 
 +<code> 
 +/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --list 
 +/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --describe "test-1" 
 +/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --delete --topic test-1 
 +/home/kafka/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --config compression.type=producer --config cleanup.policy=delete --topic sflow --partitions 10 --replication-factor 1 
 + 
 +</code>
  
 ==== clickhouse ==== ==== clickhouse ====
  
-=== таблица kafka ===+=== дополнительная информация ===
  
 +[[https://github.com/netsampler/goflow2/blob/main/pb/flow.proto|https://github.com/netsampler/goflow2/blob/main/pb/flow.proto]] \\
 +[[https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/|https://clickhouse.tech/docs/ru/engines/table-engines/integrations/kafka/]] \\
 +[[https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip/|https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip/]] \\
 +[[https://gist.github.com/sanchezzzhak/511fd140e8809857f8f1d84ddb937015|https://gist.github.com/sanchezzzhak/511fd140e8809857f8f1d84ddb937015]]
 +
 +=== таблица kafka ===
 <code> <code>
 +
 CREATE TABLE traffic.kafka CREATE TABLE traffic.kafka
 ( (
Line 121: Line 205:
  
 </code> </code>
 +
 ==== таблица raw ==== ==== таблица raw ====
  
Line 202: Line 287:
 PARTITION BY toYYYYMMDD(Date) PARTITION BY toYYYYMMDD(Date)
 ORDER BY Datetime ORDER BY Datetime
-TTL Date + toIntervalDay(7)+TTL Date   toIntervalDay(7)
 SETTINGS index_granularity = 8192 SETTINGS index_granularity = 8192
  
 </code> </code>
 +
 ==== таблица kafka_to_raw ==== ==== таблица kafka_to_raw ====
  
Line 347: Line 433:
     MPLSLastLabel     MPLSLastLabel
 FROM traffic.kafka FROM traffic.kafka
 +
 +</code>
 +
 +==== Словари ====
 +
 +protocols - словарь для текстового представления протоколов
 +
 +<code>
 +CREATE DICTIONARY dictionaries.protocols (
 +    `proto` UInt8,
 +    `name` String,
 +    `description` String
 +)
 +PRIMARY KEY proto
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/protocols.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(FLAT())
 +
 +</code>
 +
 +asnv4 - конвертация ipv4 к ASN
 +
 +<code>
 +CREATE DICTIONARY dictionaries.asnv4
 +(
 +    `network` String,
 +    `asn` UInt32,
 +    `aso` String DEFAULT '??'
 +)
 +PRIMARY KEY network
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/ip-asn-ipv4.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(IP_TRIE())
 +
 +</code>
 +
 +asnv6 - конвертация ipv6 к ASN
 +
 +<code>
 +CREATE DICTIONARY dictionaries.asnv6
 +(
 +    `network` String,
 +    `asn` UInt32,
 +    `aso` String DEFAULT '??'
 +)
 +PRIMARY KEY network
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/ip-asn-ipv6.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(IP_TRIE())
 +
 +</code>
 +
 +contryen
 +
 +<code>
 +CREATE DICTIONARY dictionaries.contryen
 +(
 +    `geoname_id` id,
 +    `locale_code` String,
 +    `continent_code` String,
 +    `continent_name` String,
 +    `country_iso_code` String,
 +    `country_name` String,
 +    `is_in_european_union` UInt8
 +)
 +PRIMARY KEY geoname_id
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/coutry-location.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(HASHED())
 +
 +</code>
 +
 +contryv4
 +
 +<code>
 +CREATE DICTIONARY dictionaries.contryv4
 +(
 +    `network` String,
 +    `geoname_id` UInt32 DEFAULT 0,
 +    `registered_country_geoname_id` UInt32 DEFAULT 0,
 +    `represented_country_geoname_id` UInt32 DEFAULT 0,
 +    `is_anonymous_proxy` UInt8 DEFAULT 0,
 +    `is_satellite_provider` UInt8 DEFAULT 0
 +)
 +PRIMARY KEY network
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/country-ipv4.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(IP_TRIE())
 +
 +</code>
 +
 +contryv6
 +
 +<code>
 +CREATE DICTIONARY dictionaries.contryv6
 +(
 +    `network` String,
 +    `geoname_id` UInt32 DEFAULT 0,
 +    `registered_country_geoname_id` UInt32 DEFAULT 0,
 +    `represented_country_geoname_id` UInt32 DEFAULT 0,
 +    `is_anonymous_proxy` UInt8 DEFAULT 0,
 +    `is_satellite_provider` UInt8 DEFAULT 0
 +)
 +PRIMARY KEY network
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/country-ipv6.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(IP_TRIE())
 +
 +</code>
 +
 +switches
 +
 +<code>
 +CREATE DICTIONARY dictionaries.switches
 +(
 +    `ip` String,
 +    `name` String
 +)
 +PRIMARY KEY ip
 +SOURCE(FILE(PATH '/var/lib/clickhouse/user_files/switches.csv' FORMAT 'CSVWithNames'))
 +LIFETIME(MIN 0 MAX 3600)
 +LAYOUT(COMPLEX_KEY_HASHED())
  
 </code> </code>
  
  
giganet/sflow.1627899022.txt.gz · Last modified: 2021/08/02 13:10 by rb

Page Tools

Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki