From c8e9cc2cc93ae9f0d15d5a9359c7310915aa3de6 Mon Sep 17 00:00:00 2001 From: Eliezer Croitoru Date: Tue, 18 Jun 2024 18:53:38 +0300 Subject: [PATCH] 1 --- .gitignore | 1 + cron/Dockerfile | 19 ++++ cron/bin/dump-registry-targets.sh | 15 +++ cron/bin/dump-targets-by-target-type.rb | 56 ++++++++++ cron/cron | 5 + docker-compose.yml | 51 +++++++++ etc/blackbox.yml | 62 +++++++++++ etc/prometheus.yml | 78 ++++++++++++++ registry-dump.sql | 138 ++++++++++++++++++++++++ 9 files changed, 425 insertions(+) create mode 100644 .gitignore create mode 100644 cron/Dockerfile create mode 100755 cron/bin/dump-registry-targets.sh create mode 100755 cron/bin/dump-targets-by-target-type.rb create mode 100644 cron/cron create mode 100644 docker-compose.yml create mode 100644 etc/blackbox.yml create mode 100644 etc/prometheus.yml create mode 100644 registry-dump.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0eaebf1 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +mysql diff --git a/cron/Dockerfile b/cron/Dockerfile new file mode 100644 index 0000000..d8e48b1 --- /dev/null +++ b/cron/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:24.04 + +RUN apt update && \ + apt install -y cron ruby ruby-mysql2 && \ + apt clean all + +COPY cron /etc/cron.d/cron +COPY bin/dump-registry-targets.sh /usr/local/bin/dump-registry-targets.sh +COPY bin/dump-targets-by-target-type.rb /usr/local/bin/dump-targets-by-target-type.rb + +RUN chmod 0644 /etc/cron.d/cron && \ + chmod +x /usr/local/bin/dump-targets-by-target-type.rb && \ + chmod +x /usr/local/bin/dump-registry-targets.sh && \ + crontab /etc/cron.d/cron && \ + ln -s /dev/stdout /var/log/cron && \ + mkdir -p /var/www/html/blackbox && \ + chmod a+wrx /var/www/html/blackbox + +CMD ["cron", "-f"] diff --git a/cron/bin/dump-registry-targets.sh b/cron/bin/dump-registry-targets.sh new file mode 100755 index 0000000..cdb12e6 --- /dev/null +++ b/cron/bin/dump-registry-targets.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +array=( blackbox-tcp_connect_tls blackbox-tcp blackbox-http_2xx_3xx_403 blackbox-icmp node_exporter mysqld_exporter ) + +for i in "${array[@]}" +do + RES=$(ruby /usr/local/bin/dump-targets-by-target-type.rb ${i}) + echo "${RES}" > "/var/www/html/blackbox/${i}.json.in" + diff "/var/www/html/blackbox/${i}.json" "/var/www/html/blackbox/${i}.json.in" >/dev/null + if [ "$?" -gt "0" ];then + mv -vf "/var/www/html/blackbox/${i}.json.in" "/var/www/html/blackbox/${i}.json" + else + rm -vf "/var/www/html/blackbox/${i}.json.in" + fi +done diff --git a/cron/bin/dump-targets-by-target-type.rb b/cron/bin/dump-targets-by-target-type.rb new file mode 100755 index 0000000..e80dd28 --- /dev/null +++ b/cron/bin/dump-targets-by-target-type.rb @@ -0,0 +1,56 @@ +#!/usr/bin/env ruby + +require "mysql2" +require "json" + +target_type = ARGV[0] + +client = Mysql2::Client.new(:host => "localhost", :username => "prom", :password => "prom", :database => "registry") +query = "SELECT t.target_id , t.target, tl.label_key,tl.label_value FROM targets t +JOIN targets_lables tl ON t.target_id = tl.target_id +WHERE t.y = 1 AND tl.target_id IN (SELECT t.target_id FROM targets t +JOIN targets_lables tl ON t.target_id = tl.target_id +WHERE t.y = 1 AND tl.label_key = 'target_type' AND tl.label_value = ?);" + +statement = client.prepare(query) + +targets = {} +results = statement.execute(target_type) + +results.each do |row| + if targets[row["target"]].nil? + targets[row["target"]] = {} + + case row["label_key"] + when 'target_type' + if row["label_value"] == target_type + targets[row["target"]][row["label_key"]] = row["label_value"] + end + else + targets[row["target"]][row["label_key"]] = row["label_value"] + end + + else + case row["label_key"] + when 'target_type' + if row["label_value"] == target_type + targets[row["target"]][row["label_key"]] = row["label_value"] + end + else + targets[row["target"]][row["label_key"]] = row["label_value"] + end + + end +end + +client.close + +def dump_targets(targets) + res = [] + targets.each_key do |key| + res << { "targets" => [ key ], "labels" => targets[key] } + end + puts JSON.pretty_generate(res) +end + +dump_targets(targets) diff --git a/cron/cron b/cron/cron new file mode 100644 index 0000000..8a37c6f --- /dev/null +++ b/cron/cron @@ -0,0 +1,5 @@ +SHELL=/bin/bash +PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin +MAILTO=root + +*/1 * * * * root bash /usr/local/bin/dump-registry-targets.sh >/dev/null 2>&1 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f594ff7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +services: + prometheus: + image: prom/prometheus:latest + restart: always + ports: + - 9090:9090 + volumes: + - ./etc:/etc/prometheus + - prometheus-data:/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + + web: + image: nginx:latest + ports: + - 8080:80 + volumes: + - /var/www/html:/usr/share/nginx/html + + blackbox: + image: prom/blackbox-exporter:latest + restart: always + ports: + - 9115:9115 + volumes: + - ./etc/blackbox.yml:/etc/blackbox_exporter/config.yml:ro + + db: + image: mariadb:lts + restart: always + environment: + MARIADB_ROOT_PASSWORD: example + MARIADB_DATABASE: registry + MARIADB_USER: prom + MARIADB_PASSWORD: prom + ports: + - 3306:3306 + volumes: + - ./mysql:/var/lib/mysql + - ./registry-dump.sql:/docker-entrypoint-initdb.d/dump.sql + + cron: + image: mycron + build: + context: ./cron + dockerfile: Dockerfile + volumes: + - /var/www/html:/var/www/html + + +volumes: + prometheus-data: diff --git a/etc/blackbox.yml b/etc/blackbox.yml new file mode 100644 index 0000000..b4c4687 --- /dev/null +++ b/etc/blackbox.yml @@ -0,0 +1,62 @@ +modules: + http_2xx: + prober: http + + http_2xx_3xx_403: + prober: http + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [ 200, 204, 301,302,307,308,403] + no_follow_redirects: false + + http_post_2xx: + prober: http + http: + method: POST + tcp_connect: + prober: tcp + pop3s_banner: + prober: tcp + tcp: + query_response: + - expect: "^+OK" + tls: true + tls_config: + insecure_skip_verify: false + grpc: + prober: grpc + grpc: + tls: true + preferred_ip_protocol: "ip4" + grpc_plain: + prober: grpc + grpc: + tls: false + service: "service1" + ssh_banner: + prober: tcp + tcp: + query_response: + - expect: "^SSH-2.0-" + - send: "SSH-2.0-blackbox-ssh-check" + + irc_banner: + prober: tcp + tcp: + query_response: + - send: "NICK prober" + - send: "USER prober prober prober :prober" + - expect: "PING :([^ ]+)" + send: "PONG ${1}" + - expect: "^:[^ ]+ 001" + icmp: + prober: icmp + icmp_ttl5: + prober: icmp + timeout: 5s + icmp: + ttl: 5 + tcp_connect_tls: + prober: tcp + tcp: + tls: true diff --git a/etc/prometheus.yml b/etc/prometheus.yml new file mode 100644 index 0000000..d36e6fa --- /dev/null +++ b/etc/prometheus.yml @@ -0,0 +1,78 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: 'blackbox-tcp' + metrics_path: /probe + params: + module: [tcp_connect] + scrape_interval: 30s + scrape_timeout: 20s + http_sd_configs: + - url: "http://web/blackbox/blackbox-tcp.json" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 # Blackbox exporter. + + - job_name: 'blackbox-icmp' + metrics_path: /probe + params: + module: [icmp] + scrape_interval: 30s + scrape_timeout: 20s + http_sd_configs: + - url: "http://web/blackbox/blackbox-icmp.json" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + + - source_labels: [__param_target] + target_label: instance + + - source_labels: [hostname] + target_label: __param_target + + - target_label: __address__ + replacement: blackbox:9115 # Blackbox exporter. + + - job_name: 'blackbox-http_2xx_3xx_403' + metrics_path: /probe + params: + module: [http_2xx_3xx_403] + scrape_interval: 30s + scrape_timeout: 20s + http_sd_configs: + - url: "http://web/blackbox/blackbox-http_2xx_3xx_403.json" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 # Blackbox exporter. + + - job_name: 'blackbox-tcp_connect_tls' + metrics_path: /probe + params: + module: [tcp_connect_tls] + scrape_interval: 30s + scrape_timeout: 20s + http_sd_configs: + - url: "http://web/blackbox/blackbox-tcp_connect_tls.json" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 # Blackbox exporter. diff --git a/registry-dump.sql b/registry-dump.sql new file mode 100644 index 0000000..cd40b5f --- /dev/null +++ b/registry-dump.sql @@ -0,0 +1,138 @@ +-- MariaDB dump 10.19 Distrib 10.5.16-MariaDB, for Linux (x86_64) +-- +-- Host: localhost Database: registry +-- ------------------------------------------------------ +-- Server version 10.5.16-MariaDB + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8mb4 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +-- +-- Table structure for table `hosts` +-- + +DROP TABLE IF EXISTS `hosts`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `hosts` ( + `host_id` int(11) NOT NULL AUTO_INCREMENT, + `hostname` varchar(255) NOT NULL, + PRIMARY KEY (`host_id`), + UNIQUE KEY `hosts_UN` (`hostname`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `hosts` +-- + +LOCK TABLES `hosts` WRITE; +/*!40000 ALTER TABLE `hosts` DISABLE KEYS */; +/*!40000 ALTER TABLE `hosts` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `labels` +-- + +DROP TABLE IF EXISTS `labels`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `labels` ( + `label_id` int(11) NOT NULL AUTO_INCREMENT, + `host_id` int(11) NOT NULL, + `label_name` text NOT NULL, + `label_value` text NOT NULL, + `created_at` timestamp NOT NULL DEFAULT current_timestamp(), + `last_modified` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp(), + PRIMARY KEY (`label_id`), + KEY `labels_FK` (`host_id`), + CONSTRAINT `labels_FK` FOREIGN KEY (`host_id`) REFERENCES `hosts` (`host_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `labels` +-- + +LOCK TABLES `labels` WRITE; +/*!40000 ALTER TABLE `labels` DISABLE KEYS */; +/*!40000 ALTER TABLE `labels` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `targets` +-- + +DROP TABLE IF EXISTS `targets`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `targets` ( + `target_id` int(11) NOT NULL AUTO_INCREMENT, + `target` varchar(255) NOT NULL, + `created_at` timestamp NOT NULL DEFAULT current_timestamp(), + `modified_at` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp(), + `y` int(11) NOT NULL DEFAULT 1, + PRIMARY KEY (`target_id`), + UNIQUE KEY `targets_UN` (`target`) +) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `targets` +-- + +LOCK TABLES `targets` WRITE; +/*!40000 ALTER TABLE `targets` DISABLE KEYS */; +INSERT INTO `targets` VALUES (3,'expired.badssl.com:443','2023-06-24 14:58:41','2023-06-24 18:31:00',0),(4,'self-signed.badssl.com:443','2023-06-24 17:25:25','2023-06-24 18:31:00',0),(5,'8.8.8.8:443','2023-06-24 18:28:57',NULL,1),(6,'8.8.4.4:443','2023-06-24 18:28:57',NULL,1); +/*!40000 ALTER TABLE `targets` ENABLE KEYS */; +UNLOCK TABLES; + +-- +-- Table structure for table `targets_lables` +-- + +DROP TABLE IF EXISTS `targets_lables`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!40101 SET character_set_client = utf8 */; +CREATE TABLE `targets_lables` ( + `label_id` int(11) NOT NULL AUTO_INCREMENT, + `target_id` int(11) NOT NULL, + `label_key` text NOT NULL, + `label_value` text NOT NULL, + `created_at` timestamp NOT NULL DEFAULT current_timestamp(), + `modified_at` timestamp NULL DEFAULT NULL ON UPDATE current_timestamp(), + PRIMARY KEY (`label_id`), + KEY `targets_lables_FK` (`target_id`), + CONSTRAINT `targets_lables_FK` FOREIGN KEY (`target_id`) REFERENCES `targets` (`target_id`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8mb4; +/*!40101 SET character_set_client = @saved_cs_client */; + +-- +-- Dumping data for table `targets_lables` +-- + +LOCK TABLES `targets_lables` WRITE; +/*!40000 ALTER TABLE `targets_lables` DISABLE KEYS */; +INSERT INTO `targets_lables` VALUES (5,3,'hostname','expired.badssl.com','2023-06-24 14:58:56','2023-06-24 17:48:04'),(6,3,'target_type','blackbox-tcp_connect_tls','2023-06-24 14:59:36','2023-06-24 17:45:15'),(8,4,'target_type','blackbox-tcp_connect_tls','2023-06-24 17:25:58','2023-06-24 17:45:15'),(9,4,'hostname','self-signed.badssl.com','2023-06-24 17:26:01','2023-06-24 17:48:04'),(10,3,'datacenter','ASN','2023-06-24 17:48:36',NULL),(11,4,'datacenter','ASN','2023-06-24 17:48:36',NULL),(12,4,'target_type','blackbox-tcp','2023-06-24 17:50:47','2023-06-24 17:51:30'),(13,3,'target_type','blackbox-tcp','2023-06-24 17:50:47','2023-06-24 17:51:30'),(14,4,'target_type','blackbox-http_2xx_3xx_403','2023-06-24 17:51:30',NULL),(15,3,'target_type','blackbox-http_2xx_3xx_403','2023-06-24 17:51:30',NULL),(16,4,'target_type','blackbox-icmp','2023-06-24 17:52:20',NULL),(17,3,'target_type','blackbox-icmp','2023-06-24 17:52:20',NULL),(18,4,'target_type','blackbox-tcp','2023-06-24 17:52:53',NULL),(19,3,'target_type','blackbox-tcp','2023-06-24 17:52:53',NULL),(20,5,'target_type','blackbox-icmp','2023-06-24 18:29:42',NULL),(21,5,'hostname','8.8.8.8','2023-06-24 18:29:42',NULL),(22,6,'target_type','blackbox-icmp','2023-06-24 18:29:42',NULL),(23,6,'hostname','8.8.4.4','2023-06-24 18:29:42',NULL); +/*!40000 ALTER TABLE `targets_lables` ENABLE KEYS */; +UNLOCK TABLES; +/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; + +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; + +-- Dump completed on 2023-06-24 18:32:06