From 305aecef208e6c5e78c90f804d6732184da6b84e Mon Sep 17 00:00:00 2001 From: root Date: Mon, 12 Feb 2024 01:42:36 +0200 Subject: [PATCH] 1 --- Makefile | 43 +++++ campus-blacklist-check.ini | 15 ++ campus-blacklist-check.service | 14 ++ campus-testing-blacklist-check.ini | 15 ++ campus-testing-blacklist-check.service | 13 ++ campus-testing-whitelist-check.ini | 15 ++ campus-testing-whitelist-check.service | 14 ++ campus-whitelist-check.ini | 15 ++ campus-whitelist-check.service | 14 ++ domain-to-search.rb | 22 +++ dst-domain-match-check.rb | 228 +++++++++++++++++++++++++ dst-total-block-check.ini | 15 ++ dst-total-block.service | 14 ++ squid.conf.example | 135 +++++++++++++++ 14 files changed, 572 insertions(+) create mode 100644 Makefile create mode 100644 campus-blacklist-check.ini create mode 100644 campus-blacklist-check.service create mode 100644 campus-testing-blacklist-check.ini create mode 100644 campus-testing-blacklist-check.service create mode 100644 campus-testing-whitelist-check.ini create mode 100644 campus-testing-whitelist-check.service create mode 100644 campus-whitelist-check.ini create mode 100644 campus-whitelist-check.service create mode 100755 domain-to-search.rb create mode 100755 dst-domain-match-check.rb create mode 100644 dst-total-block-check.ini create mode 100644 dst-total-block.service create mode 100644 squid.conf.example diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9054fbc --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +all: + echo OK +install: + mkdir -p /etc/squid/bin + cp -vf dst-domain-match-check.rb /etc/squid/bin/ + stat /etc/squid/dst-total-block-check.ini || cp -vf dst-total-block-check.ini /etc/squid/dst-total-block-check.ini;true + stat /etc/squid/campus-testing-blacklist-check.ini || cp -vf campus-testing-blacklist-check.ini /etc/squid/;true + stat /etc/squid/campus-testing-whitelist-check.ini || cp -vf campus-testing-whitelist-check.ini /etc/squid/;true + stat /etc/squid/campus-whitelist-check.ini || cp -vf campus-whitelist-check.ini /etc/squid/;true + stat /etc/squid/campus-blacklist-check.ini || cp -vf campus-blacklist-check.ini /etc/squid/;true + chmod +x /etc/squid/bin/dst-domain-match-check.rb + +install-debian-dependencies: + apt update + apt install -y ruby ruby-mysql2 + gem install ipaddress + gem install inifile + +install-services: + cp -vf dst-total-block.service /etc/systemd/system/ + cp -vf campus-testing-whitelist-check.service /etc/systemd/system/ + cp -vf campus-testing-blacklist-check.service /etc/systemd/system/ + cp -vf campus-whitelist-check.service /etc/systemd/system/ + cp -vf campus-blacklist-check.service /etc/systemd/system/ + systemctl daemon-reload + +start-services: + systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + +stop-services: + systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + + + +enable-services: + systemctl enable dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + +restart-services: + systemctl restart dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + +cleanup-sockets: + rm -vf /tmp/campus-* + rm -vf /tmp/total-block diff --git a/campus-blacklist-check.ini b/campus-blacklist-check.ini new file mode 100644 index 0000000..40b196f --- /dev/null +++ b/campus-blacklist-check.ini @@ -0,0 +1,15 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf +table=campusBlackList + +[socket] +type=unix +address=unix:/tmp/campus-blacklist +port=20002 + +[main] +debug=1 +value=2 diff --git a/campus-blacklist-check.service b/campus-blacklist-check.service new file mode 100644 index 0000000..b17885b --- /dev/null +++ b/campus-blacklist-check.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/dst-domain-match-check.rb /etc/squid/campus-blacklist-check.ini + +[Install] +WantedBy=multi-user.target diff --git a/campus-testing-blacklist-check.ini b/campus-testing-blacklist-check.ini new file mode 100644 index 0000000..a91c450 --- /dev/null +++ b/campus-testing-blacklist-check.ini @@ -0,0 +1,15 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf +table=campusTestingBlackList + +[socket] +type=unix +address=unix:/tmp/campus-testing-blacklist +port=20002 + +[main] +debug=1 +value=2 diff --git a/campus-testing-blacklist-check.service b/campus-testing-blacklist-check.service new file mode 100644 index 0000000..fcdef70 --- /dev/null +++ b/campus-testing-blacklist-check.service @@ -0,0 +1,13 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/dst-domain-match-check.rb /etc/squid/campus-testing-blacklist-check.ini +[Install] +WantedBy=multi-user.target diff --git a/campus-testing-whitelist-check.ini b/campus-testing-whitelist-check.ini new file mode 100644 index 0000000..e87fb96 --- /dev/null +++ b/campus-testing-whitelist-check.ini @@ -0,0 +1,15 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf +table=campusTestingWhiteList + +[socket] +type=unix +address=unix:/tmp/campus-testing-whitelist +port=20002 + +[main] +debug=1 +value=2 diff --git a/campus-testing-whitelist-check.service b/campus-testing-whitelist-check.service new file mode 100644 index 0000000..840ee1c --- /dev/null +++ b/campus-testing-whitelist-check.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/dst-domain-match-check.rb /etc/squid/campus-testing-whitelist-check.ini + +[Install] +WantedBy=multi-user.target diff --git a/campus-whitelist-check.ini b/campus-whitelist-check.ini new file mode 100644 index 0000000..ece3911 --- /dev/null +++ b/campus-whitelist-check.ini @@ -0,0 +1,15 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf +table=campusWhiteList + +[socket] +type=unix +address=unix:/tmp/campus-whitelist +port=20002 + +[main] +debug=1 +value=2 diff --git a/campus-whitelist-check.service b/campus-whitelist-check.service new file mode 100644 index 0000000..a50e38a --- /dev/null +++ b/campus-whitelist-check.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/dst-domain-match-check.rb /etc/squid/campus-whitelist-check.ini + +[Install] +WantedBy=multi-user.target diff --git a/domain-to-search.rb b/domain-to-search.rb new file mode 100755 index 0000000..de1f636 --- /dev/null +++ b/domain-to-search.rb @@ -0,0 +1,22 @@ +#!/usr/bin/evn ruby + +def generate_domain_array(domain) + parts = domain.split(".") + l = parts.length + + domains = [] + domains << parts.join(".") + + i = 0 + while i < l do + domains << ".#{parts[i..-1].join(".")}" + i = i +1 + end + domains +end + +domain = ARGV[0] +domains = generate_domain_array(domain) + +puts "Domain Array query:" +puts "SELECT * FROM squidconf.totalBlock WHERE dstdom IN ( #{domains.map { |d| "'#{d}'" }.join(" , ")} )" diff --git a/dst-domain-match-check.rb b/dst-domain-match-check.rb new file mode 100755 index 0000000..f1c9726 --- /dev/null +++ b/dst-domain-match-check.rb @@ -0,0 +1,228 @@ +#!/usr/bin/env ruby + +require 'inifile' +require 'pp' + +require "resolv" +require "mysql2" + +require "socket" +require "thread" +require "syslog" +require "rubygems" +require "open-uri" +require "ipaddress" + +STDOUT.sync = true +Syslog.open("#{$PROGRAM_NAME}", Syslog::LOG_PID) + +def log(msg) + Syslog.log(Syslog::LOG_ERR, "%s", msg) + STDERR.puts("STDERR: [ #{msg} ]") if $debug +end + + +$my_dir = __dir__ + +config_filename = ARGV[0] + +if config_filename.nil? or !File.exists?(config_filename) + $debug = true + log("The config file: [\"#{config_filename}\"] doesn't exist") + exit 1 +end + +config_file = IniFile.load(config_filename) + +$value_location = config_file["main"]["value"].to_i + +$table_name = config_file["mysql"]["table"] + +$debug = false + +if config_file["main"]["debug"].to_s == "1" + $debug = true +end + +log("Started with DEBUG => #{$debug}") + +socket_type = config_file["socket"]["type"] +socket_address = config_file["socket"]["address"] +port = config_file["socket"]["port"] if config_file["socket"]["port"] + +$mysql_hostname = config_file["mysql"]["host"] +$mysql_username = config_file["mysql"]["username"] +$mysql_password = config_file["mysql"]["password"] +$mysql_db = config_file["mysql"]["database"] + +trap "SIGINT" do + STDERR.puts "STDERR: Exiting" + exit 130 +end + + +def cleanupUnixSocket(unix_socket_address) + if File.exist?(unix_socket_address) + File.delete(unix_socket_address) + end +end + +def generate_domain_array(domain) + parts = domain.split(".") + l = parts.length + + domains = [] + domains << parts.join(".") + + i = 0 + while i < l do + domains << ".#{parts[i..-1].join(".")}" + i = i +1 + end + domains +end + +def check_domain_in_db(domain) + domains = generate_domain_array(domain) + begin + db_client = Mysql2::Client.new(:host => $mysql_hostname, :username => $mysql_username, :password => $mysql_password, :database => $mysql_db) + rescue => e + log(e) + log(e.inspect) + end + sql_query = "SELECT * FROM #{$table_name} WHERE dstdom IN ( #{domains.map { |d| "'#{d}'" }.join(" , ")} )" + log("Running SQL query: [ \"#{sql_query}\" ]") + results = db_client.query(sql_query) + return false if results.nil? + if results.size > 0 + return true + end + return false +end + +def check_domain(domain) + log("checking domain #{domain}") if $debug + db_res = check_domain_in_db(domain) + log("DB RES #{db_res}") if $debug + + if db_res + return true + end + + return false +end + +def requestTest(request) + return if request == nil + request = request.split + matched_to = [] + ret = "0" + log("Request size: #{request.size} , value_location #{$value_location}") if $debug + if request.size >= $value_location + begin + # check if the requst is a domain name or ip address + is_ip_address = IPAddress.valid?(request[$value_location]) + if !is_ip_address + if check_domain(request[$value_location].downcase) + ret = "1" + end + end + rescue => e + log(e) + log(e.inspect) + ret = "1" + end + end + return { "request_id" => request[0], "ret" => "#{ret}" } +end + + +def validr?(request) + if request.ascii_only? && request.valid_encoding? + return true + else + STDERR.puts("errorness line [ #{request} ]") + return false + end +end + +answers = { "0" => "ERR", "1" => "OK" } + + +log("Socket binding is starting") +case socket_type +when /^tcp/i + begin + log("Trying to bind: #{socket_address}:#{port}") + server_socket = TCPServer.new(socket_address, port) + + loop do + Thread.start(server_socket.accept) do |s| + log("#{s} is accepted") + processingtQueue = Queue.new + + proccessor = Thread.new do + loop do + incomming_request = processingtQueue.pop + return if incomming_request.nil? + Thread.new do + result = requestTest(incomming_request) if validr?(incomming_request) + s.puts("#{result["request_id"]} #{answers[result["ret"]]}") + log("result for request: #{s} => [ #{incomming_request} ] , res => #{result}") if $debug + end + end + end + + while line = s.gets + processingtQueue << line.strip.chomp + log("original request: #{s} => [ #{line.chomp} ]") if $debug + end + proccessor.join + log("#{s} is gone") + s.close + end + end + rescue => e + log(e) + log(e.inspect) + exit 10 + end +when /^unix/i + begin + log("starting to bind UNIX Socket") + if IPAddress.valid?(socket_address) + log("Cannot use IP address #{socket_address} for unix socket") + exit 1 + end + address_url = URI.parse(socket_address) + unix_socket_address = address_url.path + + cleanupUnixSocket(unix_socket_address) + log("Trying to bind unix socket") + server_socket = UNIXServer.new(unix_socket_address) + log("UNIX Socket was bounded") + + loop do + Thread.start(server_socket.accept) do |s| + log("#{s} is accepted") + while line = s.gets + line = line.strip.chomp + log("original request: #{s} => [ #{line} ]") if $debug + result = requestTest(line) if validr?(line) + s.puts("#{result["request_id"]} #{answers[result["ret"]]}") + log("result for request: #{s} => [ #{line} ] , res => #{result}") if $debug + end + log("#{s} is gone") + s.close + end + end + rescue => e + File.delete(unix_socket_address) if File.exists?(unix_socket_address) + log(e) + log(e.inspect) + exit 11 + end +else + log("Sokcet type: #{socket_type} is not supported") + exit 1 +end diff --git a/dst-total-block-check.ini b/dst-total-block-check.ini new file mode 100644 index 0000000..8dedc8e --- /dev/null +++ b/dst-total-block-check.ini @@ -0,0 +1,15 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf +table=totalBlock + +[socket] +type=unix +address=unix:/tmp/total-block +port=20002 + +[main] +debug=1 +value=1 diff --git a/dst-total-block.service b/dst-total-block.service new file mode 100644 index 0000000..c930864 --- /dev/null +++ b/dst-total-block.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/dst-domain-match-check.rb /etc/squid/dst-total-block-check.ini + +[Install] +WantedBy=multi-user.target diff --git a/squid.conf.example b/squid.conf.example new file mode 100644 index 0000000..730dba8 --- /dev/null +++ b/squid.conf.example @@ -0,0 +1,135 @@ +# +# Recommended minimum configuration: +# + +auth_param basic program /usr/lib/squid/basic_db_auth --debug -dsn "DBI:mysql:host=192.168.220.135;port=3306;database=squidconf" --user SquidConfReader --password SquidConfReader --plaintext --persist --table users --usercol username --passwdcol password --cond y=1 + +auth_param basic children 5 +auth_param basic realm Web-Proxy +auth_param basic credentialsttl 1 minute +auth_param basic casesensitive off + +acl db-auth proxy_auth REQUIRED + +## Total blacklist +external_acl_type total_block_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/total-block + +acl total_block_checker_helper external total_block_checker + +deny_info https://www.ngtech.co.il/blockPage/?url?%u total_block_checker_helper + + +## Campus Wide testing blacklist + +external_acl_type campus_testing_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-blacklist + +acl campus_testing_blacklist_checker_helper external campus_testing_blacklist_checker +deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_testing_blacklist_checker_helper + +## Campus Wide testing whitelist +external_acl_type campus_testing_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-whitelist + +acl campus_testing_whitelist_checker_helper external campus_testing_whitelist_checker + +## User blacklist +#external_acl_type user_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-blacklist + +#acl user_blacklist_checker_checker_helper external user_blacklist_checker +#deny_info https://www.ngtech.co.il/blockPage/?url?%u user_blacklist_checker_helper + +## User whitelist +#external_acl_type user_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-whitelist + +#acl user_whitelist_checker_checker_helper external user_whitelist_checker + +## Campus Wide blacklist +external_acl_type campus_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-blacklist + +acl campus_blacklist_checker_helper external campus_blacklist_checker +deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_blacklist_checker_helper + +## Campus Wide whitelist +external_acl_type campus_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-whitelist + +acl campus_whitelist_checker_helper external campus_whitelist_checker + + +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines + +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http + +# +# Recommended minimum Access Permission configuration: +# +# Deny requests to certain unsafe ports +http_access deny !localnet +http_access deny !db-auth +http_access deny !Safe_ports + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + +http_access deny to_localhost +http_access deny to_linklocal + +# +# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS +# +http_access deny total_block_checker_helper + +http_access deny campus_testing_blacklist_checker_helper +http_access allow campus_testing_whitelist_checker_helper + +#http_access deny user_blacklist +#http_access allow user_whitelist + +http_access deny campus_blacklist_checker_helper +http_access allow campus_whitelist_checker_helper + +http_access allow localnet db-auth +# For example, to allow access from your local networks, you may uncomment the +# following rule (and/or add rules that match your definition of "local"): +#http_access allow localnet + +# And finally deny all other access to this proxy +http_access deny all + +# Squid normally listens to port 3128 +http_port 3128 + +# Uncomment and adjust the following to add a disk cache directory. +#cache_dir ufs /var/spool/squid 100 16 256 + +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid + +# +# Add any of your own refresh_pattern entries above these. +# +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320