From 754fc776acb0e840d7655ae29af93aa0cc400496 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 13 Feb 2024 20:37:52 +0200 Subject: [PATCH] Added users lists helpers and config example --- Makefile | 21 ++- squid.conf.example | 42 +++-- user-list-dst-domain-match-check.rb | 240 ++++++++++++++++++++++++++++ users-blacklist.ini | 21 +++ users-blacklist.service | 14 ++ users-whitelist.ini | 21 +++ users-whitelist.service | 14 ++ 7 files changed, 348 insertions(+), 25 deletions(-) create mode 100755 user-list-dst-domain-match-check.rb create mode 100644 users-blacklist.ini create mode 100644 users-blacklist.service create mode 100644 users-whitelist.ini create mode 100644 users-whitelist.service diff --git a/Makefile b/Makefile index 9054fbc..10e062c 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,18 @@ all: echo OK install: mkdir -p /etc/squid/bin - cp -vf dst-domain-match-check.rb /etc/squid/bin/ + cp -vf dst-domain-match-check.rb /etc/squid/bin/ + cp -vf user-list-dst-domain-match-check.rb /etc/squid/bin/ stat /etc/squid/dst-total-block-check.ini || cp -vf dst-total-block-check.ini /etc/squid/dst-total-block-check.ini;true stat /etc/squid/campus-testing-blacklist-check.ini || cp -vf campus-testing-blacklist-check.ini /etc/squid/;true stat /etc/squid/campus-testing-whitelist-check.ini || cp -vf campus-testing-whitelist-check.ini /etc/squid/;true stat /etc/squid/campus-whitelist-check.ini || cp -vf campus-whitelist-check.ini /etc/squid/;true stat /etc/squid/campus-blacklist-check.ini || cp -vf campus-blacklist-check.ini /etc/squid/;true + stat /etc/squid/users-blacklist.ini || cp -vf users-blacklist.ini /etc/squid/;true + stat /etc/squid/users-whitelist.ini || cp -vf users-whitelist.ini /etc/squid/;true chmod +x /etc/squid/bin/dst-domain-match-check.rb - + chmod +x /etc/squid/bin/user-list-dst-domain-match-check.rb + install-debian-dependencies: apt update apt install -y ruby ruby-mysql2 @@ -22,22 +26,23 @@ install-services: cp -vf campus-testing-blacklist-check.service /etc/systemd/system/ cp -vf campus-whitelist-check.service /etc/systemd/system/ cp -vf campus-blacklist-check.service /etc/systemd/system/ + cp -vf users-blacklist.service /etc/systemd/system/ + cp -vf users-whitelist.service /etc/systemd/system/ systemctl daemon-reload start-services: - systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist stop-services: - systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check - - + systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist enable-services: - systemctl enable dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + systemctl enable dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist restart-services: - systemctl restart dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check + systemctl restart dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist cleanup-sockets: rm -vf /tmp/campus-* + rm -vf /tmp/users-* rm -vf /tmp/total-block diff --git a/squid.conf.example b/squid.conf.example index 730dba8..d58af41 100644 --- a/squid.conf.example +++ b/squid.conf.example @@ -1,3 +1,4 @@ +shutdown_lifetime 5 seconds # # Recommended minimum configuration: # @@ -9,47 +10,49 @@ auth_param basic realm Web-Proxy auth_param basic credentialsttl 1 minute auth_param basic casesensitive off +acl blockpage_dom dstdomain www.ngtech.co.il + acl db-auth proxy_auth REQUIRED ## Total blacklist -external_acl_type total_block_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/total-block +external_acl_type total_block_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/total-block acl total_block_checker_helper external total_block_checker -deny_info https://www.ngtech.co.il/blockPage/?url?%u total_block_checker_helper +deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u total_block_checker_helper ## Campus Wide testing blacklist -external_acl_type campus_testing_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-blacklist +external_acl_type campus_testing_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-blacklist acl campus_testing_blacklist_checker_helper external campus_testing_blacklist_checker -deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_testing_blacklist_checker_helper +deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u campus_testing_blacklist_checker_helper ## Campus Wide testing whitelist -external_acl_type campus_testing_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-whitelist +external_acl_type campus_testing_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-whitelist acl campus_testing_whitelist_checker_helper external campus_testing_whitelist_checker ## User blacklist -#external_acl_type user_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-blacklist +external_acl_type user_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/users-blacklist -#acl user_blacklist_checker_checker_helper external user_blacklist_checker -#deny_info https://www.ngtech.co.il/blockPage/?url?%u user_blacklist_checker_helper +acl user_blacklist_checker_helper external user_blacklist_checker +deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u user_blacklist_checker_helper ## User whitelist -#external_acl_type user_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-whitelist +external_acl_type user_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/users-whitelist -#acl user_whitelist_checker_checker_helper external user_whitelist_checker +acl user_whitelist_checker_helper external user_whitelist_checker ## Campus Wide blacklist -external_acl_type campus_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-blacklist +external_acl_type campus_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-blacklist acl campus_blacklist_checker_helper external campus_blacklist_checker -deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_blacklist_checker_helper +deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u campus_blacklist_checker_helper ## Campus Wide whitelist -external_acl_type campus_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-whitelist +external_acl_type campus_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-whitelist acl campus_whitelist_checker_helper external campus_whitelist_checker @@ -82,8 +85,12 @@ acl Safe_ports port 777 # multiling http # Recommended minimum Access Permission configuration: # # Deny requests to certain unsafe ports -http_access deny !localnet +#http_access deny !localnet +http_access allow blockpage_dom + http_access deny !db-auth + + http_access deny !Safe_ports # Deny CONNECT to other than secure SSL ports @@ -104,16 +111,17 @@ http_access deny total_block_checker_helper http_access deny campus_testing_blacklist_checker_helper http_access allow campus_testing_whitelist_checker_helper -#http_access deny user_blacklist -#http_access allow user_whitelist +http_access deny user_blacklist_checker_helper +http_access allow user_whitelist_checker_helper http_access deny campus_blacklist_checker_helper http_access allow campus_whitelist_checker_helper -http_access allow localnet db-auth +#http_access allow db-auth # For example, to allow access from your local networks, you may uncomment the # following rule (and/or add rules that match your definition of "local"): #http_access allow localnet +http_access allow all # And finally deny all other access to this proxy http_access deny all diff --git a/user-list-dst-domain-match-check.rb b/user-list-dst-domain-match-check.rb new file mode 100755 index 0000000..efdd1c7 --- /dev/null +++ b/user-list-dst-domain-match-check.rb @@ -0,0 +1,240 @@ +#!/usr/bin/env ruby + +require 'inifile' +require 'pp' + +require "resolv" +require "mysql2" + +require "socket" +require "thread" +require "syslog" +require "rubygems" +require "open-uri" +require "ipaddress" + +STDOUT.sync = true +Syslog.open("#{$PROGRAM_NAME}", Syslog::LOG_PID) + +def log(msg) + Syslog.log(Syslog::LOG_ERR, "%s", msg) + STDERR.puts("STDERR: [ #{msg} ]") if $debug +end + + +$my_dir = __dir__ + +config_filename = ARGV[0] + +if config_filename.nil? or !File.exists?(config_filename) + $debug = true + log("The config file: [\"#{config_filename}\"] doesn't exist") + exit 1 +end + +config_file = IniFile.load(config_filename) + +$domain_location = config_file["main"]["domain_pos"].to_i +$login_location = config_file["main"]["login_pos"].to_i + +$list_table = config_file["tables"]["list"] + +$users_table = config_file["tables"]["users"] + +$debug = false + +if config_file["main"]["debug"].to_s == "1" + $debug = true +end + +log("Started with DEBUG => #{$debug}") + +socket_type = config_file["socket"]["type"] +socket_address = config_file["socket"]["address"] +port = config_file["socket"]["port"] if config_file["socket"]["port"] + +$mysql_hostname = config_file["mysql"]["host"] +$mysql_username = config_file["mysql"]["username"] +$mysql_password = config_file["mysql"]["password"] +$mysql_db = config_file["mysql"]["database"] + +trap "SIGINT" do + STDERR.puts "STDERR: Exiting" + exit 130 +end + + +def cleanupUnixSocket(unix_socket_address) + if File.exist?(unix_socket_address) + File.delete(unix_socket_address) + end +end + +def generate_domain_array(domain) + parts = domain.split(".") + l = parts.length + + domains = [] + domains << parts.join(".") + + i = 0 + while i < l do + domains << ".#{parts[i..-1].join(".")}" + i = i +1 + end + domains +end + +def check_domain_in_db(domain, login) + domains = generate_domain_array(domain) + begin + db_client = Mysql2::Client.new(:host => $mysql_hostname, :username => $mysql_username, :password => $mysql_password, :database => $mysql_db) + rescue => e + log(e) + log(e.inspect) + end + sql_query = "SELECT u.username , ubl.dstdom, ubl.updated_at +FROM usersBlackLists ubl +LEFT JOIN users u ON u.user_id=ubl.user_id +WHERE u.username = '#{login}' and ubl.dstdom IN ( #{domains.map { |d| "'#{d}'" }.join(" , ")} ) AND ubl.y=1;" + log("Running SQL query: [ \"#{sql_query}\" ]") + results = db_client.query(sql_query) + if results.nil? + db_client.close + return false + end + if results.size > 0 + db_client.close + return true + end + db_client.close + return false +end + +def check_domain(domain, login) + log("checking domain #{domain}") if $debug + db_res = check_domain_in_db(domain, login) + log("DB RES #{db_res}") if $debug + + if db_res + return true + end + + return false +end + +def requestTest(request) + return if request == nil + request = request.split + matched_to = [] + ret = "0" + log("Request size: #{request.size} , domain_location #{$domain_location}, login_location #{$login_location}") if $debug + if request.size >= $login_location + begin + # check if the requst is a domain name or ip address and if it's valid + is_ip_address = IPAddress.valid?(request[$domain_location]) + # Check if the login name is valid (no spaces etc...) + if !is_ip_address + if check_domain(request[$domain_location].downcase, request[$login_location]) + ret = "1" + end + end + rescue => e + log(e) + log(e.inspect) + ret = "1" + end + end + return { "request_id" => request[0], "ret" => "#{ret}" } +end + + +def validr?(request) + if request.ascii_only? && request.valid_encoding? + return true + else + STDERR.puts("errorness line [ #{request} ]") + return false + end +end + +answers = { "0" => "ERR", "1" => "OK" } + + +log("Socket binding is starting") +case socket_type +when /^tcp/i + begin + log("Trying to bind: #{socket_address}:#{port}") + server_socket = TCPServer.new(socket_address, port) + + loop do + Thread.start(server_socket.accept) do |s| + log("#{s} is accepted") + processingtQueue = Queue.new + + proccessor = Thread.new do + loop do + incomming_request = processingtQueue.pop + return if incomming_request.nil? + Thread.new do + result = requestTest(incomming_request) if validr?(incomming_request) + s.puts("#{result["request_id"]} #{answers[result["ret"]]}") + log("result for request: #{s} => [ #{incomming_request} ] , res => #{result}") if $debug + end + end + end + + while line = s.gets + processingtQueue << line.strip.chomp + log("original request: #{s} => [ #{line.chomp} ]") if $debug + end + proccessor.join + log("#{s} is gone") + s.close + end + end + rescue => e + log(e) + log(e.inspect) + exit 10 + end +when /^unix/i + begin + log("starting to bind UNIX Socket") + if IPAddress.valid?(socket_address) + log("Cannot use IP address #{socket_address} for unix socket") + exit 1 + end + address_url = URI.parse(socket_address) + unix_socket_address = address_url.path + + cleanupUnixSocket(unix_socket_address) + log("Trying to bind unix socket") + server_socket = UNIXServer.new(unix_socket_address) + log("UNIX Socket was bounded") + + loop do + Thread.start(server_socket.accept) do |s| + log("#{s} is accepted") + while line = s.gets + line = line.strip.chomp + log("original request: #{s} => [ #{line} ]") if $debug + result = requestTest(line) if validr?(line) + s.puts("#{result["request_id"]} #{answers[result["ret"]]}") + log("result for request: #{s} => [ #{line} ] , res => #{result}") if $debug + end + log("#{s} is gone") + s.close + end + end + rescue => e + File.delete(unix_socket_address) if File.exists?(unix_socket_address) + log(e) + log(e.inspect) + exit 11 + end +else + log("Sokcet type: #{socket_type} is not supported") + exit 1 +end diff --git a/users-blacklist.ini b/users-blacklist.ini new file mode 100644 index 0000000..5914542 --- /dev/null +++ b/users-blacklist.ini @@ -0,0 +1,21 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf + +[socket] +type=unix +address=unix:/tmp/users-blacklist +port=20002 + +#REQUEST_ID %SRC %DST %METHOD %LOGIN +# 0 1 2 3 4 +[main] +debug=1 +domain_pos=2 +login_pos=4 + +[tables] +users=users +list=usersBlackLists diff --git a/users-blacklist.service b/users-blacklist.service new file mode 100644 index 0000000..9aea127 --- /dev/null +++ b/users-blacklist.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/user-list-dst-domain-match-check.rb /etc/squid/users-blacklist.ini + +[Install] +WantedBy=multi-user.target diff --git a/users-whitelist.ini b/users-whitelist.ini new file mode 100644 index 0000000..c8dba1b --- /dev/null +++ b/users-whitelist.ini @@ -0,0 +1,21 @@ +[mysql] +host=192.168.220.135 +username=SquidConfReader +password=SquidConfReader +database=squidconf + +[socket] +type=unix +address=unix:/tmp/users-whitelist +port=20002 + +#REQUEST_ID %SRC %DST %METHOD %LOGIN +# 0 1 2 3 4 +[main] +debug=1 +domain_pos=2 +login_pos=4 + +[tables] +users=users +list=usersWhiteLists diff --git a/users-whitelist.service b/users-whitelist.service new file mode 100644 index 0000000..682cd97 --- /dev/null +++ b/users-whitelist.service @@ -0,0 +1,14 @@ +[Unit] +Description=DstDomain checker +After=network.target +StartLimitIntervalSec=0 + +[Service] +Type=simple +Restart=always +RestartSec=3 +User=proxy +ExecStart=/usr/bin/env ruby /etc/squid/bin/user-list-dst-domain-match-check.rb /etc/squid/users-whitelist.ini + +[Install] +WantedBy=multi-user.target