Added users lists helpers and config example

Makefile

@@ -3,12 +3,16 @@ all:
 install:
 	mkdir -p /etc/squid/bin
 	cp -vf dst-domain-match-check.rb /etc/squid/bin/
+	cp -vf user-list-dst-domain-match-check.rb /etc/squid/bin/
 	stat /etc/squid/dst-total-block-check.ini || cp -vf dst-total-block-check.ini /etc/squid/dst-total-block-check.ini;true
 	stat /etc/squid/campus-testing-blacklist-check.ini || cp -vf campus-testing-blacklist-check.ini /etc/squid/;true
 	stat /etc/squid/campus-testing-whitelist-check.ini || cp -vf campus-testing-whitelist-check.ini /etc/squid/;true
 	stat /etc/squid/campus-whitelist-check.ini || cp -vf campus-whitelist-check.ini /etc/squid/;true
 	stat /etc/squid/campus-blacklist-check.ini || cp -vf campus-blacklist-check.ini /etc/squid/;true
+	stat /etc/squid/users-blacklist.ini || cp -vf users-blacklist.ini /etc/squid/;true
+	stat /etc/squid/users-whitelist.ini || cp -vf users-whitelist.ini /etc/squid/;true
 	chmod +x /etc/squid/bin/dst-domain-match-check.rb
+	chmod +x /etc/squid/bin/user-list-dst-domain-match-check.rb
 
 install-debian-dependencies:
 	apt update
@@ -22,22 +26,23 @@ install-services:
 	cp -vf campus-testing-blacklist-check.service /etc/systemd/system/
 	cp -vf campus-whitelist-check.service /etc/systemd/system/
 	cp -vf campus-blacklist-check.service /etc/systemd/system/
+	cp -vf users-blacklist.service /etc/systemd/system/
+	cp -vf users-whitelist.service /etc/systemd/system/
 	systemctl daemon-reload
 
 start-services:
-	systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check
+	systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist
 
 stop-services:
-	systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check
+	systemctl start dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist
-
-
 
 enable-services:
-	systemctl enable dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check
+	systemctl enable dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist
 
 restart-services:
-	systemctl restart dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check
+	systemctl restart dst-total-block campus-testing-whitelist-check campus-testing-blacklist-check campus-whitelist-check campus-blacklist-check users-whitelist users-blacklist
 
 cleanup-sockets:
 	rm -vf /tmp/campus-*
+	rm -vf /tmp/users-*
 	rm -vf /tmp/total-block

squid.conf.example

@@ -1,3 +1,4 @@
+shutdown_lifetime 5 seconds
 #
 # Recommended minimum configuration:
 #
@@ -9,47 +10,49 @@ auth_param basic realm Web-Proxy
 auth_param basic credentialsttl 1 minute
 auth_param basic casesensitive off
 
+acl blockpage_dom dstdomain www.ngtech.co.il
+
 acl db-auth proxy_auth REQUIRED
 
 ## Total blacklist
-external_acl_type total_block_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/total-block
+external_acl_type total_block_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/total-block
 
 acl total_block_checker_helper external total_block_checker
 
-deny_info https://www.ngtech.co.il/blockPage/?url?%u total_block_checker_helper
+deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u total_block_checker_helper
 
 
 ## Campus Wide testing blacklist
 
-external_acl_type campus_testing_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-blacklist
+external_acl_type campus_testing_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-blacklist
 
 acl campus_testing_blacklist_checker_helper external campus_testing_blacklist_checker
-deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_testing_blacklist_checker_helper
+deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u campus_testing_blacklist_checker_helper
 
 ## Campus Wide testing whitelist
-external_acl_type campus_testing_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-whitelist
+external_acl_type campus_testing_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-testing-whitelist
 
 acl campus_testing_whitelist_checker_helper external campus_testing_whitelist_checker
 
 ## User blacklist
-#external_acl_type user_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-blacklist
+external_acl_type user_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/users-blacklist
 
-#acl user_blacklist_checker_checker_helper external user_blacklist_checker
+acl user_blacklist_checker_helper external user_blacklist_checker
-#deny_info https://www.ngtech.co.il/blockPage/?url?%u user_blacklist_checker_helper
+deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u user_blacklist_checker_helper
 
 ## User whitelist
-#external_acl_type user_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/user-whitelist
+external_acl_type user_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/users-whitelist
 
-#acl user_whitelist_checker_checker_helper external user_whitelist_checker
+acl user_whitelist_checker_helper external user_whitelist_checker
 
 ## Campus Wide blacklist
-external_acl_type campus_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-blacklist
+external_acl_type campus_blacklist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-blacklist
 
 acl campus_blacklist_checker_helper external campus_blacklist_checker
-deny_info https://www.ngtech.co.il/blockPage/?url?%u campus_blacklist_checker_helper
+deny_info 302:https://www.ngtech.co.il/blockPage/?url=%u campus_blacklist_checker_helper
 
 ## Campus Wide whitelist
-external_acl_type campus_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %LOGIN /usr/bin/socat - UNIX-CONNECT:/tmp/campus-whitelist
+external_acl_type campus_whitelist_checker concurrency=10 children-max=60 children-startup=20 children-idle=20 ttl=300 %SRC %DST %METHOD %un /usr/bin/socat - UNIX-CONNECT:/tmp/campus-whitelist
 
 acl campus_whitelist_checker_helper external campus_whitelist_checker
 
@@ -82,8 +85,12 @@ acl Safe_ports port 777		# multiling http
 # Recommended minimum Access Permission configuration:
 #
 # Deny requests to certain unsafe ports
-http_access deny !localnet
+#http_access deny !localnet
+http_access allow blockpage_dom
+
 http_access deny !db-auth
+
+
 http_access deny !Safe_ports
 
 # Deny CONNECT to other than secure SSL ports
@@ -104,16 +111,17 @@ http_access deny total_block_checker_helper
 http_access deny campus_testing_blacklist_checker_helper
 http_access allow campus_testing_whitelist_checker_helper
 
-#http_access deny user_blacklist
+http_access deny user_blacklist_checker_helper
-#http_access allow user_whitelist
+http_access allow user_whitelist_checker_helper
 
 http_access deny campus_blacklist_checker_helper
 http_access allow campus_whitelist_checker_helper
 
-http_access allow localnet db-auth
+#http_access allow db-auth
 # For example, to allow access from your local networks, you may uncomment the
 # following rule (and/or add rules that match your definition of "local"):
 #http_access allow localnet
+http_access allow all
 
 # And finally deny all other access to this proxy
 http_access deny all

user-list-dst-domain-match-check.rb

@@ -0,0 +1,240 @@
+#!/usr/bin/env ruby
+
+require 'inifile'
+require 'pp'
+
+require "resolv"
+require "mysql2"
+
+require "socket"
+require "thread"
+require "syslog"
+require "rubygems"
+require "open-uri"
+require "ipaddress"
+
+STDOUT.sync = true
+Syslog.open("#{$PROGRAM_NAME}", Syslog::LOG_PID)
+
+def log(msg)
+  Syslog.log(Syslog::LOG_ERR, "%s", msg)
+  STDERR.puts("STDERR: [ #{msg} ]") if $debug
+end
+
+
+$my_dir = __dir__
+
+config_filename = ARGV[0]
+
+if config_filename.nil? or !File.exists?(config_filename)
+	$debug = true
+	log("The config file: [\"#{config_filename}\"] doesn't exist")
+	exit 1
+end
+
+config_file = IniFile.load(config_filename)
+
+$domain_location = config_file["main"]["domain_pos"].to_i
+$login_location = config_file["main"]["login_pos"].to_i
+
+$list_table = config_file["tables"]["list"]
+
+$users_table = config_file["tables"]["users"]
+
+$debug = false
+
+if config_file["main"]["debug"].to_s == "1"
+	$debug = true
+end
+
+log("Started with DEBUG => #{$debug}")
+
+socket_type = config_file["socket"]["type"]
+socket_address = config_file["socket"]["address"]
+port = config_file["socket"]["port"] if config_file["socket"]["port"]
+
+$mysql_hostname = config_file["mysql"]["host"]
+$mysql_username = config_file["mysql"]["username"]
+$mysql_password = config_file["mysql"]["password"]
+$mysql_db = config_file["mysql"]["database"]
+
+trap "SIGINT" do
+  STDERR.puts "STDERR: Exiting"
+  exit 130
+end
+
+
+def cleanupUnixSocket(unix_socket_address)
+  if File.exist?(unix_socket_address)
+    File.delete(unix_socket_address)
+  end
+end
+
+def generate_domain_array(domain)
+  parts = domain.split(".")
+  l = parts.length
+
+  domains = []
+  domains << parts.join(".")
+
+  i = 0
+  while i < l do
+        domains << ".#{parts[i..-1].join(".")}"
+    i = i +1
+  end
+  domains
+end
+
+def check_domain_in_db(domain, login)
+  domains =  generate_domain_array(domain)
+  begin
+    db_client = Mysql2::Client.new(:host => $mysql_hostname, :username => $mysql_username, :password => $mysql_password, :database => $mysql_db)
+  rescue => e
+    log(e)
+    log(e.inspect)
+  end
+  sql_query = "SELECT u.username , ubl.dstdom, ubl.updated_at 
+FROM usersBlackLists ubl  
+LEFT JOIN users u  ON u.user_id=ubl.user_id
+WHERE u.username = '#{login}' and ubl.dstdom IN ( #{domains.map { |d| "'#{d}'" }.join(" , ")} ) AND ubl.y=1;"
+  log("Running SQL query: [ \"#{sql_query}\" ]")
+  results = db_client.query(sql_query)
+  if results.nil?
+	db_client.close
+	return false
+  end
+  if results.size > 0
+    db_client.close
+    return true
+  end
+  db_client.close
+  return false
+end
+
+def check_domain(domain, login)
+  log("checking domain #{domain}") if $debug
+  db_res = check_domain_in_db(domain, login)
+  log("DB RES #{db_res}") if $debug
+
+  if db_res
+    return true
+  end
+
+  return false
+end
+
+def requestTest(request)
+  return if request == nil
+  request = request.split
+  matched_to = []
+  ret = "0"
+  log("Request size: #{request.size} , domain_location #{$domain_location}, login_location #{$login_location}") if $debug
+  if request.size >= $login_location
+    begin
+      # check if the requst is a domain name or ip address and if it's valid
+      is_ip_address = IPAddress.valid?(request[$domain_location])
+	# Check if the login name is valid (no spaces etc...)
+      if !is_ip_address
+        if check_domain(request[$domain_location].downcase, request[$login_location])
+          ret = "1"
+        end
+      end
+    rescue => e
+      log(e)
+      log(e.inspect)
+      ret = "1"
+    end
+  end
+  return { "request_id" => request[0], "ret" => "#{ret}" }
+end
+
+
+def validr?(request)
+  if request.ascii_only? && request.valid_encoding?
+    return true
+  else
+    STDERR.puts("errorness line [ #{request} ]")
+    return false
+  end
+end
+
+answers = { "0" => "ERR", "1" => "OK" }
+
+
+log("Socket binding is starting")
+case socket_type
+when /^tcp/i
+  begin
+    log("Trying to bind: #{socket_address}:#{port}")
+    server_socket = TCPServer.new(socket_address, port)
+
+    loop do
+      Thread.start(server_socket.accept) do |s|
+        log("#{s} is accepted")
+        processingtQueue = Queue.new
+
+        proccessor = Thread.new do
+          loop do
+            incomming_request = processingtQueue.pop
+            return if incomming_request.nil?
+            Thread.new do
+              result = requestTest(incomming_request) if validr?(incomming_request)
+              s.puts("#{result["request_id"]} #{answers[result["ret"]]}")
+              log("result for request: #{s} => [ #{incomming_request} ] , res => #{result}") if $debug
+            end
+          end
+        end
+
+        while line = s.gets
+          processingtQueue << line.strip.chomp
+          log("original request: #{s} => [ #{line.chomp} ]") if $debug
+        end
+        proccessor.join
+        log("#{s} is gone")
+        s.close
+      end
+    end
+  rescue => e
+    log(e)
+    log(e.inspect)
+    exit 10
+  end
+when /^unix/i
+  begin
+    log("starting to bind UNIX Socket")
+    if IPAddress.valid?(socket_address)
+      log("Cannot use IP address #{socket_address} for unix socket")
+      exit 1
+    end
+	address_url = URI.parse(socket_address)
+	unix_socket_address = address_url.path
+
+    cleanupUnixSocket(unix_socket_address)
+    log("Trying to bind unix socket")
+    server_socket = UNIXServer.new(unix_socket_address)
+    log("UNIX Socket was bounded")
+
+    loop do
+      Thread.start(server_socket.accept) do |s|
+        log("#{s} is accepted")
+        while line = s.gets
+          line = line.strip.chomp
+          log("original request: #{s} => [ #{line} ]") if $debug
+          result = requestTest(line) if validr?(line)
+          s.puts("#{result["request_id"]} #{answers[result["ret"]]}")
+          log("result for request: #{s} => [ #{line} ] , res => #{result}") if $debug
+        end
+        log("#{s} is gone")
+        s.close
+      end
+    end
+  rescue => e
+    File.delete(unix_socket_address) if File.exists?(unix_socket_address)
+    log(e)
+    log(e.inspect)
+    exit 11
+  end
+else
+  log("Sokcet type: #{socket_type}  is not supported")
+  exit 1
+end

users-blacklist.ini

@@ -0,0 +1,21 @@
+[mysql]
+host=192.168.220.135
+username=SquidConfReader
+password=SquidConfReader
+database=squidconf
+
+[socket]
+type=unix
+address=unix:/tmp/users-blacklist
+port=20002
+
+#REQUEST_ID %SRC %DST %METHOD %LOGIN
+#    0        1    2    3        4
+[main]
+debug=1
+domain_pos=2
+login_pos=4
+
+[tables]
+users=users
+list=usersBlackLists

users-blacklist.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=DstDomain checker
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=3
+User=proxy
+ExecStart=/usr/bin/env ruby /etc/squid/bin/user-list-dst-domain-match-check.rb /etc/squid/users-blacklist.ini
+
+[Install]
+WantedBy=multi-user.target

users-whitelist.ini

@@ -0,0 +1,21 @@
+[mysql]
+host=192.168.220.135
+username=SquidConfReader
+password=SquidConfReader
+database=squidconf
+
+[socket]
+type=unix
+address=unix:/tmp/users-whitelist
+port=20002
+
+#REQUEST_ID %SRC %DST %METHOD %LOGIN
+#    0        1    2    3        4
+[main]
+debug=1
+domain_pos=2
+login_pos=4
+
+[tables]
+users=users
+list=usersWhiteLists

users-whitelist.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=DstDomain checker
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=simple
+Restart=always
+RestartSec=3
+User=proxy
+ExecStart=/usr/bin/env ruby /etc/squid/bin/user-list-dst-domain-match-check.rb /etc/squid/users-whitelist.ini
+
+[Install]
+WantedBy=multi-user.target