go-shield/server/ufdbguard/ufdbguard.go

package ufdbguard

import (
	"bufio"
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"regexp"
	"server/categorization"
	"server/internal"
	"strconv"
	"strings"
	"time"
)

var (
	isDebugUfdbguard                   bool
	SelfUfdbGuardMaxUrisize            int
	SelfFinalRedirdectCode             int
	SelfFinalRedirectUrl               string
	SelfRedirectKey                    string
	SelfProxyProto                     string
	SelfClientMac                      string
	SelfMimik                          bool
	SelfWebfilterRuleName              string
	SelfIsSSNI                         bool
	SelfRemoteIP                       string
	SelfRemotePort                     int
	SelfUseRemoteUfdbguardService      bool
	SelfSquidGuardRedirectHTTPCode     int
	SelfSquidGuardWebUseExternalUri    int
	SelfSquidGuardWebExternalUri       string
	SelfSquidGuardWebExternalUriSSL    string
	SelfHttps                          bool
	SelfUfdbGuardWebFilteringCacheTime int
	SelfCached                         bool
	SelfInactiveService                bool
	SelfUfdbgclientSockTimeOut         time.Duration
	SelfCategory                       int
	SelfWebfilteringToken              string
	SelfRuleID                         int
	SelfCategoryName                   string
	SelfToken                          string
)

func InitUfdbguard(debug bool) {
	//Self := &UFDBDefaultVars{}
	//log.SetFlags(log.LstdFlags | log.Lshortfile)
	//if _slog, err := syslog.New(syslog.LOG_DEBUG, "go-shield-connector-ufdb"); err == nil {
	//	log.SetOutput(_slog)
	//}
	SelfRemoteIP = internal.GetSocketInfoString("PythonUfdbServer")
	SelfRemotePort = internal.GetSocketInfoInt("PythonUfdbPort")
	SelfUseRemoteUfdbguardService = internal.GetSocketInfoBool("UseRemoteUfdbguardService")
	SelfSquidGuardRedirectHTTPCode = internal.GetSocketInfoInt("SquidGuardRedirectHTTPCode")
	SelfUfdbGuardMaxUrisize = internal.GetSocketInfoInt("UfdbGuardMaxUrisize")
	SelfSquidGuardWebUseExternalUri = internal.GetSocketInfoInt("SquidGuardWebUseExternalUri")
	SelfSquidGuardWebExternalUri = internal.GetSocketInfoString("SquidGuardWebExternalUri")
	SelfSquidGuardWebExternalUriSSL = internal.GetSocketInfoString("SquidGuardWebExternalUriSSL")
	SelfUfdbGuardWebFilteringCacheTime = internal.GetSocketInfoInt("UfdbGuardWebFilteringCacheTime")
	SelfUfdbgclientSockTimeOut = time.Duration(internal.GetSocketInfoInt("UfdbgclientSockTimeOut")) * time.Second
	if SelfUfdbgclientSockTimeOut == 0 {
		SelfUfdbgclientSockTimeOut = time.Duration(120) * time.Second
	}
	if SelfUfdbGuardWebFilteringCacheTime == 0 {
		SelfUfdbGuardWebFilteringCacheTime = 300
	}
	if SelfSquidGuardRedirectHTTPCode < 300 {
		SelfSquidGuardRedirectHTTPCode = 302
	}
	if SelfUfdbGuardMaxUrisize == 0 {
		SelfUfdbGuardMaxUrisize = 640
	}

	isDebugUfdbguard = debug

	isBumped()
	if !SelfUseRemoteUfdbguardService {
		checkLocalConfig()
		if SelfRemoteIP == "all" {
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: Warning, unable to found the remote TCP Addr config, assume 127.0.0.1")
			}
			SelfRemoteIP = "127.0.0.1"
		}
		if len(SelfRemoteIP) == 0 {
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: Warning, unable to found the remote TCP Addr config, assume 127.0.0.1")
			}
			SelfRemoteIP = "127.0.0.1"
		}
		if SelfRemotePort == 0 {
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: Warning, unable to found the remote port config, assume 3977 port")
			}
			SelfRemotePort = 3977
		}
	}
	if SelfRemotePort > 0 {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: Redirect Code..............: %d", SelfSquidGuardRedirectHTTPCode)
			log.Printf("[UFDB_CLASS]: Connect to.................: ufdb://%s:%d", SelfRemoteIP, SelfRemotePort)
			log.Printf("[UFDB_CLASS]: SelfSquidGuardWebUseExternalUri: %d", SelfSquidGuardWebUseExternalUri)
			log.Printf("[UFDB_CLASS]: SelfSquidGuardWebExternalUri...: %s", SelfSquidGuardWebExternalUri)
			log.Printf("[UFDB_CLASS]: SelfSquidGuardWebExternalUriSSL: %s", SelfSquidGuardWebExternalUriSSL)
			log.Printf("[UFDB_CLASS]: Listen port: %s:%d", SelfRemoteIP, SelfRemotePort)
			if SelfMimik {
				log.Printf("[UFDB_CLASS]:MIMIK = %t", SelfMimik)
			}
		}
	}

}
func isBumped() {
	if _, err := os.Stat("/etc/squid3/listen_ports.conf"); errors.Is(err, os.ErrNotExist) {
		if isDebugUfdbguard {
			log.Printf("/etc/squid3/listen_ports.conf not exit: ", err)
		}
		return
	}
	f, err := os.Open("/etc/squid3/listen_ports.conf")

	if err != nil {
		if isDebugUfdbguard {
			log.Printf("error reading /etc/squid3/listen_ports.conf: ", err)
		}
		return
	}

	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		reSSLBump := regexp.MustCompile("^http_port.*?ssl-bump")
		if reSSLBump.MatchString(scanner.Text()) {
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]:MIMIK Found SSL port in proxy configuration")
			}
			SelfMimik = true
			return
		}
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]:MIMIK * not * Found SSL port in proxy configuration")
		}

	}
	if err := scanner.Err(); err != nil {
		if isDebugUfdbguard {
			log.Printf("error scanning /etc/squid3/listen_ports.conf: ", err)
		}
		return
	}
}
func checkLocalConfig() {
	if _, err := os.Stat("/etc/squid3/ufdbGuard.conf"); errors.Is(err, os.ErrNotExist) {
		if isDebugUfdbguard {
			log.Printf("/etc/squid3/ufdbGuard.conf not exit: ", err)
		}
		return
	}
	f, err := os.Open("/etc/squid3/ufdbGuard.conf")

	if err != nil {
		if isDebugUfdbguard {
			log.Printf("error reading /etc/squid3/ufdbGuard.conf: ", err)
		}
		return
	}

	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		reInterface := regexp.MustCompile("^interface\\s+(.+)")
		if reInterface.MatchString(scanner.Text()) {
			rs := reInterface.FindStringSubmatch(scanner.Text())
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: Found Interface %s in ufdbGuard.conf", rs[1])
			}
			SelfRemoteIP = rs[1]
			if SelfRemoteIP == "all" {
				SelfRemoteIP = "127.0.0.1"
			}
		}
		rePort := regexp.MustCompile("^port\\s+([0-9]+)")
		if rePort.MatchString(scanner.Text()) {
			rs := rePort.FindStringSubmatch(scanner.Text())
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: Found Port %s in ufdbGuard.conf", rs[1])
			}
			SelfRemotePort, _ = strconv.Atoi(rs[1])
		}

	}
	if err := scanner.Err(); err != nil {
		if isDebugUfdbguard {
			log.Printf("error scanning /etc/squid3/ufdbGuard.conf: ", err)
		}
		return
	}

}
func Process(proxyUrl string, urlDomain string, clientIp string, clientHostname string, username string, clientMacAddr string, proxyIP string, proxyPort int) bool {
	if isDebugUfdbguard {
		log.Printf("Starting UFDB Process")
	}
	sourceUrl := proxyUrl
	categoryName := "Unknown"
	toUfdbCdir := ""
	toUfdb := ""
	proto := ""
	cdirToCheck := ""
	SelfCategoryName = categoryName
	SelfToken = ""
	var Tokens []string
	if len(username) == 0 {
		username = "-"
	}
	if len(clientHostname) == 0 {
		clientHostname = "-"
	}
	if len(proxyIP) == 0 {
		proxyIP = "127.0.0.1"
	}
	if proxyPort == 0 {
		proxyPort = 3128
	}

	matches, _ := regexp.MatchString("\\/ufdbguard\\.php\\?rule-id=[0-9]+", proxyUrl)
	if matches {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: [CLIENT]: Loop to Web-filtering error page")
		}
		return false
	}

	if checkIPAddressType(urlDomain, isDebugUfdbguard) {
		re := regexp.MustCompile("^([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)")
		if re.MatchString(urlDomain) {
			rs := re.FindStringSubmatch(urlDomain)
			cdirToCheck := rs[1] + "." + rs[2] + "." + rs[3] + ".cdir"
			toUfdbCdir = fmt.Sprintf("http://%s %s/%s %s GET myip=%s myport=%s\n", cdirToCheck, clientIp, clientHostname, username, proxyIP, strconv.Itoa(proxyPort))
		}
		ip2LongDomain := binary.BigEndian.Uint32(net.ParseIP(urlDomain)[12:16])
		newDomain := strconv.FormatUint(uint64(ip2LongDomain), 10) + ".addr"
		proxyUrl = strings.ReplaceAll(proxyUrl, urlDomain, newDomain)
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: [CLIENT] replace [' + %s + '] to [' + %s + ']:", urlDomain, newDomain)
		}
		SelfProxyProto = "GET"
	}
	proxyUrl = strings.ReplaceAll(proxyUrl, "https", "http")
	proxyUrl = strings.ReplaceAll(proxyUrl, ":443", "")
	if strings.Index(proxyUrl, "http://") == -1 {
		proto = "http://"
	}
	if SelfUfdbGuardMaxUrisize == 0 {
		SelfUfdbGuardMaxUrisize = 640
	}
	if len(proxyUrl) > SelfUfdbGuardMaxUrisize {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: [CLIENT] ALERT!...: URL %s exceed %d bytes, cut it!", urlDomain, SelfUfdbGuardMaxUrisize)
		}
		proxyUrl = proxyUrl[0:SelfUfdbGuardMaxUrisize] + "..."
	}
	if len(toUfdbCdir) > 0 {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: [CLIENT] Pass to Web-Filtering service (CDIR)")
		}
		SelfClientMac = clientMacAddr
		if sendToUfdb(toUfdbCdir, sourceUrl, clientIp, username, cdirToCheck) {
			categoryName = "Unknown"
			finalRedirdectCode := SelfFinalRedirdectCode
			finalRedirectUrl := SelfFinalRedirectUrl
			finalRedirectKey := SelfRedirectKey
			category := SelfCategory
			if category > 0 {
				categoryName = categorization.CategoryIntToString(category)
				SelfCategoryName = categoryName
			}
			if isDebugUfdbguard {
				log.Printf("[UFDB_CLASS]: SelfCategory %d Name: %s", category, categoryName)
				log.Printf("[UFDB_CLASS]: [CLIENT] CATEGORY=%d", SelfCategory)
			}
			ap1 := fmt.Sprintf("status=%s %s=%s", finalRedirdectCode, finalRedirectKey, finalRedirectUrl)
			Tokens = append(Tokens, ap1)
			Tokens = append(Tokens, "shieldsblock=yes")
			Tokens = append(Tokens, SelfWebfilteringToken)
			ap2 := fmt.Sprintf("category=%s category-name=%s clog=cinfo:%s-%s;", strconv.Itoa(category), categoryName, strconv.Itoa(category), categoryName)
			Tokens = append(Tokens, ap2)
			SelfToken = strings.Join(Tokens, " ")
			return true

		}
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: FATAL! Exception while requesting CDIR to Web-Filtering Engine service")
		}
		return false
	}
	toUfdb = fmt.Sprintf("%s%s %s/%s %s GET myip=%s myport=%s\n", proto, proxyUrl, clientIp, clientHostname, username, proxyIP, strconv.Itoa(proxyPort))

	if sendToUfdb(toUfdb, sourceUrl, clientIp, username, clientHostname) {
		category := SelfCategory
		if category > 0 {
			categoryName = categorization.CategoryIntToString(category)
			SelfCategoryName = categoryName
		}
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: SelfCategory %d Name: %s", category, categoryName)
		}
		return true
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS]: FATAL! Exception while requesting Web-Filtering Engine service")
	}
	return false
}

func sendToUfdb(query string, sourceUrl string, clientIp string, uid string, hostname string) bool {
	if isDebugUfdbguard {
		log.Printf("Starting SentToUfdb")
	}
	connected := false
	redirection := ""
	key := "url"
	categoryFound := ""
	SelfRuleID = 0
	SelfWebfilterRuleName = ""
	response := ""
	var matches bool

	matches, _ = regexp.MatchString("\\s+CONNECT\\s+", query)
	if matches {
		connected = true
	}
	if SelfProxyProto == "CONNECT" {
		connected = true
	}
	matches, _ = regexp.MatchString("^([0-9\\.]+)$", hostname)
	if !matches {
		SelfIsSSNI = true
	}
	//To Improve Later
	if SelfMimik {
		if SelfHttps {
			if !SelfIsSSNI {
				if isDebugUfdbguard {
					log.Printf("[UFDB_CLASS]: %s: MIMIK but SNI not set, return false", hostname)
					log.Printf("[UFDB_CLASS]: OK: PASS")
					return false
				}
			}
		}
	}
	if connected {
		if SelfMimik {
			matches, _ = regexp.MatchString("^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])$", hostname)
			if matches {
				if isDebugUfdbguard {
					log.Printf("[UFDB_CLASS]: [%s]:IPv4 Connect received, but mimiked proxy, waiting bumped session..", hostname)
					log.Printf("[UFDB_CLASS]: OK: PASS")
				}
				return false
			}
			matches, _ = regexp.MatchString("^(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|0?[0-9]?[0-9])$", hostname)
			if matches {
				if isDebugUfdbguard {
					log.Printf("[UFDB_CLASS]: [%s]:IPv6 Connect received, but mimiked proxy, waiting bumped session..", hostname)
					log.Printf("[UFDB_CLASS]: OK: PASS")
				}
				return false
			}
		}
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS][222]: query: [%s]", strings.TrimSpace(query))
	}
	SelfCached = false
	md5query := internal.Md5string("UFDBCACHE_" + strings.TrimSpace(query))
	if SelfUfdbGuardWebFilteringCacheTime > 1 {
		//val, err := internal.MC.Get(md5query)
		//if err != nil {
		//	if isDebugUfdbguard {
		//		log.Printf("Error Getting MD5 from memcache: %s", err)
		//	}
		//	panic(err)
		//}
		//if val != nil {
		//	SelfCached = true
		//	response = string(val.Value)
		//
		//}
		if val, err := internal.Fetch(md5query); val != nil {
			SelfCached = true
			response = *val
		} else {
			if isDebugUfdbguard {
				log.Printf("Error Getting MD5 from in memory cache: %s", err)
			}
		}
	}

	if !SelfCached {

		response = sendSocket(query)
		if isDebugUfdbguard {
			log.Printf("####### RESPONSE FROM UFDB WITH VAL %s##############", response)
		}
	} else {
		if isDebugUfdbguard {
			log.Printf("####### RESPONSE FROM CACHE WITH VAL %s##############", response)
		}
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS][360]: response: %s", response)
	}
	if SelfInactiveService {
		return false
	}
	if response == "OK" {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: OK: PASS")
		}
		if !SelfCached {
			if SelfUfdbGuardWebFilteringCacheTime > 1 {
				//internal.MC.Set(&memcache.Item{key: md5query, Value: []byte(response), Expiration: int32(SelfUfdbGuardWebFilteringCacheTime)})
				internal.Append(md5query, response)
			}
		}
		return false
	}
	if len(response) == 0 {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS][238]: UNKNOWN: \"PASS\"")
		}
		return false
	}
	if !SelfCached {
		if SelfUfdbGuardWebFilteringCacheTime > 1 {
			//internal.MC.Set(&memcache.Item{key: md5query, Value: []byte(response), Expiration: int32(SelfUfdbGuardWebFilteringCacheTime)})
			internal.Append(md5query, response)
		}
	}
	re := regexp.MustCompile("rewrite-url=\"(.*?)\"")
	matches = re.MatchString(response)
	key = "rewrite"
	rs := re.FindStringSubmatch(response)
	if !matches {
		re2 := regexp.MustCompile("url=\"(.*?)\"")
		matches = re2.MatchString(response)
		key = "url"
		rs = re2.FindStringSubmatch(response)
	}
	if matches {
		redirection = rs[1]

	}
	if len(redirection) == 0 {
		redirection = response
	}
	redirection = strings.ReplaceAll(redirection, "??", "?")
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS][297]: redirection = %s (299)", redirection)
	}
	RedirectionSource := redirection
	re3 := regexp.MustCompile("rule-id=([0-9]+).*?targetgroup=(.+?)&")
	matches = re3.MatchString(RedirectionSource)
	if matches {
		rs = re3.FindStringSubmatch(RedirectionSource)
		categoryFound = rs[2]
		categoryFound = strings.ReplaceAll(categoryFound, "P", "")
		SelfRuleID, _ = strconv.Atoi(rs[1])
	}
	re4 := regexp.MustCompile("clientgroup=(.+?)&")
	matches = re4.MatchString(RedirectionSource)
	if matches {
		rs = re4.FindStringSubmatch(RedirectionSource)
		SelfWebfilterRuleName = rs[1]
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS]: categoryFound = %s ruleid=%d", categoryFound, SelfRuleID)
	}

	if strings.Index(redirection, "=%a") > 0 {
		redirection = strings.ReplaceAll(redirection, "clientaddr=%a", "clientaddr="+clientIp)
	}

	if strings.Index(redirection, "=%i") > 0 {
		redirection = strings.ReplaceAll(redirection, "clientuser=%i", "clientuser="+uid)
	}

	if strings.Index(redirection, "=%u") > 0 {
		redirection = strings.ReplaceAll(redirection, "url=%u", "url="+url.PathEscape(sourceUrl))
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS]: redirection %s", redirection)
	}

	redirection = strings.ReplaceAll(redirection, "\"", "")
	SelfFinalRedirectUrl = redirection
	SelfRedirectKey = key
	SelfCategory, _ = strconv.Atoi(categoryFound)
	SelfWebfilteringToken = fmt.Sprintf("webfiltering=block,%s,%s  srcurl=\"%s\"", strconv.Itoa(SelfRuleID), categoryFound, url.PathEscape(sourceUrl))
	return true
}

func sendSocket(query string) string {
	SelfInactiveService = false
	response := ""
	if SelfRemotePort == 0 {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: Configuration Error, no port set... Aborting!")
		}
		return ""
	}
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS]: Send to service %s", query)
	}

	d := net.Dialer{Timeout: SelfUfdbgclientSockTimeOut}
	conn, err := d.Dial("tcp", SelfRemoteIP+":"+strconv.Itoa(SelfRemotePort))
	if err != nil {

		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: Connection Error: Unable to connect to %s %d ERROR: %s", SelfRemoteIP, SelfRemotePort, err)
		}
		return ""
	}
	fmt.Fprintf(conn, query+"\n")
	message, errB := bufio.NewReader(conn).ReadString('\n')
	if errB != nil {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]: Connection Error:  Unable to receive data from %s %d ERROR: %s", SelfRemoteIP, SelfRemotePort, errB)
		}
		return ""
	}
	response = strings.TrimSpace(message)
	if isDebugUfdbguard {
		log.Printf("[UFDB_CLASS]: RESPONSE: %s", response)
	}
	if strings.Index(response, "?loading-database=yes") == 0 {
		SelfInactiveService = true
	}
	if strings.Index(response, "?fatalerror=yes") == 0 {
		SelfInactiveService = true
	}
	if SelfInactiveService {
		if isDebugUfdbguard {
			log.Printf("[UFDB_CLASS]:FATAL Error Load-database or Web-Filtering error!!")
		}
	}
	return response
}
func checkIPAddressType(ip string, isDebugUfdbguard bool) bool {
	if net.ParseIP(ip) == nil {
		if isDebugUfdbguard {
			log.Printf("Invalid IP Address: %s\n", ip)
		}
		return false
	}
	for i := 0; i < len(ip); i++ {
		switch ip[i] {
		case '.':
			if isDebugUfdbguard {
				log.Printf("Given IP Address %s is IPV4 type\n", ip)
			}
			return true
		case ':':
			if isDebugUfdbguard {
				log.Printf("Given IP Address %s is IPV6 type\n", ip)
			}
			return true
		}
	}
	return false
}