Add: periodically audit domain registrations

Reviewed main.go, internal/dlc rule types and cmd/datdump: existing commands generate lists but do not audit registrations. Reuse rule constants and add an isolated checker with ICANN public suffix boundaries, registry RDAP, DNS confirmation and rule history.

Run a weekly advisory workflow and preserve source evidence for manual review. Compile all commands in PR CI. Go package build passed; test execution is left to the contributor.
This commit is contained in:
EpLiar committed 2026-11-07 12:18:28 +08:00
1 parent 16fd8005d3
commit eb7c97de44
12 files changed
+1214

No files matched your search

+54
View File
@@ -0,0 +1,54 @@
name: Audit domain registrations
on:
schedule:
- cron: "17 3 * * 1"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
audit:
name: Audit domain registrations
if: github.event_name != 'schedule' || github.repository == 'v2fly/domain-list-community'
runs-on: ubuntu-latest
timeout-minutes: 330
steps:
- name: Checkout codebase and domain history
uses: actions/checkout@v7
with:
fetch-depth: 0
persist-credentials: false
- name: Setup Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod
- name: Audit domain registrations
run: go run ./cmd/domaincheck --outputdir=domain-audit --timeout=5h
- name: Publish audit summary
if: always()
run: |
if [ -f domain-audit/summary.md ]; then
cat domain-audit/summary.md >> "$GITHUB_STEP_SUMMARY"
else
echo "::warning::The domain audit did not produce a summary. Check the audit logs."
echo "The domain audit did not produce a summary. Check the audit logs." >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload audit report
if: always()
uses: actions/upload-artifact@v7
with:
name: domain-audit-${{ github.run_number }}
path: |
domain-audit/report.json
domain-audit/summary.md
if-no-files-found: error
+4
View File
@@ -27,6 +27,10 @@ jobs:
go-version-file: test/go.mod
cache-dependency-path: test/go.sum
- name: Build Go commands
working-directory: test
run: go build ./...
- name: Build .dat and .yml
run: |
[ -d ./base/data/ ] && [ -d ./test/data/ ] || exit 1
+1
View File
@@ -3,6 +3,7 @@
/domain-list-community
/domain-list-community.exe
/domain-audit/
# Generated dat files.
/*.dat
+302
View File
@@ -0,0 +1,302 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
)
const bootstrapURL = "https://data.iana.org/rdap/dns.json"
type checkResult struct {
*domainEntry
Status string `json:"status"`
RDAPURL string `json:"rdap_url,omitempty"`
HTTPStatus int `json:"http_status,omitempty"`
Registered time.Time `json:"registered"`
DNS []string `json:"dns,omitempty"`
RegistryHold []string `json:"registry_status,omitempty"`
Detail string `json:"detail,omitempty"`
}
type rdapResponse struct {
ObjectClassName string `json:"objectClassName"`
LDHName string `json:"ldhName"`
Status []string `json:"status"`
ErrorCode int `json:"errorCode"`
Events []struct {
Action string `json:"eventAction"`
Date string `json:"eventDate"`
} `json:"events"`
}
type hostLimiter struct {
mu sync.Mutex
next time.Time
}
func (limiter *hostLimiter) wait(ctx context.Context, interval time.Duration) error {
for {
if err := ctx.Err(); err != nil {
return err
}
limiter.mu.Lock()
delay := time.Until(limiter.next)
if delay <= 0 {
limiter.next = time.Now().Add(interval)
limiter.mu.Unlock()
return nil
}
limiter.mu.Unlock()
if err := pause(ctx, delay); err != nil {
return err
}
}
}
func (limiter *hostLimiter) deferUntil(until time.Time) {
limiter.mu.Lock()
defer limiter.mu.Unlock()
if until.After(limiter.next) {
limiter.next = until
}
}
type checker struct {
client *http.Client
services map[string]string
interval time.Duration
limiters sync.Map
resolvers []string
}
func (checker *checker) limiterFor(host string) *hostLimiter {
value, _ := checker.limiters.LoadOrStore(host, &hostLimiter{})
return value.(*hostLimiter)
}
func newChecker(ctx context.Context, client *http.Client, interval time.Duration) (*checker, error) {
result := &checker{
client: client, interval: interval, services: make(map[string]string),
resolvers: []string{"https://dns.google/resolve", "https://cloudflare-dns.com/dns-query"},
}
clientCopy := *client
clientCopy.CheckRedirect = func(request *http.Request, previous []*http.Request) error {
if len(previous) >= 10 || request.URL.Scheme != "https" {
return fmt.Errorf("refusing excessive redirects or an insecure redirect")
}
if client.CheckRedirect != nil {
if err := client.CheckRedirect(request, previous); err != nil {
return err
}
}
return result.limiterFor(request.URL.Host).wait(request.Context(), interval)
}
result.client = &clientCopy
var bootstrap struct {
Services [][][]string `json:"services"`
}
status, err := result.getJSON(ctx, bootstrapURL, &bootstrap)
if err != nil {
return nil, err
}
if status != http.StatusOK {
return nil, fmt.Errorf("bootstrap returned HTTP %d", status)
}
for _, service := range bootstrap.Services {
if len(service) != 2 {
return nil, fmt.Errorf("invalid RDAP bootstrap service")
}
for _, endpoint := range service[1] {
parsed, err := url.Parse(endpoint)
if err != nil || parsed.Scheme != "https" || parsed.Host == "" {
continue
}
for _, suffix := range service[0] {
result.services[strings.ToLower(suffix)] = strings.TrimRight(endpoint, "/")
}
break
}
}
if len(result.services) == 0 {
return nil, fmt.Errorf("RDAP bootstrap contains no HTTPS services")
}
return result, nil
}
func pause(ctx context.Context, delay time.Duration) error {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func retryDelay(value string, attempt int) time.Duration {
if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {
return time.Duration(seconds) * time.Second
}
if date, err := http.ParseTime(value); err == nil && time.Until(date) > 0 {
return time.Until(date)
}
return time.Duration(1<<attempt) * time.Second
}
func (checker *checker) getJSON(ctx context.Context, endpoint string, target any) (int, error) {
parsed, err := url.Parse(endpoint)
if err != nil {
return 0, err
}
limiter := checker.limiterFor(parsed.Host)
for attempt := 0; attempt < 3; attempt++ {
if err := limiter.wait(ctx, checker.interval); err != nil {
return 0, err
}
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
if err != nil {
return 0, err
}
request.Header.Set("Accept", "application/rdap+json, application/json")
if parsed.Query().Get("type") == "NS" {
request.Header.Set("Accept", "application/dns-json")
}
request.Header.Set("User-Agent", "domain-list-community-domain-audit/1.0 (+https://github.com/v2fly/domain-list-community)")
response, requestErr := checker.client.Do(request)
if requestErr != nil {
if attempt < 2 && ctx.Err() == nil {
limiter.deferUntil(time.Now().Add(retryDelay("", attempt)))
continue
}
return 0, requestErr
}
status := response.StatusCode
if status == http.StatusTooManyRequests || status >= 500 {
_ = response.Body.Close()
until := time.Now().Add(retryDelay(response.Header.Get("Retry-After"), attempt))
limiter.deferUntil(until)
checker.limiterFor(response.Request.URL.Host).deferUntil(until)
if attempt < 2 {
continue
}
return status, fmt.Errorf("HTTP %d after three attempts", status)
}
// A bare proxy/web-server 404 is not evidence of a missing RDAP object.
if status != http.StatusOK && status != http.StatusNotFound {
_ = response.Body.Close()
return status, fmt.Errorf("HTTP %d", status)
}
decodeErr := json.NewDecoder(io.LimitReader(response.Body, 2*1024*1024)).Decode(target)
_ = response.Body.Close()
return status, decodeErr
}
return 0, fmt.Errorf("request attempts exhausted")
}
func (checker *checker) rdapURL(domain string) string {
for suffix := domain; suffix != ""; {
if endpoint := checker.services[suffix]; endpoint != "" {
return endpoint + "/domain/" + url.PathEscape(domain)
}
_, rest, found := strings.Cut(suffix, ".")
if !found {
break
}
suffix = rest
}
return ""
}
func (checker *checker) dnsStatus(ctx context.Context, resolver, domain string) string {
endpoint := resolver + "?name=" + url.QueryEscape(domain+".") + "&type=NS"
var response struct {
Status *int `json:"Status"`
Question []struct {
Name string `json:"name"`
Type int `json:"type"`
} `json:"Question"`
}
status, err := checker.getJSON(ctx, endpoint, &response)
if err != nil || status != http.StatusOK || response.Status == nil {
return "unknown"
}
if len(response.Question) != 1 || !strings.EqualFold(strings.TrimSuffix(response.Question[0].Name, "."), domain) || response.Question[0].Type != 2 {
return "unknown"
}
switch *response.Status {
case 0:
return "NOERROR"
case 3:
return "NXDOMAIN"
default:
return fmt.Sprintf("RCODE_%d", *response.Status)
}
}
func (checker *checker) check(ctx context.Context, entry *domainEntry) checkResult {
result := checkResult{domainEntry: entry, Status: "unknown"}
result.RDAPURL = checker.rdapURL(entry.Domain)
if result.RDAPURL == "" {
result.Status = "unsupported"
result.Detail = "No HTTPS RDAP service in IANA bootstrap"
return result
}
var response rdapResponse
status, err := checker.getJSON(ctx, result.RDAPURL, &response)
result.HTTPStatus = status
if err != nil {
result.Detail = err.Error()
return result
}
if status == http.StatusNotFound {
if response.ErrorCode != http.StatusNotFound {
result.Detail = "HTTP 404 without an RDAP errorCode of 404"
return result
}
for _, resolver := range checker.resolvers {
result.DNS = append(result.DNS, checker.dnsStatus(ctx, resolver, entry.Domain))
}
if len(result.DNS) == 2 && result.DNS[0] == "NXDOMAIN" && result.DNS[1] == "NXDOMAIN" {
result.Status = "possibly_unregistered"
result.Detail = "Registry RDAP 404 and NXDOMAIN from two resolvers; review before removal"
} else {
result.Detail = "Registry RDAP 404 without two NXDOMAIN confirmations"
}
return result
}
if response.ObjectClassName != "domain" || !strings.EqualFold(strings.TrimSuffix(response.LDHName, "."), entry.Domain) {
result.Detail = "RDAP response does not identify the queried domain"
return result
}
result.Status = "registered"
result.RegistryHold = response.Status
for _, event := range response.Events {
if event.Action != "registration" {
continue
}
registered, err := time.Parse(time.RFC3339, event.Date)
if err != nil {
result.Status = "unknown"
result.Detail = "Invalid RDAP registration event date"
return result
}
if result.Registered.IsZero() || registered.Before(result.Registered) {
result.Registered = registered
}
}
// Registry timestamps and commit clocks may differ; require a full day of separation.
if !entry.FirstSeen.IsZero() && result.Registered.After(entry.FirstSeen.Add(24*time.Hour)) {
result.Status = "possibly_reregistered"
result.Detail = "Registration postdates earliest rule history by more than 24 hours; ownership needs manual verification"
}
return result
}
+159
View File
@@ -0,0 +1,159 @@
package main
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strings"
"sync/atomic"
"testing"
"time"
)
func TestCheckRegistryAndDNS(t *testing.T) {
firstSeen := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
registered := `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2000-01-01T00:00:00Z"}]}`
testCases := []struct {
name string
rdapStatus int
rdapBody string
dnsBodies []string
wantStatus string
wantDNS []string
wantHold []string
}{
{name: "registered", rdapStatus: 200, rdapBody: registered, wantStatus: "registered"},
{name: "registered on hold", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","status":["server hold"]}`, wantStatus: "registered", wantHold: []string{"server hold"}},
{name: "two NXDOMAIN confirmations", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(3), dnsResponse(3)}, wantStatus: "possibly_unregistered", wantDNS: []string{"NXDOMAIN", "NXDOMAIN"}},
{name: "NOERROR without answers is not NXDOMAIN", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(3), dnsResponse(0)}, wantStatus: "unknown", wantDNS: []string{"NXDOMAIN", "NOERROR"}},
{name: "SERVFAIL is not NXDOMAIN", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(2), dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"RCODE_2", "NXDOMAIN"}},
{name: "missing DNS status", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{`{"Question":[{"name":"example.com.","type":2}]}`, dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"unknown", "NXDOMAIN"}},
{name: "different DNS question", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{`{"Status":3,"Question":[{"name":"other.com.","type":2}]}`, dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"unknown", "NXDOMAIN"}},
{name: "HTML 404", rdapStatus: 404, rdapBody: "<html>Not found</html>", wantStatus: "unknown"},
{name: "404 without RDAP error", rdapStatus: 404, rdapBody: `{}`, wantStatus: "unknown"},
{name: "RDAP error disagrees with HTTP", rdapStatus: 404, rdapBody: `{"errorCode":400}`, wantStatus: "unknown"},
{name: "RDAP domain mismatch", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"other.com"}`, wantStatus: "unknown"},
{name: "RDAP object class mismatch", rdapStatus: 200, rdapBody: `{"objectClassName":"entity","ldhName":"example.com"}`, wantStatus: "unknown"},
{name: "invalid registration date", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"invalid"}]}`, wantStatus: "unknown"},
{name: "registration exactly 24 hours later", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2020-01-02T00:00:00Z"}]}`, wantStatus: "registered"},
{name: "registration beyond timestamp margin", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"EXAMPLE.COM.","events":[{"eventAction":"registration","eventDate":"2020-01-02T00:00:01Z"}]}`, wantStatus: "possibly_reregistered"},
{name: "earliest registration event", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2024-01-01T00:00:00Z"},{"eventAction":"registration","eventDate":"2000-01-01T00:00:00Z"}]}`, wantStatus: "registered"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Path == "/rdap/domain/example.com" {
writer.Header().Set("Content-Type", "application/rdap+json")
writer.WriteHeader(tc.rdapStatus)
fmt.Fprint(writer, tc.rdapBody)
return
}
index := slices.Index([]string{"/dns-a", "/dns-b"}, request.URL.Path)
if index < 0 || index >= len(tc.dnsBodies) {
t.Errorf("unexpected DNS request: %s", request.URL)
writer.WriteHeader(http.StatusBadRequest)
return
}
if request.URL.Query().Get("name") != "example.com." || request.URL.Query().Get("type") != "NS" {
t.Errorf("DNS query = %s, want registration domain NS question", request.URL.RawQuery)
}
if request.Header.Get("Accept") != "application/dns-json" {
t.Errorf("DNS Accept header = %q, want application/dns-json", request.Header.Get("Accept"))
}
writer.Header().Set("Content-Type", "application/dns-json")
fmt.Fprint(writer, tc.dnsBodies[index])
}))
defer server.Close()
checker := &checker{
client: server.Client(), services: map[string]string{"com": server.URL + "/rdap"},
interval: time.Nanosecond, resolvers: []string{server.URL + "/dns-a", server.URL + "/dns-b"},
}
result := checker.check(context.Background(), &domainEntry{Domain: "example.com", FirstSeen: firstSeen})
if result.Status != tc.wantStatus {
t.Errorf("check() status = %q, want %q; detail: %s", result.Status, tc.wantStatus, result.Detail)
}
if !slices.Equal(result.DNS, tc.wantDNS) {
t.Errorf("DNS evidence = %v, want %v", result.DNS, tc.wantDNS)
}
if !slices.Equal(result.RegistryHold, tc.wantHold) {
t.Errorf("registry status = %v, want %v", result.RegistryHold, tc.wantHold)
}
if result.HTTPStatus != tc.rdapStatus || result.RDAPURL != server.URL+"/rdap/domain/example.com" {
t.Errorf("registry evidence = HTTP %d %q, want original registry response", result.HTTPStatus, result.RDAPURL)
}
})
}
}
func dnsResponse(status int) string {
return fmt.Sprintf(`{"Status":%d,"Question":[{"name":"example.com.","type":2}]}`, status)
}
func TestCheckRequiresHistoryForReregistration(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
fmt.Fprint(writer, `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2024-01-01T00:00:00Z"}]}`)
}))
defer server.Close()
checker := &checker{client: server.Client(), services: map[string]string{"com": server.URL}, interval: time.Nanosecond}
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
if result.Status != "registered" || result.Registered.IsZero() {
t.Errorf("check without history = %+v, want registered with date and no ownership inference", result)
}
}
func TestCheckUnsupportedRegistry(t *testing.T) {
checker := &checker{services: map[string]string{}}
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
if result.Status != "unsupported" || result.RDAPURL != "" {
t.Errorf("check unsupported registry = %+v, want unsupported without a network request", result)
}
}
func TestCheckDeadlineIsUnknown(t *testing.T) {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancel()
checker := &checker{client: &http.Client{}, services: map[string]string{"com": "https://registry.invalid"}, interval: time.Nanosecond}
result := checker.check(ctx, &domainEntry{Domain: "example.com"})
if result.Status != "unknown" || !strings.Contains(result.Detail, "deadline exceeded") {
t.Errorf("expired audit deadline = %+v, want unknown with deadline detail", result)
}
}
func TestCheckRetriesTemporaryRegistryFailures(t *testing.T) {
testCases := []struct {
name string
failureStatus int
failures int32
wantRequests int32
wantHTTP int
wantStatus string
}{
{name: "rate limit then success", failureStatus: http.StatusTooManyRequests, failures: 1, wantRequests: 2, wantHTTP: http.StatusOK, wantStatus: "registered"},
{name: "unavailable then success", failureStatus: http.StatusServiceUnavailable, failures: 1, wantRequests: 2, wantHTTP: http.StatusOK, wantStatus: "registered"},
{name: "persistent rate limit", failureStatus: http.StatusTooManyRequests, failures: 3, wantRequests: 3, wantHTTP: http.StatusTooManyRequests, wantStatus: "unknown"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
var requests atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if requests.Add(1) <= tc.failures {
writer.Header().Set("Retry-After", "0")
writer.WriteHeader(tc.failureStatus)
return
}
fmt.Fprint(writer, `{"objectClassName":"domain","ldhName":"example.com"}`)
}))
defer server.Close()
checker := &checker{client: server.Client(), services: map[string]string{"com": server.URL}, interval: time.Nanosecond}
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
if requests.Load() != tc.wantRequests {
t.Errorf("registry request count = %d, want %d", requests.Load(), tc.wantRequests)
}
if result.Status != tc.wantStatus || result.HTTPStatus != tc.wantHTTP {
t.Errorf("registry result = %q HTTP %d, want %q HTTP %d; detail: %s", result.Status, result.HTTPStatus, tc.wantStatus, tc.wantHTTP, result.Detail)
}
})
}
}
+198
View File
@@ -0,0 +1,198 @@
package main
import (
"bufio"
"context"
"fmt"
"io"
"io/fs"
"net"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"time"
"github.com/v2fly/domain-list-community/internal/dlc"
"golang.org/x/net/publicsuffix"
)
type source struct {
File string `json:"file"`
Line int `json:"line"`
Rule string `json:"rule"`
}
type domainEntry struct {
Domain string `json:"domain"`
Sources []source `json:"sources"`
FirstSeen time.Time `json:"first_seen"`
}
type inventory struct {
Domains map[string]*domainEntry
LiteralRules int
Skipped map[string]int
}
func literalDomain(line string) (string, string) {
line, _, _ = strings.Cut(line, "#")
line = strings.TrimSpace(line)
if line == "" {
return "", ""
}
typ, value, specified := strings.Cut(strings.ToLower(line), ":")
if !specified {
return strings.Fields(typ)[0], ""
}
switch typ {
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain:
fields := strings.Fields(value)
if len(fields) == 0 {
return "", "empty_domain"
}
return fields[0], ""
default:
return "", typ
}
}
func registrationDomain(host string) string {
if net.ParseIP(host) != nil {
return ""
}
// Ignore private PSL boundaries: a tenant of github.io is not a registry object.
for candidate := host; candidate != ""; {
suffix, icann := publicsuffix.PublicSuffix(candidate)
if icann {
if host == suffix {
return ""
}
prefix := strings.TrimSuffix(host, "."+suffix)
labels := strings.Split(prefix, ".")
return labels[len(labels)-1] + "." + suffix
}
_, rest, found := strings.Cut(candidate, ".")
if !found {
return ""
}
candidate = rest
}
return ""
}
func loadInventory(dataPath string) (*inventory, error) {
result := &inventory{Domains: make(map[string]*domainEntry), Skipped: make(map[string]int)}
err := filepath.WalkDir(dataPath, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if entry.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
lineNumber := 0
for scanner.Scan() {
lineNumber++
host, skip := literalDomain(scanner.Text())
if skip != "" {
result.Skipped[skip]++
continue
}
if host == "" {
continue
}
result.LiteralRules++
domain := registrationDomain(host)
if domain == "" {
result.Skipped["no_registry_domain"]++
continue
}
if result.Domains[domain] == nil {
result.Domains[domain] = &domainEntry{Domain: domain}
}
result.Domains[domain].Sources = append(result.Domains[domain].Sources, source{
File: filepath.ToSlash(path), Line: lineNumber, Rule: strings.TrimSpace(scanner.Text()),
})
}
return scanner.Err()
})
if err != nil {
return nil, err
}
if len(result.Domains) == 0 {
return nil, fmt.Errorf("no registrable domains found in %q", dataPath)
}
return result, nil
}
func readHistory(reader io.Reader, domains map[string]*domainEntry) error {
scanner := bufio.NewScanner(reader)
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
var committed time.Time
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "DLC-COMMIT ") {
parsed, err := time.Parse(time.RFC3339, strings.TrimPrefix(line, "DLC-COMMIT "))
if err != nil {
return err
}
committed = parsed
continue
}
if !strings.HasPrefix(line, "+") || strings.HasPrefix(line, "+++") || committed.IsZero() {
continue
}
host, _ := literalDomain(strings.TrimPrefix(line, "+"))
if host == "" {
continue
}
entry := domains[registrationDomain(host)]
if entry != nil && (entry.FirstSeen.IsZero() || committed.Before(entry.FirstSeen)) {
entry.FirstSeen = committed
}
}
return scanner.Err()
}
func loadHistory(ctx context.Context, dataPath string, domains map[string]*domainEntry) error {
shallow, err := exec.CommandContext(ctx, "git", "rev-parse", "--is-shallow-repository").Output()
if err != nil {
return err
}
if strings.TrimSpace(string(shallow)) != "false" {
return fmt.Errorf("complete git history is required; checkout with fetch-depth: 0")
}
command := exec.CommandContext(ctx, "git", "log", "--format=DLC-COMMIT %cI", "--no-renames", "--diff-filter=AM", "--unified=0", "-p", "--", dataPath)
command.Stderr = os.Stderr
pipe, err := command.StdoutPipe()
if err != nil {
return err
}
if err := command.Start(); err != nil {
return err
}
readErr := readHistory(pipe, domains)
if readErr != nil {
_ = command.Process.Kill()
}
if err := command.Wait(); err != nil {
return err
}
return readErr
}
func sortedDomains(domains map[string]*domainEntry) []string {
names := make([]string, 0, len(domains))
for name := range domains {
names = append(names, name)
}
slices.Sort(names)
return names
}
+120
View File
@@ -0,0 +1,120 @@
package main
import (
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
)
func TestRegistrationDomain(t *testing.T) {
testCases := []struct {
host string
want string
}{
{host: "mail.example.com", want: "example.com"},
{host: "mail.example.co.uk", want: "example.co.uk"},
{host: "mail.example.com.cn", want: "example.com.cn"},
{host: "tenant.github.io", want: "github.io"},
{host: "tenant.blogspot.com", want: "blogspot.com"},
{host: "co.uk"},
{host: "com"},
{host: "router.local"},
{host: "hidden.onion"},
{host: "127.0.0.1"},
{host: "::1"},
}
for _, tc := range testCases {
t.Run(tc.host, func(t *testing.T) {
if got := registrationDomain(tc.host); got != tc.want {
t.Errorf("registrationDomain(%q) = %q, want %q", tc.host, got, tc.want)
}
})
}
}
func TestLoadInventory(t *testing.T) {
dataPath := t.TempDir()
files := map[string]string{
"first": "# Source rules\nExample.COM @cn &second # comment\nfull:Mail.Example.COM @ads\ninclude:second\nkeyword:example\nregexp:^example\\.com$\ngoogle\nrouter.local\n",
"second": "domain: example.com @ads\nfull:api.example.co.uk\n",
}
for name, content := range files {
if err := os.WriteFile(filepath.Join(dataPath, name), []byte(content), 0644); err != nil {
t.Fatalf("failed to write list %q: %v", name, err)
}
}
data, err := loadInventory(dataPath)
if err != nil {
t.Fatalf("loadInventory() got unexpected error: %v", err)
}
if got := sortedDomains(data.Domains); !slices.Equal(got, []string{"example.co.uk", "example.com"}) {
t.Fatalf("registration domains = %v, want example.co.uk and example.com", got)
}
if data.LiteralRules != 6 {
t.Errorf("literal rule count = %d, want 6", data.LiteralRules)
}
for kind, count := range map[string]int{"include": 1, "keyword": 1, "regexp": 1, "no_registry_domain": 2} {
if data.Skipped[kind] != count {
t.Errorf("skipped %s = %d, want %d", kind, data.Skipped[kind], count)
}
}
sources := data.Domains["example.com"].Sources
if len(sources) != 3 {
t.Fatalf("example.com source count = %d, want 3", len(sources))
}
if sources[0].File != filepath.ToSlash(filepath.Join(dataPath, "first")) || sources[0].Line != 2 || sources[0].Rule != "Example.COM @cn &second # comment" {
t.Errorf("first source = %+v, want original rule at first:2", sources[0])
}
if sources[1].Line != 3 || sources[2].Line != 1 {
t.Errorf("source line numbers = %d, %d, want 3, 1", sources[1].Line, sources[2].Line)
}
}
func TestLoadInventoryRejectsEmptyInput(t *testing.T) {
if _, err := loadInventory(t.TempDir()); err == nil {
t.Fatal("loadInventory() accepted an empty data directory")
}
}
func TestReadHistoryUsesEarliestRegistrationDomainAddition(t *testing.T) {
domains := map[string]*domainEntry{
"example.com": {Domain: "example.com"},
"github.io": {Domain: "github.io"},
"missing.org": {Domain: "missing.org"},
}
history := strings.Join([]string{
"DLC-COMMIT 2024-02-01T00:00:00Z",
"+++ b/data/first",
"+domain:example.com @cn",
"+tenant.github.io",
"DLC-COMMIT 2020-01-01T00:00:00Z",
"+full:mail.example.com @ads",
"DLC-COMMIT 2019-01-01T00:00:00Z",
"+include:missing.org",
"+regexp:^missing\\.org$",
"-missing.org",
"+unrelated.net",
}, "\n")
if err := readHistory(strings.NewReader(history), domains); err != nil {
t.Fatalf("readHistory() got unexpected error: %v", err)
}
wantFirstSeen := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
if !domains["example.com"].FirstSeen.Equal(wantFirstSeen) {
t.Errorf("example.com first seen = %v, want %v", domains["example.com"].FirstSeen, wantFirstSeen)
}
if domains["github.io"].FirstSeen.Year() != 2024 {
t.Errorf("github.io first seen = %v, want tenant rule addition", domains["github.io"].FirstSeen)
}
if !domains["missing.org"].FirstSeen.IsZero() {
t.Errorf("nonliteral or removed rule was used as history: %v", domains["missing.org"].FirstSeen)
}
}
func TestReadHistoryRejectsInvalidCommitDate(t *testing.T) {
if err := readHistory(strings.NewReader("DLC-COMMIT invalid\n+example.com\n"), map[string]*domainEntry{}); err == nil {
t.Fatal("readHistory() accepted an invalid commit date")
}
}
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"context"
"flag"
"fmt"
"net/http"
"os"
"os/signal"
"sync"
"sync/atomic"
"time"
)
var (
dataPath = flag.String("datapath", "data", "Directory containing domain lists")
outputDir = flag.String("outputdir", "domain-audit", "Directory for audit reports")
workers = flag.Int("workers", 12, "Maximum concurrent domain checks")
interval = flag.Duration("interval", 500*time.Millisecond, "Minimum interval between requests to each HTTP host")
timeout = flag.Duration("timeout", 5*time.Hour, "Maximum audit duration, including retries")
)
func run(ctx context.Context) error {
if *workers < 1 || *interval <= 0 || *timeout <= 0 {
return fmt.Errorf("workers, interval and timeout must be positive")
}
ctx, cancel := context.WithTimeout(ctx, *timeout)
defer cancel()
data, err := loadInventory(*dataPath)
if err != nil {
return fmt.Errorf("load domain lists: %w", err)
}
fmt.Printf("Auditing %d registration domains from %d literal rules\n", len(data.Domains), data.LiteralRules)
if err := loadHistory(ctx, *dataPath, data.Domains); err != nil {
return fmt.Errorf("load domain history: %w", err)
}
client := &http.Client{Timeout: 20 * time.Second}
checker, err := newChecker(ctx, client, *interval)
if err != nil {
return fmt.Errorf("load RDAP bootstrap: %w", err)
}
names := sortedDomains(data.Domains)
results := make([]checkResult, len(names))
jobs := make(chan int)
var group sync.WaitGroup
var completed atomic.Int64
for range *workers {
group.Go(func() {
for index := range jobs {
results[index] = checker.check(ctx, data.Domains[names[index]])
if count := completed.Add(1); count%1000 == 0 || count == int64(len(names)) {
fmt.Printf("Checked %d/%d registration domains\n", count, len(names))
}
}
})
}
for index := range names {
jobs <- index
}
close(jobs)
group.Wait()
if err := writeReports(*outputDir, data, results); err != nil {
return err
}
if ctx.Err() != nil {
return fmt.Errorf("audit incomplete; partial results saved: %w", ctx.Err())
}
return nil
}
func main() {
flag.Parse()
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()
if err := run(ctx); err != nil {
fmt.Fprintf(os.Stderr, "[Error] %v\n", err)
os.Exit(1)
}
}
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"encoding/json"
"fmt"
"html"
"os"
"path/filepath"
"slices"
"strings"
"time"
)
func writeReports(outputDir string, data *inventory, results []checkResult) error {
counts := make(map[string]int)
missingHistory := 0
missingRegistration := 0
for _, result := range results {
counts[result.Status]++
if result.FirstSeen.IsZero() {
missingHistory++
}
if result.Status == "registered" && result.Registered.IsZero() {
missingRegistration++
}
}
report := struct {
GeneratedAt time.Time `json:"generated_at"`
Commit string `json:"commit,omitempty"`
LiteralRules int `json:"literal_rules"`
RegistrationDomains int `json:"registration_domains"`
SkippedRules map[string]int `json:"skipped_rules"`
Counts map[string]int `json:"counts"`
MissingHistory int `json:"missing_history"`
MissingRegistration int `json:"missing_registration_date"`
Results []checkResult `json:"results"`
}{
GeneratedAt: time.Now().UTC(), Commit: os.Getenv("GITHUB_SHA"),
LiteralRules: data.LiteralRules, RegistrationDomains: len(results), SkippedRules: data.Skipped,
Counts: counts, MissingHistory: missingHistory, MissingRegistration: missingRegistration, Results: results,
}
encoded, err := json.MarshalIndent(report, "", " ")
if err != nil {
return fmt.Errorf("encode audit report: %w", err)
}
if err := os.MkdirAll(outputDir, 0755); err != nil {
return err
}
if err := os.WriteFile(filepath.Join(outputDir, "report.json"), append(encoded, '\n'), 0644); err != nil {
return err
}
var summary strings.Builder
fmt.Fprintln(&summary, "# Domain registration audit")
fmt.Fprintf(&summary, "\nScanned %d literal rules, grouped into %d registration domains.\n", data.LiteralRules, len(results))
fmt.Fprintln(&summary, "\nThis is an advisory report for manual review. No rules are removed automatically. Registered does not mean ownership is correct. A later registration date suggests a lapse but does not prove a change of owner.")
fmt.Fprintln(&summary, "\n| Result | Domains |\n| --- | ---: |")
for _, status := range []string{"registered", "possibly_unregistered", "possibly_reregistered", "unknown", "unsupported"} {
fmt.Fprintf(&summary, "| %s | %d |\n", status, counts[status])
}
fmt.Fprintf(&summary, "\nUnknown and unsupported results are coverage gaps, not evidence of expiry. Registration-date comparison was unavailable for %d registered domains without a registration date, and %d domains without rule history.\n", missingRegistration, missingHistory)
if counts["unknown"] > 0 {
fmt.Fprintln(&summary, "\n**Coverage is incomplete.** Inspect `unknown` results in the JSON report before drawing conclusions.")
}
fmt.Fprintln(&summary, "\nSkipped rules:")
skipTypes := make([]string, 0, len(data.Skipped))
for typ := range data.Skipped {
skipTypes = append(skipTypes, typ)
}
slices.Sort(skipTypes)
for _, typ := range skipTypes {
fmt.Fprintf(&summary, "- %s: %d\n", markdownText(typ), data.Skipped[typ])
}
fmt.Fprintln(&summary, "\nIncludes are covered through their source files. Keywords and regular expressions cannot be queried as DNS names. TLD-only, special-use and private-network names without an ICANN registration boundary are excluded.")
fmt.Fprintln(&summary, "\n## Review candidates\n\nA missing registration requires registry RDAP `errorCode: 404` plus NXDOMAIN from both Google and Cloudflare DNS for the registration domain. NOERROR (including NODATA), DNS failures, non-RDAP 404 responses and unsupported registries are not classified as unregistered.")
fmt.Fprintln(&summary, "\n| Domain | Finding | First in history (UTC) | Registered (UTC) | Source locations |\n| --- | --- | --- | --- | --- |")
candidates := 0
for _, result := range results {
if result.Status != "possibly_unregistered" && result.Status != "possibly_reregistered" {
continue
}
candidates++
if candidates > 100 {
continue
}
locations := make([]string, 0, 3)
for _, source := range result.Sources[:min(3, len(result.Sources))] {
locations = append(locations, fmt.Sprintf("%s:%d", markdownText(source.File), source.Line))
}
fmt.Fprintf(&summary, "| %s | %s | %s | %s | %s |\n", markdownText(result.Domain), result.Status, reportDate(result.FirstSeen), reportDate(result.Registered), strings.Join(locations, ", "))
}
fmt.Fprintf(&summary, "\n%d review candidates. Summary shows at most 100; download `report.json` for every result, source rule, RDAP URL, registry status and DNS confirmation.\n", candidates)
fmt.Fprintln(&summary, "\nHistory uses the earliest addition of any rule for the registration domain in this repository, with a 24-hour margin for timestamp differences. Imported lists and registry timestamp changes may need additional verification. DNS is queried only to confirm RDAP-not-found candidates; subdomain availability and live ownership changes without re-registration are outside this audit.")
if err := os.WriteFile(filepath.Join(outputDir, "summary.md"), []byte(summary.String()), 0644); err != nil {
return err
}
fmt.Printf("Audit reports saved to %s: %d review candidates, %d unknown, %d unsupported\n", outputDir, candidates, counts["unknown"], counts["unsupported"])
if counts["registered"]+counts["possibly_unregistered"]+counts["possibly_reregistered"] == 0 {
return fmt.Errorf("no conclusive registry responses; reports saved with coverage gaps")
}
return nil
}
func markdownText(value string) string {
return strings.NewReplacer("|", "\\|", "\n", " ", "\r", " ").Replace(html.EscapeString(value))
}
func reportDate(value time.Time) string {
if value.IsZero() {
return "unknown"
}
return value.UTC().Format(time.DateOnly)
}
+182
View File
@@ -0,0 +1,182 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
func TestWriteReportsPreservesEvidenceAndCoverage(t *testing.T) {
outputDir := filepath.Join(t.TempDir(), "audit")
firstSeen := time.Date(2020, time.July, 18, 13, 5, 41, 0, time.UTC)
registered := time.Date(2026, time.October, 30, 8, 0, 0, 0, time.UTC)
sources := []source{
{File: "data/apple", Line: 8, Rule: "domain:example.com @cn"},
{File: "data/beats", Line: 12, Rule: "full:www.example.com"},
{File: "<data>|sample", Line: 7, Rule: "example.com"},
{File: "data/other", Line: 3, Rule: "example.com &apple"},
}
results := []checkResult{
{
domainEntry: &domainEntry{Domain: "example.com", Sources: sources, FirstSeen: firstSeen},
Status: "possibly_reregistered", RDAPURL: "https://registry.example/domain/example.com",
HTTPStatus: 200, Registered: registered, RegistryHold: []string{"active"},
Detail: "Review registration history",
},
{
domainEntry: &domainEntry{Domain: "example.net", FirstSeen: firstSeen},
Status: "registered",
},
{
domainEntry: &domainEntry{Domain: "example.org"}, Status: "unknown",
HTTPStatus: 403, Detail: "HTTP 403",
},
}
data := &inventory{LiteralRules: 6, Skipped: map[string]int{"keyword": 2}}
if err := writeReports(outputDir, data, results); err != nil {
t.Fatalf("writeReports() got unexpected error: %v", err)
}
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
if err != nil {
t.Fatalf("failed to read report: %v", err)
}
var report struct {
GeneratedAt time.Time `json:"generated_at"`
LiteralRules int `json:"literal_rules"`
RegistrationDomains int `json:"registration_domains"`
SkippedRules map[string]int `json:"skipped_rules"`
Counts map[string]int `json:"counts"`
MissingHistory int `json:"missing_history"`
MissingRegistration int `json:"missing_registration_date"`
Results []struct {
Domain string `json:"domain"`
Sources []source `json:"sources"`
FirstSeen time.Time `json:"first_seen"`
Status string `json:"status"`
RDAPURL string `json:"rdap_url"`
HTTPStatus int `json:"http_status"`
Registered time.Time `json:"registered"`
RegistryHold []string `json:"registry_status"`
Detail string `json:"detail"`
} `json:"results"`
}
if err := json.Unmarshal(encoded, &report); err != nil {
t.Fatalf("failed to decode report: %v", err)
}
if report.GeneratedAt.IsZero() || report.LiteralRules != 6 || report.RegistrationDomains != 3 {
t.Errorf("report metadata = %v, %d rules, %d domains", report.GeneratedAt, report.LiteralRules, report.RegistrationDomains)
}
wantCounts := map[string]int{"possibly_reregistered": 1, "registered": 1, "unknown": 1}
if !reflect.DeepEqual(report.Counts, wantCounts) {
t.Errorf("report counts = %v, want %v", report.Counts, wantCounts)
}
if report.SkippedRules["keyword"] != 2 || report.MissingHistory != 1 || report.MissingRegistration != 1 {
t.Errorf("coverage counts = skipped %v, history %d, registration %d", report.SkippedRules, report.MissingHistory, report.MissingRegistration)
}
if len(report.Results) != len(results) {
t.Fatalf("report has %d results, want %d", len(report.Results), len(results))
}
first := report.Results[0]
if first.Domain != "example.com" || !reflect.DeepEqual(first.Sources, sources) || !first.FirstSeen.Equal(firstSeen) || !first.Registered.Equal(registered) {
t.Errorf("report did not preserve source and date evidence: %+v", first)
}
if first.Status != "possibly_reregistered" || first.RDAPURL != results[0].RDAPURL || first.HTTPStatus != 200 || !reflect.DeepEqual(first.RegistryHold, []string{"active"}) || first.Detail != results[0].Detail {
t.Errorf("report did not preserve registry evidence: %+v", first)
}
summary, err := os.ReadFile(filepath.Join(outputDir, "summary.md"))
if err != nil {
t.Fatalf("failed to read summary: %v", err)
}
for _, want := range []string{"Coverage is incomplete", "| unknown | 1 |", "data/apple:8", "&lt;data&gt;\\|sample:7", "2020-07-18", "2026-10-30"} {
if !strings.Contains(string(summary), want) {
t.Errorf("summary is missing %q", want)
}
}
if strings.Contains(string(summary), "data/other:3") {
t.Error("summary contains more than three source locations for one candidate")
}
}
func TestWriteReportsLimitsSummaryOnly(t *testing.T) {
outputDir := t.TempDir()
results := make([]checkResult, 101)
for index := range results {
results[index] = checkResult{
domainEntry: &domainEntry{Domain: fmt.Sprintf("candidate%03d.com", index)},
Status: "possibly_unregistered", HTTPStatus: 404,
DNS: []string{"NXDOMAIN", "NXDOMAIN"},
}
}
data := &inventory{LiteralRules: len(results), Skipped: make(map[string]int)}
if err := writeReports(outputDir, data, results); err != nil {
t.Fatalf("writeReports() got unexpected error: %v", err)
}
summary, err := os.ReadFile(filepath.Join(outputDir, "summary.md"))
if err != nil {
t.Fatalf("failed to read summary: %v", err)
}
if !strings.Contains(string(summary), "candidate099.com") || strings.Contains(string(summary), "candidate100.com") {
t.Error("summary did not stop after the first 100 candidates")
}
if !strings.Contains(string(summary), "101 review candidates") {
t.Error("summary did not report the complete candidate count")
}
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
if err != nil {
t.Fatalf("failed to read report: %v", err)
}
var report struct {
Results []struct {
Domain string `json:"domain"`
DNS []string `json:"dns"`
} `json:"results"`
}
if err := json.Unmarshal(encoded, &report); err != nil {
t.Fatalf("failed to decode report: %v", err)
}
if len(report.Results) != 101 {
t.Fatalf("report has %d results, want 101", len(report.Results))
}
if report.Results[100].Domain != "candidate100.com" || !reflect.DeepEqual(report.Results[100].DNS, []string{"NXDOMAIN", "NXDOMAIN"}) {
t.Errorf("report lost candidate or DNS evidence: %+v", report.Results[100])
}
}
func TestWriteReportsPreservesInconclusiveResults(t *testing.T) {
outputDir := t.TempDir()
data := &inventory{LiteralRules: 2, Skipped: make(map[string]int)}
results := []checkResult{
{domainEntry: &domainEntry{Domain: "example.com"}, Status: "unknown", Detail: "query failed"},
{domainEntry: &domainEntry{Domain: "example.cn"}, Status: "unsupported"},
}
if err := writeReports(outputDir, data, results); err == nil {
t.Fatal("writeReports() = nil, want error for no conclusive responses")
}
for _, name := range []string{"report.json", "summary.md"} {
content, err := os.ReadFile(filepath.Join(outputDir, name))
if err != nil {
t.Fatalf("inconclusive audit did not preserve %s: %v", name, err)
}
if len(content) == 0 {
t.Errorf("inconclusive audit wrote an empty %s", name)
}
}
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
if err != nil {
t.Fatalf("failed to read report: %v", err)
}
var report struct {
Counts map[string]int `json:"counts"`
}
if err := json.Unmarshal(encoded, &report); err != nil {
t.Fatalf("failed to decode report: %v", err)
}
if report.Counts["unknown"] != 1 || report.Counts["unsupported"] != 1 || report.Counts["possibly_unregistered"] != 0 {
t.Errorf("inconclusive report counts = %v", report.Counts)
}
}
+1
View File
@@ -4,6 +4,7 @@ go 1.26.10
require (
github.com/v2fly/v2ray-core/v5 v5.56.0
golang.org/x/net v0.62.0
google.golang.org/protobuf v1.36.12
)
+2
View File
@@ -10,6 +10,8 @@ github.com/v2fly/v2ray-core/v5 v5.56.0 h1:X1owws+cU9R8p8uT4lMBmp6ThwBOwGB+e8UWb4
github.com/v2fly/v2ray-core/v5 v5.56.0/go.mod h1:43pJ3c9f096GomLtH5BKFd9gXTgwZvGiFuusENcHiWE=
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=