mirror of
https://github.com/v2fly/domain-list-community.git
synced 2026-11-20 06:15:14 +02:00
Add: periodically audit domain registrations
Reviewed main.go, internal/dlc rule types and cmd/datdump: existing commands generate lists but do not audit registrations. Reuse rule constants and add an isolated checker with ICANN public suffix boundaries, registry RDAP, DNS confirmation and rule history. Run a weekly advisory workflow and preserve source evidence for manual review. Compile all commands in PR CI. Go package build passed; test execution is left to the contributor.
This commit is contained in:
1 parent
16fd8005d3
commit
eb7c97de44
12 files changed
+1214
No files matched your search
@@ -0,0 +1,54 @@
|
||||
name: Audit domain registrations
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
name: Audit domain registrations
|
||||
if: github.event_name != 'schedule' || github.repository == 'v2fly/domain-list-community'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 330
|
||||
|
||||
steps:
|
||||
- name: Checkout codebase and domain history
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Go
|
||||
uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Audit domain registrations
|
||||
run: go run ./cmd/domaincheck --outputdir=domain-audit --timeout=5h
|
||||
|
||||
- name: Publish audit summary
|
||||
if: always()
|
||||
run: |
|
||||
if [ -f domain-audit/summary.md ]; then
|
||||
cat domain-audit/summary.md >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo "::warning::The domain audit did not produce a summary. Check the audit logs."
|
||||
echo "The domain audit did not produce a summary. Check the audit logs." >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Upload audit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: domain-audit-${{ github.run_number }}
|
||||
path: |
|
||||
domain-audit/report.json
|
||||
domain-audit/summary.md
|
||||
if-no-files-found: error
|
||||
@@ -27,6 +27,10 @@ jobs:
|
||||
go-version-file: test/go.mod
|
||||
cache-dependency-path: test/go.sum
|
||||
|
||||
- name: Build Go commands
|
||||
working-directory: test
|
||||
run: go build ./...
|
||||
|
||||
- name: Build .dat and .yml
|
||||
run: |
|
||||
[ -d ./base/data/ ] && [ -d ./test/data/ ] || exit 1
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
/domain-list-community
|
||||
/domain-list-community.exe
|
||||
/domain-audit/
|
||||
|
||||
# Generated dat files.
|
||||
/*.dat
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const bootstrapURL = "https://data.iana.org/rdap/dns.json"
|
||||
|
||||
type checkResult struct {
|
||||
*domainEntry
|
||||
Status string `json:"status"`
|
||||
RDAPURL string `json:"rdap_url,omitempty"`
|
||||
HTTPStatus int `json:"http_status,omitempty"`
|
||||
Registered time.Time `json:"registered"`
|
||||
DNS []string `json:"dns,omitempty"`
|
||||
RegistryHold []string `json:"registry_status,omitempty"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
type rdapResponse struct {
|
||||
ObjectClassName string `json:"objectClassName"`
|
||||
LDHName string `json:"ldhName"`
|
||||
Status []string `json:"status"`
|
||||
ErrorCode int `json:"errorCode"`
|
||||
Events []struct {
|
||||
Action string `json:"eventAction"`
|
||||
Date string `json:"eventDate"`
|
||||
} `json:"events"`
|
||||
}
|
||||
|
||||
type hostLimiter struct {
|
||||
mu sync.Mutex
|
||||
next time.Time
|
||||
}
|
||||
|
||||
func (limiter *hostLimiter) wait(ctx context.Context, interval time.Duration) error {
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
limiter.mu.Lock()
|
||||
delay := time.Until(limiter.next)
|
||||
if delay <= 0 {
|
||||
limiter.next = time.Now().Add(interval)
|
||||
limiter.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
limiter.mu.Unlock()
|
||||
if err := pause(ctx, delay); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (limiter *hostLimiter) deferUntil(until time.Time) {
|
||||
limiter.mu.Lock()
|
||||
defer limiter.mu.Unlock()
|
||||
if until.After(limiter.next) {
|
||||
limiter.next = until
|
||||
}
|
||||
}
|
||||
|
||||
type checker struct {
|
||||
client *http.Client
|
||||
services map[string]string
|
||||
interval time.Duration
|
||||
limiters sync.Map
|
||||
resolvers []string
|
||||
}
|
||||
|
||||
func (checker *checker) limiterFor(host string) *hostLimiter {
|
||||
value, _ := checker.limiters.LoadOrStore(host, &hostLimiter{})
|
||||
return value.(*hostLimiter)
|
||||
}
|
||||
|
||||
func newChecker(ctx context.Context, client *http.Client, interval time.Duration) (*checker, error) {
|
||||
result := &checker{
|
||||
client: client, interval: interval, services: make(map[string]string),
|
||||
resolvers: []string{"https://dns.google/resolve", "https://cloudflare-dns.com/dns-query"},
|
||||
}
|
||||
clientCopy := *client
|
||||
clientCopy.CheckRedirect = func(request *http.Request, previous []*http.Request) error {
|
||||
if len(previous) >= 10 || request.URL.Scheme != "https" {
|
||||
return fmt.Errorf("refusing excessive redirects or an insecure redirect")
|
||||
}
|
||||
if client.CheckRedirect != nil {
|
||||
if err := client.CheckRedirect(request, previous); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return result.limiterFor(request.URL.Host).wait(request.Context(), interval)
|
||||
}
|
||||
result.client = &clientCopy
|
||||
var bootstrap struct {
|
||||
Services [][][]string `json:"services"`
|
||||
}
|
||||
status, err := result.getJSON(ctx, bootstrapURL, &bootstrap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status != http.StatusOK {
|
||||
return nil, fmt.Errorf("bootstrap returned HTTP %d", status)
|
||||
}
|
||||
for _, service := range bootstrap.Services {
|
||||
if len(service) != 2 {
|
||||
return nil, fmt.Errorf("invalid RDAP bootstrap service")
|
||||
}
|
||||
for _, endpoint := range service[1] {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil || parsed.Scheme != "https" || parsed.Host == "" {
|
||||
continue
|
||||
}
|
||||
for _, suffix := range service[0] {
|
||||
result.services[strings.ToLower(suffix)] = strings.TrimRight(endpoint, "/")
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(result.services) == 0 {
|
||||
return nil, fmt.Errorf("RDAP bootstrap contains no HTTPS services")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func pause(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func retryDelay(value string, attempt int) time.Duration {
|
||||
if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
if date, err := http.ParseTime(value); err == nil && time.Until(date) > 0 {
|
||||
return time.Until(date)
|
||||
}
|
||||
return time.Duration(1<<attempt) * time.Second
|
||||
}
|
||||
|
||||
func (checker *checker) getJSON(ctx context.Context, endpoint string, target any) (int, error) {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
limiter := checker.limiterFor(parsed.Host)
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if err := limiter.wait(ctx, checker.interval); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
request.Header.Set("Accept", "application/rdap+json, application/json")
|
||||
if parsed.Query().Get("type") == "NS" {
|
||||
request.Header.Set("Accept", "application/dns-json")
|
||||
}
|
||||
request.Header.Set("User-Agent", "domain-list-community-domain-audit/1.0 (+https://github.com/v2fly/domain-list-community)")
|
||||
response, requestErr := checker.client.Do(request)
|
||||
if requestErr != nil {
|
||||
if attempt < 2 && ctx.Err() == nil {
|
||||
limiter.deferUntil(time.Now().Add(retryDelay("", attempt)))
|
||||
continue
|
||||
}
|
||||
return 0, requestErr
|
||||
}
|
||||
status := response.StatusCode
|
||||
if status == http.StatusTooManyRequests || status >= 500 {
|
||||
_ = response.Body.Close()
|
||||
until := time.Now().Add(retryDelay(response.Header.Get("Retry-After"), attempt))
|
||||
limiter.deferUntil(until)
|
||||
checker.limiterFor(response.Request.URL.Host).deferUntil(until)
|
||||
if attempt < 2 {
|
||||
continue
|
||||
}
|
||||
return status, fmt.Errorf("HTTP %d after three attempts", status)
|
||||
}
|
||||
// A bare proxy/web-server 404 is not evidence of a missing RDAP object.
|
||||
if status != http.StatusOK && status != http.StatusNotFound {
|
||||
_ = response.Body.Close()
|
||||
return status, fmt.Errorf("HTTP %d", status)
|
||||
}
|
||||
decodeErr := json.NewDecoder(io.LimitReader(response.Body, 2*1024*1024)).Decode(target)
|
||||
_ = response.Body.Close()
|
||||
return status, decodeErr
|
||||
}
|
||||
return 0, fmt.Errorf("request attempts exhausted")
|
||||
}
|
||||
|
||||
func (checker *checker) rdapURL(domain string) string {
|
||||
for suffix := domain; suffix != ""; {
|
||||
if endpoint := checker.services[suffix]; endpoint != "" {
|
||||
return endpoint + "/domain/" + url.PathEscape(domain)
|
||||
}
|
||||
_, rest, found := strings.Cut(suffix, ".")
|
||||
if !found {
|
||||
break
|
||||
}
|
||||
suffix = rest
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (checker *checker) dnsStatus(ctx context.Context, resolver, domain string) string {
|
||||
endpoint := resolver + "?name=" + url.QueryEscape(domain+".") + "&type=NS"
|
||||
var response struct {
|
||||
Status *int `json:"Status"`
|
||||
Question []struct {
|
||||
Name string `json:"name"`
|
||||
Type int `json:"type"`
|
||||
} `json:"Question"`
|
||||
}
|
||||
status, err := checker.getJSON(ctx, endpoint, &response)
|
||||
if err != nil || status != http.StatusOK || response.Status == nil {
|
||||
return "unknown"
|
||||
}
|
||||
if len(response.Question) != 1 || !strings.EqualFold(strings.TrimSuffix(response.Question[0].Name, "."), domain) || response.Question[0].Type != 2 {
|
||||
return "unknown"
|
||||
}
|
||||
switch *response.Status {
|
||||
case 0:
|
||||
return "NOERROR"
|
||||
case 3:
|
||||
return "NXDOMAIN"
|
||||
default:
|
||||
return fmt.Sprintf("RCODE_%d", *response.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func (checker *checker) check(ctx context.Context, entry *domainEntry) checkResult {
|
||||
result := checkResult{domainEntry: entry, Status: "unknown"}
|
||||
result.RDAPURL = checker.rdapURL(entry.Domain)
|
||||
if result.RDAPURL == "" {
|
||||
result.Status = "unsupported"
|
||||
result.Detail = "No HTTPS RDAP service in IANA bootstrap"
|
||||
return result
|
||||
}
|
||||
var response rdapResponse
|
||||
status, err := checker.getJSON(ctx, result.RDAPURL, &response)
|
||||
result.HTTPStatus = status
|
||||
if err != nil {
|
||||
result.Detail = err.Error()
|
||||
return result
|
||||
}
|
||||
if status == http.StatusNotFound {
|
||||
if response.ErrorCode != http.StatusNotFound {
|
||||
result.Detail = "HTTP 404 without an RDAP errorCode of 404"
|
||||
return result
|
||||
}
|
||||
for _, resolver := range checker.resolvers {
|
||||
result.DNS = append(result.DNS, checker.dnsStatus(ctx, resolver, entry.Domain))
|
||||
}
|
||||
if len(result.DNS) == 2 && result.DNS[0] == "NXDOMAIN" && result.DNS[1] == "NXDOMAIN" {
|
||||
result.Status = "possibly_unregistered"
|
||||
result.Detail = "Registry RDAP 404 and NXDOMAIN from two resolvers; review before removal"
|
||||
} else {
|
||||
result.Detail = "Registry RDAP 404 without two NXDOMAIN confirmations"
|
||||
}
|
||||
return result
|
||||
}
|
||||
if response.ObjectClassName != "domain" || !strings.EqualFold(strings.TrimSuffix(response.LDHName, "."), entry.Domain) {
|
||||
result.Detail = "RDAP response does not identify the queried domain"
|
||||
return result
|
||||
}
|
||||
result.Status = "registered"
|
||||
result.RegistryHold = response.Status
|
||||
for _, event := range response.Events {
|
||||
if event.Action != "registration" {
|
||||
continue
|
||||
}
|
||||
registered, err := time.Parse(time.RFC3339, event.Date)
|
||||
if err != nil {
|
||||
result.Status = "unknown"
|
||||
result.Detail = "Invalid RDAP registration event date"
|
||||
return result
|
||||
}
|
||||
if result.Registered.IsZero() || registered.Before(result.Registered) {
|
||||
result.Registered = registered
|
||||
}
|
||||
}
|
||||
// Registry timestamps and commit clocks may differ; require a full day of separation.
|
||||
if !entry.FirstSeen.IsZero() && result.Registered.After(entry.FirstSeen.Add(24*time.Hour)) {
|
||||
result.Status = "possibly_reregistered"
|
||||
result.Detail = "Registration postdates earliest rule history by more than 24 hours; ownership needs manual verification"
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCheckRegistryAndDNS(t *testing.T) {
|
||||
firstSeen := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
registered := `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2000-01-01T00:00:00Z"}]}`
|
||||
testCases := []struct {
|
||||
name string
|
||||
rdapStatus int
|
||||
rdapBody string
|
||||
dnsBodies []string
|
||||
wantStatus string
|
||||
wantDNS []string
|
||||
wantHold []string
|
||||
}{
|
||||
{name: "registered", rdapStatus: 200, rdapBody: registered, wantStatus: "registered"},
|
||||
{name: "registered on hold", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","status":["server hold"]}`, wantStatus: "registered", wantHold: []string{"server hold"}},
|
||||
{name: "two NXDOMAIN confirmations", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(3), dnsResponse(3)}, wantStatus: "possibly_unregistered", wantDNS: []string{"NXDOMAIN", "NXDOMAIN"}},
|
||||
{name: "NOERROR without answers is not NXDOMAIN", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(3), dnsResponse(0)}, wantStatus: "unknown", wantDNS: []string{"NXDOMAIN", "NOERROR"}},
|
||||
{name: "SERVFAIL is not NXDOMAIN", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{dnsResponse(2), dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"RCODE_2", "NXDOMAIN"}},
|
||||
{name: "missing DNS status", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{`{"Question":[{"name":"example.com.","type":2}]}`, dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"unknown", "NXDOMAIN"}},
|
||||
{name: "different DNS question", rdapStatus: 404, rdapBody: `{"errorCode":404}`, dnsBodies: []string{`{"Status":3,"Question":[{"name":"other.com.","type":2}]}`, dnsResponse(3)}, wantStatus: "unknown", wantDNS: []string{"unknown", "NXDOMAIN"}},
|
||||
{name: "HTML 404", rdapStatus: 404, rdapBody: "<html>Not found</html>", wantStatus: "unknown"},
|
||||
{name: "404 without RDAP error", rdapStatus: 404, rdapBody: `{}`, wantStatus: "unknown"},
|
||||
{name: "RDAP error disagrees with HTTP", rdapStatus: 404, rdapBody: `{"errorCode":400}`, wantStatus: "unknown"},
|
||||
{name: "RDAP domain mismatch", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"other.com"}`, wantStatus: "unknown"},
|
||||
{name: "RDAP object class mismatch", rdapStatus: 200, rdapBody: `{"objectClassName":"entity","ldhName":"example.com"}`, wantStatus: "unknown"},
|
||||
{name: "invalid registration date", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"invalid"}]}`, wantStatus: "unknown"},
|
||||
{name: "registration exactly 24 hours later", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2020-01-02T00:00:00Z"}]}`, wantStatus: "registered"},
|
||||
{name: "registration beyond timestamp margin", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"EXAMPLE.COM.","events":[{"eventAction":"registration","eventDate":"2020-01-02T00:00:01Z"}]}`, wantStatus: "possibly_reregistered"},
|
||||
{name: "earliest registration event", rdapStatus: 200, rdapBody: `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2024-01-01T00:00:00Z"},{"eventAction":"registration","eventDate":"2000-01-01T00:00:00Z"}]}`, wantStatus: "registered"},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if request.URL.Path == "/rdap/domain/example.com" {
|
||||
writer.Header().Set("Content-Type", "application/rdap+json")
|
||||
writer.WriteHeader(tc.rdapStatus)
|
||||
fmt.Fprint(writer, tc.rdapBody)
|
||||
return
|
||||
}
|
||||
index := slices.Index([]string{"/dns-a", "/dns-b"}, request.URL.Path)
|
||||
if index < 0 || index >= len(tc.dnsBodies) {
|
||||
t.Errorf("unexpected DNS request: %s", request.URL)
|
||||
writer.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if request.URL.Query().Get("name") != "example.com." || request.URL.Query().Get("type") != "NS" {
|
||||
t.Errorf("DNS query = %s, want registration domain NS question", request.URL.RawQuery)
|
||||
}
|
||||
if request.Header.Get("Accept") != "application/dns-json" {
|
||||
t.Errorf("DNS Accept header = %q, want application/dns-json", request.Header.Get("Accept"))
|
||||
}
|
||||
writer.Header().Set("Content-Type", "application/dns-json")
|
||||
fmt.Fprint(writer, tc.dnsBodies[index])
|
||||
}))
|
||||
defer server.Close()
|
||||
checker := &checker{
|
||||
client: server.Client(), services: map[string]string{"com": server.URL + "/rdap"},
|
||||
interval: time.Nanosecond, resolvers: []string{server.URL + "/dns-a", server.URL + "/dns-b"},
|
||||
}
|
||||
result := checker.check(context.Background(), &domainEntry{Domain: "example.com", FirstSeen: firstSeen})
|
||||
if result.Status != tc.wantStatus {
|
||||
t.Errorf("check() status = %q, want %q; detail: %s", result.Status, tc.wantStatus, result.Detail)
|
||||
}
|
||||
if !slices.Equal(result.DNS, tc.wantDNS) {
|
||||
t.Errorf("DNS evidence = %v, want %v", result.DNS, tc.wantDNS)
|
||||
}
|
||||
if !slices.Equal(result.RegistryHold, tc.wantHold) {
|
||||
t.Errorf("registry status = %v, want %v", result.RegistryHold, tc.wantHold)
|
||||
}
|
||||
if result.HTTPStatus != tc.rdapStatus || result.RDAPURL != server.URL+"/rdap/domain/example.com" {
|
||||
t.Errorf("registry evidence = HTTP %d %q, want original registry response", result.HTTPStatus, result.RDAPURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func dnsResponse(status int) string {
|
||||
return fmt.Sprintf(`{"Status":%d,"Question":[{"name":"example.com.","type":2}]}`, status)
|
||||
}
|
||||
|
||||
func TestCheckRequiresHistoryForReregistration(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
fmt.Fprint(writer, `{"objectClassName":"domain","ldhName":"example.com","events":[{"eventAction":"registration","eventDate":"2024-01-01T00:00:00Z"}]}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
checker := &checker{client: server.Client(), services: map[string]string{"com": server.URL}, interval: time.Nanosecond}
|
||||
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
|
||||
if result.Status != "registered" || result.Registered.IsZero() {
|
||||
t.Errorf("check without history = %+v, want registered with date and no ownership inference", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckUnsupportedRegistry(t *testing.T) {
|
||||
checker := &checker{services: map[string]string{}}
|
||||
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
|
||||
if result.Status != "unsupported" || result.RDAPURL != "" {
|
||||
t.Errorf("check unsupported registry = %+v, want unsupported without a network request", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDeadlineIsUnknown(t *testing.T) {
|
||||
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
defer cancel()
|
||||
checker := &checker{client: &http.Client{}, services: map[string]string{"com": "https://registry.invalid"}, interval: time.Nanosecond}
|
||||
result := checker.check(ctx, &domainEntry{Domain: "example.com"})
|
||||
if result.Status != "unknown" || !strings.Contains(result.Detail, "deadline exceeded") {
|
||||
t.Errorf("expired audit deadline = %+v, want unknown with deadline detail", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckRetriesTemporaryRegistryFailures(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
failureStatus int
|
||||
failures int32
|
||||
wantRequests int32
|
||||
wantHTTP int
|
||||
wantStatus string
|
||||
}{
|
||||
{name: "rate limit then success", failureStatus: http.StatusTooManyRequests, failures: 1, wantRequests: 2, wantHTTP: http.StatusOK, wantStatus: "registered"},
|
||||
{name: "unavailable then success", failureStatus: http.StatusServiceUnavailable, failures: 1, wantRequests: 2, wantHTTP: http.StatusOK, wantStatus: "registered"},
|
||||
{name: "persistent rate limit", failureStatus: http.StatusTooManyRequests, failures: 3, wantRequests: 3, wantHTTP: http.StatusTooManyRequests, wantStatus: "unknown"},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var requests atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
||||
if requests.Add(1) <= tc.failures {
|
||||
writer.Header().Set("Retry-After", "0")
|
||||
writer.WriteHeader(tc.failureStatus)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(writer, `{"objectClassName":"domain","ldhName":"example.com"}`)
|
||||
}))
|
||||
defer server.Close()
|
||||
checker := &checker{client: server.Client(), services: map[string]string{"com": server.URL}, interval: time.Nanosecond}
|
||||
result := checker.check(context.Background(), &domainEntry{Domain: "example.com"})
|
||||
if requests.Load() != tc.wantRequests {
|
||||
t.Errorf("registry request count = %d, want %d", requests.Load(), tc.wantRequests)
|
||||
}
|
||||
if result.Status != tc.wantStatus || result.HTTPStatus != tc.wantHTTP {
|
||||
t.Errorf("registry result = %q HTTP %d, want %q HTTP %d; detail: %s", result.Status, result.HTTPStatus, tc.wantStatus, tc.wantHTTP, result.Detail)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/v2fly/domain-list-community/internal/dlc"
|
||||
"golang.org/x/net/publicsuffix"
|
||||
)
|
||||
|
||||
type source struct {
|
||||
File string `json:"file"`
|
||||
Line int `json:"line"`
|
||||
Rule string `json:"rule"`
|
||||
}
|
||||
|
||||
type domainEntry struct {
|
||||
Domain string `json:"domain"`
|
||||
Sources []source `json:"sources"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
}
|
||||
|
||||
type inventory struct {
|
||||
Domains map[string]*domainEntry
|
||||
LiteralRules int
|
||||
Skipped map[string]int
|
||||
}
|
||||
|
||||
func literalDomain(line string) (string, string) {
|
||||
line, _, _ = strings.Cut(line, "#")
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
return "", ""
|
||||
}
|
||||
typ, value, specified := strings.Cut(strings.ToLower(line), ":")
|
||||
if !specified {
|
||||
return strings.Fields(typ)[0], ""
|
||||
}
|
||||
switch typ {
|
||||
case dlc.RuleTypeDomain, dlc.RuleTypeFullDomain:
|
||||
fields := strings.Fields(value)
|
||||
if len(fields) == 0 {
|
||||
return "", "empty_domain"
|
||||
}
|
||||
return fields[0], ""
|
||||
default:
|
||||
return "", typ
|
||||
}
|
||||
}
|
||||
|
||||
func registrationDomain(host string) string {
|
||||
if net.ParseIP(host) != nil {
|
||||
return ""
|
||||
}
|
||||
// Ignore private PSL boundaries: a tenant of github.io is not a registry object.
|
||||
for candidate := host; candidate != ""; {
|
||||
suffix, icann := publicsuffix.PublicSuffix(candidate)
|
||||
if icann {
|
||||
if host == suffix {
|
||||
return ""
|
||||
}
|
||||
prefix := strings.TrimSuffix(host, "."+suffix)
|
||||
labels := strings.Split(prefix, ".")
|
||||
return labels[len(labels)-1] + "." + suffix
|
||||
}
|
||||
_, rest, found := strings.Cut(candidate, ".")
|
||||
if !found {
|
||||
return ""
|
||||
}
|
||||
candidate = rest
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func loadInventory(dataPath string) (*inventory, error) {
|
||||
result := &inventory{Domains: make(map[string]*domainEntry), Skipped: make(map[string]int)}
|
||||
err := filepath.WalkDir(dataPath, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if entry.IsDir() {
|
||||
return nil
|
||||
}
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineNumber := 0
|
||||
for scanner.Scan() {
|
||||
lineNumber++
|
||||
host, skip := literalDomain(scanner.Text())
|
||||
if skip != "" {
|
||||
result.Skipped[skip]++
|
||||
continue
|
||||
}
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
result.LiteralRules++
|
||||
domain := registrationDomain(host)
|
||||
if domain == "" {
|
||||
result.Skipped["no_registry_domain"]++
|
||||
continue
|
||||
}
|
||||
if result.Domains[domain] == nil {
|
||||
result.Domains[domain] = &domainEntry{Domain: domain}
|
||||
}
|
||||
result.Domains[domain].Sources = append(result.Domains[domain].Sources, source{
|
||||
File: filepath.ToSlash(path), Line: lineNumber, Rule: strings.TrimSpace(scanner.Text()),
|
||||
})
|
||||
}
|
||||
return scanner.Err()
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(result.Domains) == 0 {
|
||||
return nil, fmt.Errorf("no registrable domains found in %q", dataPath)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func readHistory(reader io.Reader, domains map[string]*domainEntry) error {
|
||||
scanner := bufio.NewScanner(reader)
|
||||
scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
|
||||
var committed time.Time
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "DLC-COMMIT ") {
|
||||
parsed, err := time.Parse(time.RFC3339, strings.TrimPrefix(line, "DLC-COMMIT "))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
committed = parsed
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(line, "+") || strings.HasPrefix(line, "+++") || committed.IsZero() {
|
||||
continue
|
||||
}
|
||||
host, _ := literalDomain(strings.TrimPrefix(line, "+"))
|
||||
if host == "" {
|
||||
continue
|
||||
}
|
||||
entry := domains[registrationDomain(host)]
|
||||
if entry != nil && (entry.FirstSeen.IsZero() || committed.Before(entry.FirstSeen)) {
|
||||
entry.FirstSeen = committed
|
||||
}
|
||||
}
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
func loadHistory(ctx context.Context, dataPath string, domains map[string]*domainEntry) error {
|
||||
shallow, err := exec.CommandContext(ctx, "git", "rev-parse", "--is-shallow-repository").Output()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(string(shallow)) != "false" {
|
||||
return fmt.Errorf("complete git history is required; checkout with fetch-depth: 0")
|
||||
}
|
||||
command := exec.CommandContext(ctx, "git", "log", "--format=DLC-COMMIT %cI", "--no-renames", "--diff-filter=AM", "--unified=0", "-p", "--", dataPath)
|
||||
command.Stderr = os.Stderr
|
||||
pipe, err := command.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := command.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
readErr := readHistory(pipe, domains)
|
||||
if readErr != nil {
|
||||
_ = command.Process.Kill()
|
||||
}
|
||||
if err := command.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
return readErr
|
||||
}
|
||||
|
||||
func sortedDomains(domains map[string]*domainEntry) []string {
|
||||
names := make([]string, 0, len(domains))
|
||||
for name := range domains {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRegistrationDomain(t *testing.T) {
|
||||
testCases := []struct {
|
||||
host string
|
||||
want string
|
||||
}{
|
||||
{host: "mail.example.com", want: "example.com"},
|
||||
{host: "mail.example.co.uk", want: "example.co.uk"},
|
||||
{host: "mail.example.com.cn", want: "example.com.cn"},
|
||||
{host: "tenant.github.io", want: "github.io"},
|
||||
{host: "tenant.blogspot.com", want: "blogspot.com"},
|
||||
{host: "co.uk"},
|
||||
{host: "com"},
|
||||
{host: "router.local"},
|
||||
{host: "hidden.onion"},
|
||||
{host: "127.0.0.1"},
|
||||
{host: "::1"},
|
||||
}
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.host, func(t *testing.T) {
|
||||
if got := registrationDomain(tc.host); got != tc.want {
|
||||
t.Errorf("registrationDomain(%q) = %q, want %q", tc.host, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInventory(t *testing.T) {
|
||||
dataPath := t.TempDir()
|
||||
files := map[string]string{
|
||||
"first": "# Source rules\nExample.COM @cn &second # comment\nfull:Mail.Example.COM @ads\ninclude:second\nkeyword:example\nregexp:^example\\.com$\ngoogle\nrouter.local\n",
|
||||
"second": "domain: example.com @ads\nfull:api.example.co.uk\n",
|
||||
}
|
||||
for name, content := range files {
|
||||
if err := os.WriteFile(filepath.Join(dataPath, name), []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write list %q: %v", name, err)
|
||||
}
|
||||
}
|
||||
data, err := loadInventory(dataPath)
|
||||
if err != nil {
|
||||
t.Fatalf("loadInventory() got unexpected error: %v", err)
|
||||
}
|
||||
if got := sortedDomains(data.Domains); !slices.Equal(got, []string{"example.co.uk", "example.com"}) {
|
||||
t.Fatalf("registration domains = %v, want example.co.uk and example.com", got)
|
||||
}
|
||||
if data.LiteralRules != 6 {
|
||||
t.Errorf("literal rule count = %d, want 6", data.LiteralRules)
|
||||
}
|
||||
for kind, count := range map[string]int{"include": 1, "keyword": 1, "regexp": 1, "no_registry_domain": 2} {
|
||||
if data.Skipped[kind] != count {
|
||||
t.Errorf("skipped %s = %d, want %d", kind, data.Skipped[kind], count)
|
||||
}
|
||||
}
|
||||
sources := data.Domains["example.com"].Sources
|
||||
if len(sources) != 3 {
|
||||
t.Fatalf("example.com source count = %d, want 3", len(sources))
|
||||
}
|
||||
if sources[0].File != filepath.ToSlash(filepath.Join(dataPath, "first")) || sources[0].Line != 2 || sources[0].Rule != "Example.COM @cn &second # comment" {
|
||||
t.Errorf("first source = %+v, want original rule at first:2", sources[0])
|
||||
}
|
||||
if sources[1].Line != 3 || sources[2].Line != 1 {
|
||||
t.Errorf("source line numbers = %d, %d, want 3, 1", sources[1].Line, sources[2].Line)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadInventoryRejectsEmptyInput(t *testing.T) {
|
||||
if _, err := loadInventory(t.TempDir()); err == nil {
|
||||
t.Fatal("loadInventory() accepted an empty data directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHistoryUsesEarliestRegistrationDomainAddition(t *testing.T) {
|
||||
domains := map[string]*domainEntry{
|
||||
"example.com": {Domain: "example.com"},
|
||||
"github.io": {Domain: "github.io"},
|
||||
"missing.org": {Domain: "missing.org"},
|
||||
}
|
||||
history := strings.Join([]string{
|
||||
"DLC-COMMIT 2024-02-01T00:00:00Z",
|
||||
"+++ b/data/first",
|
||||
"+domain:example.com @cn",
|
||||
"+tenant.github.io",
|
||||
"DLC-COMMIT 2020-01-01T00:00:00Z",
|
||||
"+full:mail.example.com @ads",
|
||||
"DLC-COMMIT 2019-01-01T00:00:00Z",
|
||||
"+include:missing.org",
|
||||
"+regexp:^missing\\.org$",
|
||||
"-missing.org",
|
||||
"+unrelated.net",
|
||||
}, "\n")
|
||||
if err := readHistory(strings.NewReader(history), domains); err != nil {
|
||||
t.Fatalf("readHistory() got unexpected error: %v", err)
|
||||
}
|
||||
wantFirstSeen := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
|
||||
if !domains["example.com"].FirstSeen.Equal(wantFirstSeen) {
|
||||
t.Errorf("example.com first seen = %v, want %v", domains["example.com"].FirstSeen, wantFirstSeen)
|
||||
}
|
||||
if domains["github.io"].FirstSeen.Year() != 2024 {
|
||||
t.Errorf("github.io first seen = %v, want tenant rule addition", domains["github.io"].FirstSeen)
|
||||
}
|
||||
if !domains["missing.org"].FirstSeen.IsZero() {
|
||||
t.Errorf("nonliteral or removed rule was used as history: %v", domains["missing.org"].FirstSeen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadHistoryRejectsInvalidCommitDate(t *testing.T) {
|
||||
if err := readHistory(strings.NewReader("DLC-COMMIT invalid\n+example.com\n"), map[string]*domainEntry{}); err == nil {
|
||||
t.Fatal("readHistory() accepted an invalid commit date")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
dataPath = flag.String("datapath", "data", "Directory containing domain lists")
|
||||
outputDir = flag.String("outputdir", "domain-audit", "Directory for audit reports")
|
||||
workers = flag.Int("workers", 12, "Maximum concurrent domain checks")
|
||||
interval = flag.Duration("interval", 500*time.Millisecond, "Minimum interval between requests to each HTTP host")
|
||||
timeout = flag.Duration("timeout", 5*time.Hour, "Maximum audit duration, including retries")
|
||||
)
|
||||
|
||||
func run(ctx context.Context) error {
|
||||
if *workers < 1 || *interval <= 0 || *timeout <= 0 {
|
||||
return fmt.Errorf("workers, interval and timeout must be positive")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, *timeout)
|
||||
defer cancel()
|
||||
data, err := loadInventory(*dataPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load domain lists: %w", err)
|
||||
}
|
||||
fmt.Printf("Auditing %d registration domains from %d literal rules\n", len(data.Domains), data.LiteralRules)
|
||||
if err := loadHistory(ctx, *dataPath, data.Domains); err != nil {
|
||||
return fmt.Errorf("load domain history: %w", err)
|
||||
}
|
||||
client := &http.Client{Timeout: 20 * time.Second}
|
||||
checker, err := newChecker(ctx, client, *interval)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load RDAP bootstrap: %w", err)
|
||||
}
|
||||
names := sortedDomains(data.Domains)
|
||||
results := make([]checkResult, len(names))
|
||||
jobs := make(chan int)
|
||||
var group sync.WaitGroup
|
||||
var completed atomic.Int64
|
||||
for range *workers {
|
||||
group.Go(func() {
|
||||
for index := range jobs {
|
||||
results[index] = checker.check(ctx, data.Domains[names[index]])
|
||||
if count := completed.Add(1); count%1000 == 0 || count == int64(len(names)) {
|
||||
fmt.Printf("Checked %d/%d registration domains\n", count, len(names))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
for index := range names {
|
||||
jobs <- index
|
||||
}
|
||||
close(jobs)
|
||||
group.Wait()
|
||||
if err := writeReports(*outputDir, data, results); err != nil {
|
||||
return err
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return fmt.Errorf("audit incomplete; partial results saved: %w", ctx.Err())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
if err := run(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[Error] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeReports(outputDir string, data *inventory, results []checkResult) error {
|
||||
counts := make(map[string]int)
|
||||
missingHistory := 0
|
||||
missingRegistration := 0
|
||||
for _, result := range results {
|
||||
counts[result.Status]++
|
||||
if result.FirstSeen.IsZero() {
|
||||
missingHistory++
|
||||
}
|
||||
if result.Status == "registered" && result.Registered.IsZero() {
|
||||
missingRegistration++
|
||||
}
|
||||
}
|
||||
report := struct {
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
LiteralRules int `json:"literal_rules"`
|
||||
RegistrationDomains int `json:"registration_domains"`
|
||||
SkippedRules map[string]int `json:"skipped_rules"`
|
||||
Counts map[string]int `json:"counts"`
|
||||
MissingHistory int `json:"missing_history"`
|
||||
MissingRegistration int `json:"missing_registration_date"`
|
||||
Results []checkResult `json:"results"`
|
||||
}{
|
||||
GeneratedAt: time.Now().UTC(), Commit: os.Getenv("GITHUB_SHA"),
|
||||
LiteralRules: data.LiteralRules, RegistrationDomains: len(results), SkippedRules: data.Skipped,
|
||||
Counts: counts, MissingHistory: missingHistory, MissingRegistration: missingRegistration, Results: results,
|
||||
}
|
||||
encoded, err := json.MarshalIndent(report, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode audit report: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(outputDir, "report.json"), append(encoded, '\n'), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
var summary strings.Builder
|
||||
fmt.Fprintln(&summary, "# Domain registration audit")
|
||||
fmt.Fprintf(&summary, "\nScanned %d literal rules, grouped into %d registration domains.\n", data.LiteralRules, len(results))
|
||||
fmt.Fprintln(&summary, "\nThis is an advisory report for manual review. No rules are removed automatically. Registered does not mean ownership is correct. A later registration date suggests a lapse but does not prove a change of owner.")
|
||||
fmt.Fprintln(&summary, "\n| Result | Domains |\n| --- | ---: |")
|
||||
for _, status := range []string{"registered", "possibly_unregistered", "possibly_reregistered", "unknown", "unsupported"} {
|
||||
fmt.Fprintf(&summary, "| %s | %d |\n", status, counts[status])
|
||||
}
|
||||
fmt.Fprintf(&summary, "\nUnknown and unsupported results are coverage gaps, not evidence of expiry. Registration-date comparison was unavailable for %d registered domains without a registration date, and %d domains without rule history.\n", missingRegistration, missingHistory)
|
||||
if counts["unknown"] > 0 {
|
||||
fmt.Fprintln(&summary, "\n**Coverage is incomplete.** Inspect `unknown` results in the JSON report before drawing conclusions.")
|
||||
}
|
||||
fmt.Fprintln(&summary, "\nSkipped rules:")
|
||||
skipTypes := make([]string, 0, len(data.Skipped))
|
||||
for typ := range data.Skipped {
|
||||
skipTypes = append(skipTypes, typ)
|
||||
}
|
||||
slices.Sort(skipTypes)
|
||||
for _, typ := range skipTypes {
|
||||
fmt.Fprintf(&summary, "- %s: %d\n", markdownText(typ), data.Skipped[typ])
|
||||
}
|
||||
fmt.Fprintln(&summary, "\nIncludes are covered through their source files. Keywords and regular expressions cannot be queried as DNS names. TLD-only, special-use and private-network names without an ICANN registration boundary are excluded.")
|
||||
fmt.Fprintln(&summary, "\n## Review candidates\n\nA missing registration requires registry RDAP `errorCode: 404` plus NXDOMAIN from both Google and Cloudflare DNS for the registration domain. NOERROR (including NODATA), DNS failures, non-RDAP 404 responses and unsupported registries are not classified as unregistered.")
|
||||
fmt.Fprintln(&summary, "\n| Domain | Finding | First in history (UTC) | Registered (UTC) | Source locations |\n| --- | --- | --- | --- | --- |")
|
||||
candidates := 0
|
||||
for _, result := range results {
|
||||
if result.Status != "possibly_unregistered" && result.Status != "possibly_reregistered" {
|
||||
continue
|
||||
}
|
||||
candidates++
|
||||
if candidates > 100 {
|
||||
continue
|
||||
}
|
||||
locations := make([]string, 0, 3)
|
||||
for _, source := range result.Sources[:min(3, len(result.Sources))] {
|
||||
locations = append(locations, fmt.Sprintf("%s:%d", markdownText(source.File), source.Line))
|
||||
}
|
||||
fmt.Fprintf(&summary, "| %s | %s | %s | %s | %s |\n", markdownText(result.Domain), result.Status, reportDate(result.FirstSeen), reportDate(result.Registered), strings.Join(locations, ", "))
|
||||
}
|
||||
fmt.Fprintf(&summary, "\n%d review candidates. Summary shows at most 100; download `report.json` for every result, source rule, RDAP URL, registry status and DNS confirmation.\n", candidates)
|
||||
fmt.Fprintln(&summary, "\nHistory uses the earliest addition of any rule for the registration domain in this repository, with a 24-hour margin for timestamp differences. Imported lists and registry timestamp changes may need additional verification. DNS is queried only to confirm RDAP-not-found candidates; subdomain availability and live ownership changes without re-registration are outside this audit.")
|
||||
if err := os.WriteFile(filepath.Join(outputDir, "summary.md"), []byte(summary.String()), 0644); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("Audit reports saved to %s: %d review candidates, %d unknown, %d unsupported\n", outputDir, candidates, counts["unknown"], counts["unsupported"])
|
||||
if counts["registered"]+counts["possibly_unregistered"]+counts["possibly_reregistered"] == 0 {
|
||||
return fmt.Errorf("no conclusive registry responses; reports saved with coverage gaps")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markdownText(value string) string {
|
||||
return strings.NewReplacer("|", "\\|", "\n", " ", "\r", " ").Replace(html.EscapeString(value))
|
||||
}
|
||||
|
||||
func reportDate(value time.Time) string {
|
||||
if value.IsZero() {
|
||||
return "unknown"
|
||||
}
|
||||
return value.UTC().Format(time.DateOnly)
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWriteReportsPreservesEvidenceAndCoverage(t *testing.T) {
|
||||
outputDir := filepath.Join(t.TempDir(), "audit")
|
||||
firstSeen := time.Date(2020, time.July, 18, 13, 5, 41, 0, time.UTC)
|
||||
registered := time.Date(2026, time.October, 30, 8, 0, 0, 0, time.UTC)
|
||||
sources := []source{
|
||||
{File: "data/apple", Line: 8, Rule: "domain:example.com @cn"},
|
||||
{File: "data/beats", Line: 12, Rule: "full:www.example.com"},
|
||||
{File: "<data>|sample", Line: 7, Rule: "example.com"},
|
||||
{File: "data/other", Line: 3, Rule: "example.com &apple"},
|
||||
}
|
||||
results := []checkResult{
|
||||
{
|
||||
domainEntry: &domainEntry{Domain: "example.com", Sources: sources, FirstSeen: firstSeen},
|
||||
Status: "possibly_reregistered", RDAPURL: "https://registry.example/domain/example.com",
|
||||
HTTPStatus: 200, Registered: registered, RegistryHold: []string{"active"},
|
||||
Detail: "Review registration history",
|
||||
},
|
||||
{
|
||||
domainEntry: &domainEntry{Domain: "example.net", FirstSeen: firstSeen},
|
||||
Status: "registered",
|
||||
},
|
||||
{
|
||||
domainEntry: &domainEntry{Domain: "example.org"}, Status: "unknown",
|
||||
HTTPStatus: 403, Detail: "HTTP 403",
|
||||
},
|
||||
}
|
||||
data := &inventory{LiteralRules: 6, Skipped: map[string]int{"keyword": 2}}
|
||||
if err := writeReports(outputDir, data, results); err != nil {
|
||||
t.Fatalf("writeReports() got unexpected error: %v", err)
|
||||
}
|
||||
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read report: %v", err)
|
||||
}
|
||||
var report struct {
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
LiteralRules int `json:"literal_rules"`
|
||||
RegistrationDomains int `json:"registration_domains"`
|
||||
SkippedRules map[string]int `json:"skipped_rules"`
|
||||
Counts map[string]int `json:"counts"`
|
||||
MissingHistory int `json:"missing_history"`
|
||||
MissingRegistration int `json:"missing_registration_date"`
|
||||
Results []struct {
|
||||
Domain string `json:"domain"`
|
||||
Sources []source `json:"sources"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
Status string `json:"status"`
|
||||
RDAPURL string `json:"rdap_url"`
|
||||
HTTPStatus int `json:"http_status"`
|
||||
Registered time.Time `json:"registered"`
|
||||
RegistryHold []string `json:"registry_status"`
|
||||
Detail string `json:"detail"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &report); err != nil {
|
||||
t.Fatalf("failed to decode report: %v", err)
|
||||
}
|
||||
if report.GeneratedAt.IsZero() || report.LiteralRules != 6 || report.RegistrationDomains != 3 {
|
||||
t.Errorf("report metadata = %v, %d rules, %d domains", report.GeneratedAt, report.LiteralRules, report.RegistrationDomains)
|
||||
}
|
||||
wantCounts := map[string]int{"possibly_reregistered": 1, "registered": 1, "unknown": 1}
|
||||
if !reflect.DeepEqual(report.Counts, wantCounts) {
|
||||
t.Errorf("report counts = %v, want %v", report.Counts, wantCounts)
|
||||
}
|
||||
if report.SkippedRules["keyword"] != 2 || report.MissingHistory != 1 || report.MissingRegistration != 1 {
|
||||
t.Errorf("coverage counts = skipped %v, history %d, registration %d", report.SkippedRules, report.MissingHistory, report.MissingRegistration)
|
||||
}
|
||||
if len(report.Results) != len(results) {
|
||||
t.Fatalf("report has %d results, want %d", len(report.Results), len(results))
|
||||
}
|
||||
first := report.Results[0]
|
||||
if first.Domain != "example.com" || !reflect.DeepEqual(first.Sources, sources) || !first.FirstSeen.Equal(firstSeen) || !first.Registered.Equal(registered) {
|
||||
t.Errorf("report did not preserve source and date evidence: %+v", first)
|
||||
}
|
||||
if first.Status != "possibly_reregistered" || first.RDAPURL != results[0].RDAPURL || first.HTTPStatus != 200 || !reflect.DeepEqual(first.RegistryHold, []string{"active"}) || first.Detail != results[0].Detail {
|
||||
t.Errorf("report did not preserve registry evidence: %+v", first)
|
||||
}
|
||||
summary, err := os.ReadFile(filepath.Join(outputDir, "summary.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read summary: %v", err)
|
||||
}
|
||||
for _, want := range []string{"Coverage is incomplete", "| unknown | 1 |", "data/apple:8", "<data>\\|sample:7", "2020-07-18", "2026-10-30"} {
|
||||
if !strings.Contains(string(summary), want) {
|
||||
t.Errorf("summary is missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(string(summary), "data/other:3") {
|
||||
t.Error("summary contains more than three source locations for one candidate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportsLimitsSummaryOnly(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
results := make([]checkResult, 101)
|
||||
for index := range results {
|
||||
results[index] = checkResult{
|
||||
domainEntry: &domainEntry{Domain: fmt.Sprintf("candidate%03d.com", index)},
|
||||
Status: "possibly_unregistered", HTTPStatus: 404,
|
||||
DNS: []string{"NXDOMAIN", "NXDOMAIN"},
|
||||
}
|
||||
}
|
||||
data := &inventory{LiteralRules: len(results), Skipped: make(map[string]int)}
|
||||
if err := writeReports(outputDir, data, results); err != nil {
|
||||
t.Fatalf("writeReports() got unexpected error: %v", err)
|
||||
}
|
||||
summary, err := os.ReadFile(filepath.Join(outputDir, "summary.md"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read summary: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(summary), "candidate099.com") || strings.Contains(string(summary), "candidate100.com") {
|
||||
t.Error("summary did not stop after the first 100 candidates")
|
||||
}
|
||||
if !strings.Contains(string(summary), "101 review candidates") {
|
||||
t.Error("summary did not report the complete candidate count")
|
||||
}
|
||||
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read report: %v", err)
|
||||
}
|
||||
var report struct {
|
||||
Results []struct {
|
||||
Domain string `json:"domain"`
|
||||
DNS []string `json:"dns"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &report); err != nil {
|
||||
t.Fatalf("failed to decode report: %v", err)
|
||||
}
|
||||
if len(report.Results) != 101 {
|
||||
t.Fatalf("report has %d results, want 101", len(report.Results))
|
||||
}
|
||||
if report.Results[100].Domain != "candidate100.com" || !reflect.DeepEqual(report.Results[100].DNS, []string{"NXDOMAIN", "NXDOMAIN"}) {
|
||||
t.Errorf("report lost candidate or DNS evidence: %+v", report.Results[100])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportsPreservesInconclusiveResults(t *testing.T) {
|
||||
outputDir := t.TempDir()
|
||||
data := &inventory{LiteralRules: 2, Skipped: make(map[string]int)}
|
||||
results := []checkResult{
|
||||
{domainEntry: &domainEntry{Domain: "example.com"}, Status: "unknown", Detail: "query failed"},
|
||||
{domainEntry: &domainEntry{Domain: "example.cn"}, Status: "unsupported"},
|
||||
}
|
||||
if err := writeReports(outputDir, data, results); err == nil {
|
||||
t.Fatal("writeReports() = nil, want error for no conclusive responses")
|
||||
}
|
||||
for _, name := range []string{"report.json", "summary.md"} {
|
||||
content, err := os.ReadFile(filepath.Join(outputDir, name))
|
||||
if err != nil {
|
||||
t.Fatalf("inconclusive audit did not preserve %s: %v", name, err)
|
||||
}
|
||||
if len(content) == 0 {
|
||||
t.Errorf("inconclusive audit wrote an empty %s", name)
|
||||
}
|
||||
}
|
||||
encoded, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read report: %v", err)
|
||||
}
|
||||
var report struct {
|
||||
Counts map[string]int `json:"counts"`
|
||||
}
|
||||
if err := json.Unmarshal(encoded, &report); err != nil {
|
||||
t.Fatalf("failed to decode report: %v", err)
|
||||
}
|
||||
if report.Counts["unknown"] != 1 || report.Counts["unsupported"] != 1 || report.Counts["possibly_unregistered"] != 0 {
|
||||
t.Errorf("inconclusive report counts = %v", report.Counts)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ go 1.26.10
|
||||
|
||||
require (
|
||||
github.com/v2fly/v2ray-core/v5 v5.56.0
|
||||
golang.org/x/net v0.62.0
|
||||
google.golang.org/protobuf v1.36.12
|
||||
)
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ github.com/v2fly/v2ray-core/v5 v5.56.0 h1:X1owws+cU9R8p8uT4lMBmp6ThwBOwGB+e8UWb4
|
||||
github.com/v2fly/v2ray-core/v5 v5.56.0/go.mod h1:43pJ3c9f096GomLtH5BKFd9gXTgwZvGiFuusENcHiWE=
|
||||
go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM=
|
||||
go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
|
||||
golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
|
||||
|
||||
Reference in new issue
Block a user