mindboost/education-flagger
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enrich /lookup with university domain list check

Browse Source 
Add a second detection path alongside ASN lookup: a self-maintained
list of university domains (uni_domains.txt) loaded at startup.

- New /lookup params: email= (extracts domain from address), domain= unchanged
- Suffix matching: insti.uni-stuttgart.de matches list entry uni-stuttgart.de
  without false-positives (evil-uni-stuttgart.de does not match)
- New response fields: asn_match, domain_match, matched_domain (omitempty)
- nren remains true if either asn_match OR domain_match is true (backwards compat)
- /healthz now returns JSON body: {"asn_count":N,"domain_count":N}
- asn-updater: new update_uni_domains() merges hs-kompass.de TSV + Hipo JSON
  (configurable via UNI_DOMAIN_COUNTRIES / HS_KOMPASS_URL env vars)
- 7 new tests; all existing tests pass unchanged

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Robert Rapp
2026-03-17 15:10:49 +01:00
parent 15476898c3
commit 082ecc579a
7 changed files with 435 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
main.go
View File

@@ -3,6 +3,7 @@ package main
import (
"bufio"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
@@ -21,18 +22,22 @@ type asnRecord struct {
}
type lookupResponse struct {
Domain string `json:"domain"`
NREN bool `json:"nren"`
ASN *uint `json:"asn,omitempty"`
ASNOrg string `json:"asn_org,omitempty"`
IPs []string `json:"ips"`
MatchedIP string `json:"matched_ip,omitempty"`
Error string `json:"error,omitempty"`
Domain string `json:"domain"`
NREN bool `json:"nren"`
ASNMatch bool `json:"asn_match"`
DomainMatch bool `json:"domain_match"`
MatchedDomain string `json:"matched_domain,omitempty"`
ASN *uint `json:"asn,omitempty"`
ASNOrg string `json:"asn_org,omitempty"`
IPs []string `json:"ips"`
MatchedIP string `json:"matched_ip,omitempty"`
Error string `json:"error,omitempty"`
}
type server struct {
db *maxminddb.Reader
nrenASNs map[uint]struct{}
uniDomains map[string]struct{}
ready atomic.Bool
versionTag string
minASN int
@@ -62,6 +67,65 @@ func loadASNSet(path string) (map[uint]struct{}, error) {
return set, sc.Err()
}
func loadDomainSet(path string) (map[string]struct{}, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
set := make(map[string]struct{}, 2048)
sc := bufio.NewScanner(f)
for sc.Scan() {
line := strings.ToLower(strings.TrimSpace(sc.Text()))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if !strings.Contains(line, ".") {
log.Printf("[warn] skipping invalid domain entry: %s", line)
continue
}
set[line] = struct{}{}
}
return set, sc.Err()
}
// matchesUniDomain checks if domain or any parent domain (label-aligned suffix)
// appears in set. Returns (true, matchedEntry) on match, (false, "") otherwise.
// Prevents false positives: "evil-uni-stuttgart.de" does not match "uni-stuttgart.de".
func matchesUniDomain(domain string, set map[string]struct{}) (bool, string) {
d := strings.TrimRight(strings.ToLower(domain), ".")
if d == "" {
return false, ""
}
if _, ok := set[d]; ok {
return true, d
}
for {
dot := strings.Index(d, ".")
if dot < 0 {
break
}
d = d[dot+1:]
if !strings.Contains(d, ".") {
// fewer than 2 labels remain — stop
break
}
if _, ok := set[d]; ok {
return true, d
}
}
return false, ""
}
func extractDomain(s string) string {
if strings.Contains(s, "@") {
parts := strings.SplitN(s, "@", 2)
return strings.ToLower(strings.TrimSpace(parts[1]))
}
return strings.ToLower(strings.TrimSpace(s))
}
func firstForwardedFor(r *http.Request) string {
xff := r.Header.Get("X-Forwarded-For")
if xff == "" {
@@ -93,6 +157,16 @@ func writeJSON(w http.ResponseWriter, status int, payload any) {
_ = json.NewEncoder(w).Encode(payload)
}
func (s *server) healthzHandler(w http.ResponseWriter, _ *http.Request) {
status := http.StatusOK
if s.asnCount < s.minASN {
status = http.StatusServiceUnavailable
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
fmt.Fprintf(w, `{"asn_count":%d,"domain_count":%d}`, s.asnCount, len(s.uniDomains))
}
func (s *server) authHandler(w http.ResponseWriter, r *http.Request) {
if !s.ready.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
@@ -142,8 +216,20 @@ func (s *server) lookupHandler(w http.ResponseWriter, r *http.Request) {
return
}
domain := strings.TrimSpace(r.URL.Query().Get("domain"))
if domain == "" {
emailParam := r.URL.Query().Get("email")
domainParam := r.URL.Query().Get("domain")
var rawInput string
resp := lookupResponse{}
if emailParam != "" {
rawInput = emailParam
if !strings.Contains(emailParam, "@") {
resp.Error = "email param has no @ — interpreted as bare domain"
}
} else if domainParam != "" {
rawInput = strings.TrimSpace(domainParam)
} else {
writeJSON(w, http.StatusBadRequest, lookupResponse{
NREN: false,
Error: "missing domain",
@@ -151,55 +237,53 @@ func (s *server) lookupHandler(w http.ResponseWriter, r *http.Request) {
return
}
ips, err := net.LookupIP(domain)
if err != nil || len(ips) == 0 {
writeJSON(w, http.StatusOK, lookupResponse{
Domain: domain,
NREN: false,
Error: "domain lookup failed",
})
return
}
domain := extractDomain(rawInput)
resp.Domain = domain
resp := lookupResponse{
Domain: domain,
NREN: false,
IPs: make([]string, 0, len(ips)),
}
ips, dnsErr := net.LookupIP(domain)
if dnsErr == nil && len(ips) > 0 {
resp.IPs = make([]string, 0, len(ips))
var firstASN *uint
var firstOrg string
var firstASN *uint
var firstOrg string
for _, ip := range ips {
ipStr := ip.String()
resp.IPs = append(resp.IPs, ipStr)
for _, ip := range ips {
ipStr := ip.String()
resp.IPs = append(resp.IPs, ipStr)
if s.db == nil {
continue
}
var rec asnRecord
if err := s.db.Lookup(ip, &rec); err != nil || rec.ASN == 0 {
continue
}
var rec asnRecord
if err := s.db.Lookup(ip, &rec); err != nil || rec.ASN == 0 {
continue
if firstASN == nil {
firstASN = new(uint)
*firstASN = rec.ASN
firstOrg = rec.Org
}
if _, ok := s.nrenASNs[rec.ASN]; ok {
asn := rec.ASN
resp.ASNMatch = true
resp.ASN = &asn
resp.ASNOrg = rec.Org
resp.MatchedIP = ipStr
break
}
}
if firstASN == nil {
firstASN = new(uint)
*firstASN = rec.ASN
firstOrg = rec.Org
}
if _, ok := s.nrenASNs[rec.ASN]; ok {
asn := rec.ASN
resp.NREN = true
resp.ASN = &asn
resp.ASNOrg = rec.Org
resp.MatchedIP = ipStr
writeJSON(w, http.StatusOK, resp)
return
if !resp.ASNMatch && firstASN != nil {
resp.ASN = firstASN
resp.ASNOrg = firstOrg
}
} else if resp.Error == "" {
resp.Error = "domain lookup failed"
}
if firstASN != nil {
resp.ASN = firstASN
resp.ASNOrg = firstOrg
}
resp.DomainMatch, resp.MatchedDomain = matchesUniDomain(domain, s.uniDomains)
resp.NREN = resp.ASNMatch || resp.DomainMatch
writeJSON(w, http.StatusOK, resp)
}
@@ -223,9 +307,21 @@ func main() {
}
asnCount := len(set)
uniDomainsPath := getenv("UNI_DOMAINS_PATH", "/data/uni_domains.txt")
uniDomains, err := loadDomainSet(uniDomainsPath)
if err != nil {
if os.IsNotExist(err) {
log.Printf("[warn] uni_domains.txt not found — domain_match will always be false")
} else {
log.Printf("[warn] failed to load uni_domains.txt: %v", err)
}
uniDomains = make(map[string]struct{})
}
s := &server{
db: db,
nrenASNs: set,
uniDomains: uniDomains,
versionTag: version,
minASN: minASN,
asnCount: asnCount,
@@ -235,13 +331,7 @@ func main() {
mux := http.NewServeMux()
mux.HandleFunc("/auth", s.authHandler)
mux.HandleFunc("/lookup", s.lookupHandler)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
if s.asnCount < s.minASN {
w.WriteHeader(http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
})
mux.HandleFunc("/healthz", s.healthzHandler)
srv := &http.Server{
Addr: addr,
@@ -249,7 +339,7 @@ func main() {
ReadHeaderTimeout: 2 * time.Second,
}
log.Printf("listening on %s (asn_count=%d, min_asn=%d)", addr, asnCount, minASN)
log.Printf("listening on %s (asn_count=%d, min_asn=%d, domain_count=%d)", addr, asnCount, minASN, len(uniDomains))
log.Fatal(srv.ListenAndServe())
}