package main import ( "io/ioutil" "log" "net" "net/url" "strings" "time" ) type Checker struct { data ConcurrentMap blockedMap ConcurrentMap directMap ConcurrentMap interval int64 timeout int } type CheckerItem struct { IsHTTPS bool Method string URL string Domain string Host string Data []byte SuccessCount uint FailCount uint } //NewChecker args: //timeout : tcp timeout milliseconds ,connect to host //interval: recheck domain interval seconds func NewChecker(timeout int, interval int64, blockedFile, directFile string) Checker { ch := Checker{ data: NewConcurrentMap(), interval: interval, timeout: timeout, } ch.blockedMap = ch.loadMap(blockedFile) ch.directMap = ch.loadMap(directFile) if !ch.blockedMap.IsEmpty() { log.Printf("blocked file loaded , domains : %d", ch.blockedMap.Count()) } if !ch.directMap.IsEmpty() { log.Printf("direct file loaded , domains : %d", ch.directMap.Count()) } ch.start() return ch } func (c *Checker) loadMap(f string) (dataMap ConcurrentMap) { dataMap = NewConcurrentMap() if PathExists(f) { _contents, err := ioutil.ReadFile(f) if err != nil { log.Printf("load file err:%s", err) return } for _, line := range strings.Split(string(_contents), "\n") { line = strings.Trim(line, "\r \t") if line != "" { dataMap.Set(line, true) } } } return } func (c *Checker) start() { go func() { for { for _, v := range c.data.Items() { go func(item CheckerItem) { if c.isNeedCheck(item) { //log.Printf("check %s", item.Domain) var conn net.Conn var err error if item.IsHTTPS { conn, err = ConnectHost(item.Host, c.timeout) if err == nil { conn.SetDeadline(time.Now().Add(time.Millisecond)) conn.Close() } } else { err = HTTPGet(item.URL, c.timeout) } if err != nil { item.FailCount = item.FailCount + 1 } else { item.SuccessCount = item.SuccessCount + 1 } c.data.Set(item.Host, item) } }(v.(CheckerItem)) } time.Sleep(time.Second * time.Duration(c.interval)) } }() } func (c *Checker) isNeedCheck(item CheckerItem) bool { var minCount uint = 5 if (item.SuccessCount >= minCount && item.SuccessCount > item.FailCount) || (item.FailCount >= minCount && item.SuccessCount > item.FailCount) || c.domainIsInMap(item.Host, false) || c.domainIsInMap(item.Host, true) { return false } return true } func (c *Checker) IsBlocked(address string) (blocked bool, failN, successN uint) { if c.domainIsInMap(address, true) { return true, 0, 0 } if c.domainIsInMap(address, false) { return false, 0, 0 } _item, ok := c.data.Get(address) if !ok { return true, 0, 0 } item := _item.(CheckerItem) return item.FailCount >= item.SuccessCount, item.FailCount, item.SuccessCount } func (c *Checker) domainIsInMap(address string, blockedMap bool) bool { u, err := url.Parse("http://" + address) if err != nil { log.Printf("blocked check , url parse err:%s", err) return true } domainSlice := strings.Split(u.Hostname(), ".") if len(domainSlice) > 1 { subSlice := domainSlice[:len(domainSlice)-1] topDomain := strings.Join(domainSlice[len(domainSlice)-1:], ".") checkDomain := topDomain for i := len(subSlice) - 1; i >= 0; i-- { checkDomain = subSlice[i] + "." + checkDomain if !blockedMap && c.directMap.Has(checkDomain) { return true } if blockedMap && c.blockedMap.Has(checkDomain) { return true } } } return false } func (c *Checker) Add(address string, isHTTPS bool, method, URL string, data []byte) { if c.domainIsInMap(address, false) || c.domainIsInMap(address, true) { return } if !isHTTPS && strings.ToLower(method) != "get" { return } var item CheckerItem u := strings.Split(address, ":") item = CheckerItem{ URL: URL, Domain: u[0], Host: address, Data: data, IsHTTPS: isHTTPS, Method: method, } c.data.SetIfAbsent(item.Host, item) } type BasicAuth struct { data ConcurrentMap } func NewBasicAuth() BasicAuth { return BasicAuth{ data: NewConcurrentMap(), } } func (ba *BasicAuth) AddFromFile(file string) (n int, err error) { _content, err := ioutil.ReadFile(file) if err != nil { return } userpassArr := strings.Split(strings.Replace(string(_content), "\r", "", -1), "\n") for _, userpass := range userpassArr { if strings.HasPrefix("#", userpass) { continue } u := strings.Split(strings.Trim(userpass, " "), ":") if len(u) == 2 { ba.data.Set(u[0], u[1]) n++ } } return } func (ba *BasicAuth) Add(userpassArr []string) (n int) { for _, userpass := range userpassArr { u := strings.Split(userpass, ":") if len(u) == 2 { ba.data.Set(u[0], u[1]) n++ } } return } func (ba *BasicAuth) Check(userpass string) (ok bool) { u := strings.Split(strings.Trim(userpass, " "), ":") if len(u) == 2 { if p, _ok := ba.data.Get(u[0]); _ok { return p.(string) == u[1] } } return } func (ba *BasicAuth) Total() (n int) { n = ba.data.Count() return }