kylefang/cow
1
0
Fork 0
mirror of https://github.com/zhigang1992/cow.git synced 2026-01-12 17:12:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop', version 0.9.6

Browse Source
This commit is contained in:
Chen Yufei
2015-06-07 22:03:08 +08:00
parent 05a37010e5 fde02a83cb
commit 7f8245aa79
22 changed files with 303 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CHANGELOG
View File

@@ -1,3 +1,10 @@
0.9.6 (2015-06-07)
* Reload config by sending SIGUSR1 on Unix system
* Load blocked/direct/stat file from same directory as rc file by default
* Allow user to specify blocked/direct/stat file path
* Detect arm without vfp in install script.
* Fix estimate timeout bug
0.9.5 (2015-05-12)
* Support new encryption method "chacha20" and "salsa20"
* Avoid biased parent proxy selection for hash load balacing

8
README-en.md
View File

@@ -2,7 +2,7 @@
COW is a HTTP proxy to simplify bypassing the great firewall. It tries to automatically identify blocked websites and only use parent proxy for those sites.
Current version: 0.9.5 [CHANGELOG](CHANGELOG)
Current version: 0.9.6 [CHANGELOG](CHANGELOG)
[![Build Status](https://travis-ci.org/cyfdecyf/cow.png?branch=master)](https://travis-ci.org/cyfdecyf/cow)
## Features
@@ -51,8 +51,8 @@ Command line options can override options in the configuration file For more det
In ideal situation, you don't need to specify which sites are blocked and which are not, but COW hasen't reached that goal. So you may need to manually specify this if COW made the wrong judgement.
- `~/.cow/blocked` for blocked sites
- `~/.cow/direct` for directly accessible sites
- `<dir containing rc file>/blocked` for blocked sites
- `<dir containing rc file>/direct` for directly accessible sites
- One line for each domain
- `google.com` means `*.google.com`
- You can use domains like `google.com.hk`
@@ -61,7 +61,7 @@ In ideal situation, you don't need to specify which sites are blocked and which
## Visited site recording
COW records all visited hosts and visit count in `~/.cow/stat`, which is a json file.
COW records all visited hosts and visit count in `stat` (which is a json file) under the same directory with config file.
- **For unknown site, first try direct access, use parent proxy upon failure. After 2 minutes, try direct access again**
- Builtin [common blocked site](site_blocked.go) in order to reduce time to discover blockage and the use parent proxy

9
README.md
View File

@@ -4,7 +4,7 @@ COW 是一个简化穿墙的 HTTP 代理服务器。它能自动检测被墙网
[English README](README-en.md).
当前版本：0.9.5 [CHANGELOG](CHANGELOG)
当前版本：0.9.6 [CHANGELOG](CHANGELOG)
[![Build Status](https://travis-ci.org/cyfdecyf/cow.png?branch=master)](https://travis-ci.org/cyfdecyf/cow)
**欢迎在 develop branch 进行开发并发送 pull request :)**
@@ -80,7 +80,8 @@ PAC url 为 `http://<listen address>/pac`，也可将浏览器的 HTTP/HTTPS 代
**一般情况下无需手工指定被墙和直连网站，该功能只是是为了处理特殊情况和性能优化。**
`~/.cow/blocked` 和 `~/.cow/direct` 可指定被墙和直连网站（`direct` 中的 host 会添加到 PAC）：
配置文件所在目录下的 `blocked` 和 `direct` 可指定被墙和直连网站（`direct` 中的 host 会添加到 PAC）。
Windows 下文件名为 `blocked.txt` 和 `direct.txt`。
- 每行一个域名或者主机名（COW 会先检查主机名是否在列表中，再检查域名）
- 二级域名如 `google.com` 相当于 `*.google.com`
@@ -91,7 +92,7 @@ PAC url 为 `http://<listen address>/pac`，也可将浏览器的 HTTP/HTTPS 代
## 访问网站记录
COW 在 `~/.cow/stat` json 文件中记录经常访问网站被墙和直连访问的次数。
COW 在配置文件所在目录下的 `stat` json 文件中记录经常访问网站被墙和直连访问的次数。
- **对未知网站，先尝试直接连接，失败后使用二级代理重试请求，2 分钟后再尝试直接**
- 内置[常见被墙网站](site_blocked.go)，减少检测被墙所需时间（可手工添加）
@@ -125,10 +126,10 @@ COW 默认配置下检测到被墙后，过两分钟再次尝试直连也是为
贡献代码：
- @fzerorubigd: various bug fixes and feature implementation
- @tevino: http parent proxy basic authentication
- @xupefei: 提供 cow-hide.exe 以在 windows 上在后台执行 cow.exe
- @sunteya: 改进启动和安装脚本
- @fzerorubigd: identify blocked site by HTTP error code and various bug fixes
Bug reporter:

99
config.go
View File

@@ -16,7 +16,7 @@ import (
)
const (
version = "0.9.5"
version = "0.9.6"
defaultListenAddr = "127.0.0.1:7777"
defaultEstimateTarget = "example.com"
)
@@ -40,10 +40,10 @@ var defaultTunnelAllowedPort = []string{
}
type Config struct {
RcFile string // config file
LogFile string
AlwaysProxy bool
LoadBalance LoadBalanceMode
RcFile string // config file
LogFile string // path for log file
AlwaysProxy bool // whether we should alwyas use parent proxy
LoadBalance LoadBalanceMode // select load balance mode
TunnelAllowedPort map[string]bool // allowed ports to create tunnel
@@ -64,6 +64,11 @@ type Config struct {
HttpErrorCode int
dir string // directory containing config file
StatFile string // Path for stat file
BlockedFile string // blocked sites specified by user
DirectFile string // direct sites specified by user
// not configurable in config file
PrintVer bool
EstimateTimeout bool // Whether to run estimateTimeout().
@@ -76,24 +81,15 @@ type Config struct {
var config Config
var configNeedUpgrade bool // whether should upgrade config file
var configPath struct {
dir string // directory containing config file and blocked site list
alwaysBlocked string // blocked sites specified by user
alwaysDirect string // direct sites specified by user
stat string // site visit statistics
}
func printVersion() {
fmt.Println("cow version", version)
}
func init() {
initConfigDir()
// fmt.Println("home dir:", homeDir)
configPath.alwaysBlocked = path.Join(configPath.dir, alwaysBlockedFname)
configPath.alwaysDirect = path.Join(configPath.dir, alwaysDirectFname)
configPath.stat = path.Join(configPath.dir, statFname)
func initConfig(rcFile string) {
config.dir = path.Dir(rcFile)
config.BlockedFile = path.Join(config.dir, blockedFname)
config.DirectFile = path.Join(config.dir, directFname)
config.StatFile = path.Join(config.dir, statFname)
config.DetectSSLErr = false
config.AlwaysProxy = false
@@ -117,7 +113,7 @@ func parseCmdLineConfig() *Config {
var c Config
var listenAddr string
flag.StringVar(&c.RcFile, "rc", path.Join(configPath.dir, rcFname), "configuration file")
flag.StringVar(&c.RcFile, "rc", "", "config file, defaults to $HOME/.cow/rc on Unix, ./rc.txt on Windows")
// Specifying listen default value to StringVar would override config file options
flag.StringVar(&listenAddr, "listen", "", "listen address, disables listen in config")
flag.IntVar(&c.Core, "core", 2, "number of cores to use")
@@ -126,6 +122,17 @@ func parseCmdLineConfig() *Config {
flag.BoolVar(&c.EstimateTimeout, "estimate", true, "enable/disable estimate timeout")
flag.Parse()
if c.RcFile == "" {
c.RcFile = getDefaultRcFile()
} else {
c.RcFile = expandTilde(c.RcFile)
}
if err := isFileExists(c.RcFile); err != nil {
Fatal("fail to get config file:", err)
}
initConfig(c.RcFile)
if listenAddr != "" {
configParser{}.ParseListen(listenAddr)
cmdHasListenAddr = true // must come after parse
@@ -346,7 +353,7 @@ func (p configParser) ParseListen(val string) {
}
func (p configParser) ParseLogFile(val string) {
config.LogFile = val
config.LogFile = expandTilde(val)
}
func (p configParser) ParseAddrInPAC(val string) {
@@ -450,6 +457,24 @@ func (p configParser) ParseLoadBalance(val string) {
}
}
func (p configParser) ParseStatFile(val string) {
config.StatFile = expandTilde(val)
}
func (p configParser) ParseBlockedFile(val string) {
config.BlockedFile = expandTilde(val)
if err := isFileExists(config.BlockedFile); err != nil {
Fatal("blocked file:", err)
}
}
func (p configParser) ParseDirectFile(val string) {
config.DirectFile = expandTilde(val)
if err := isFileExists(config.DirectFile); err != nil {
Fatal("direct file:", err)
}
}
var shadow struct {
parent *shadowsocksParent
passwd string
@@ -527,12 +552,9 @@ func (p configParser) ParseUserPasswd(val string) {
}
func (p configParser) ParseUserPasswdFile(val string) {
exist, err := isFileExists(val)
err := isFileExists(val)
if err != nil {
Fatal("userPasswdFile error:", err)
}
if !exist {
Fatal("userPasswdFile", val, "does not exist")
Fatal("userPasswdFile:", err)
}
config.UserPasswdFile = val
}
@@ -575,12 +597,7 @@ func parseConfig(rc string, override *Config) {
// fmt.Println("rcFile:", path)
f, err := os.Open(expandTilde(rc))
if err != nil {
if os.IsNotExist(err) {
fmt.Printf("Config file %s not found, using default options\n", rc)
} else {
fmt.Println("Error opening config file:", err)
}
return
Fatal("Error opening config file:", err)
}
IgnoreUTF8BOM(f)
@@ -730,21 +747,3 @@ func checkConfig() {
listenProxy = []Proxy{newHttpProxy(defaultListenAddr, "")}
}
}
func mkConfigDir() (err error) {
if configPath.dir == "" {
return os.ErrNotExist
}
exists, err := isDirExists(configPath.dir)
if err != nil {
errl.Printf("Error checking config directory: %v\n", err)
return
}
if exists {
return
}
if err = os.Mkdir(configPath.dir, 0755); err != nil {
errl.Printf("Error create config directory %s: %v\n", configPath.dir, err)
}
return
}

1
config_test.go
View File

@@ -24,6 +24,7 @@ func TestParseListen(t *testing.T) {
}
func TestTunnelAllowedPort(t *testing.T) {
initConfig("")
parser := configParser{}
parser.ParseTunnelAllowedPort("1, 2, 3, 4, 5")
parser.ParseTunnelAllowedPort("6")

13
config_unix.go
View File

@@ -7,15 +7,14 @@ import (
)
const (
rcFname = "rc"
alwaysBlockedFname = "blocked"
alwaysDirectFname = "direct"
statFname = "stat"
rcFname = "rc"
blockedFname = "blocked"
directFname = "direct"
statFname = "stat"
newLine = "\n"
)
func initConfigDir() {
home := getUserHomeDir()
configPath.dir = path.Join(home, ".cow")
func getDefaultRcFile() string {
return path.Join(path.Join(getUserHomeDir(), ".cow", rcFname))
}

12
config_windows.go
View File

@@ -6,16 +6,16 @@ import (
)
const (
rcFname = "rc.txt"
alwaysBlockedFname = "blocked.txt"
alwaysDirectFname = "direct.txt"
statFname = "stat.txt"
rcFname = "rc.txt"
blockedFname = "blocked.txt"
directFname = "direct.txt"
statFname = "stat.txt"
newLine = "\r\n"
)
func initConfigDir() {
func getDefaultRcFile() string {
// On windows, put the configuration file in the same directory of cow executable
// This is not a reliable way to detect binary directory, but it works for double click and run
configPath.dir = path.Dir(os.Args[0])
return path.Join(path.Dir(os.Args[0]), rcFname)
}

6
doc/sample-config/rc
View File

@@ -148,3 +148,9 @@ listen = http://127.0.0.1:7777
# （Chrome 遇到 SSL 错误会直接关闭连接，而不是让用户选择是否继续）
# 可能将可直连网站误判为被墙网站，当 GFW 进行 SSL 中间人攻击时可以考虑使用
#detectSSLErr = false
# 修改 stat/blocked/direct 文件路径，如不指定，默认在配置文件所在目录下
# 执行 cow 的用户需要有对 stat 文件所在目录的写权限才能更新 stat 文件
#statFile = <dir to rc file>/stat
#blockedFile = <dir to rc file>/blocked
#directFile = <dir to rc file>/direct

8
doc/sample-config/rc-en
View File

@@ -173,3 +173,11 @@ listen = http://127.0.0.1:7777
# This detection is no reliable, may mistaken normal sites as blocked.
# Only consider this option when GFW is making middle man attack.
#detectSSLErr = false
# Change the stat/blocked/direct file position, defaults to files under directory
# containing rc file.
# The cow user must write access to directory containing the stat file in order
# to update stat.
#statFile = <dir to rc file>/stat
#blockedFile = <dir to rc file>/blocked
#directFile = <dir to rc file>/direct

34
estimate_timeout.go
View File

@@ -1,6 +1,7 @@
package main
import (
"fmt"
"io"
"net"
"time"
@@ -17,28 +18,19 @@ const maxTimeout = 15 * time.Second
var dialTimeout = defaultDialTimeout
var readTimeout = defaultReadTimeout
var estimateReq = []byte("GET / HTTP/1.1\r\n" +
"Host: " + config.EstimateTarget + "\r\n" +
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:11.0) Gecko/20100101 Firefox/11.0\r\n" +
"Accept: */*\r\n" +
"Accept-Language: en-us,en;q=0.5\r\n" +
"Accept-Encoding: gzip, deflate\r\n" +
"Connection: close\r\n\r\n")
// estimateTimeout tries to fetch a url and adjust timeout value according to
// how much time is spent on connect and fetch. This avoids incorrectly
// considering non-blocked sites as blocked when network connection is bad.
func estimateTimeout() {
func estimateTimeout(host string, payload []byte) {
//debug.Println("estimating timeout")
buf := connectBuf.Get()
defer connectBuf.Put(buf)
var est time.Duration
start := time.Now()
c, err := net.Dial("tcp", config.EstimateTarget+":80")
c, err := net.Dial("tcp", host+":80")
if err != nil {
errl.Printf("estimateTimeout: can't connect to %s: %v, network has problem?\n",
config.EstimateTarget, err)
host, err)
goto onErr
}
defer c.Close()
@@ -59,7 +51,8 @@ func estimateTimeout() {
start = time.Now()
// include time spent on sending request, reading all content to make it a
// little longer
if _, err = c.Write(estimateReq); err != nil {
if _, err = c.Write(payload); err != nil {
errl.Println("estimateTimeout: error sending request:", err)
goto onErr
}
@@ -68,7 +61,7 @@ func estimateTimeout() {
}
if err != io.EOF {
errl.Printf("estimateTimeout: error getting %s: %v, network has problem?\n",
config.EstimateTarget, err)
host, err)
goto onErr
}
est = time.Now().Sub(start) * 10
@@ -90,10 +83,21 @@ onErr:
}
func runEstimateTimeout() {
const estimateReq = "GET / HTTP/1.1\r\n" +
"Host: %s\r\n" +
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:11.0) Gecko/20100101 Firefox/11.0\r\n" +
"Accept: */*\r\n" +
"Accept-Language: en-us,en;q=0.5\r\n" +
"Accept-Encoding: gzip, deflate\r\n" +
"Connection: close\r\n\r\n"
readTimeout = config.ReadTimeout
dialTimeout = config.DialTimeout
payload := []byte(fmt.Sprintf(estimateReq, config.EstimateTarget))
for {
estimateTimeout()
estimateTimeout(config.EstimateTarget, payload)
time.Sleep(time.Minute)
}
}

11
install-cow.sh
View File

@@ -1,6 +1,6 @@
#!/bin/bash
version=0.9.5
version=0.9.6
arch=`uname -m`
case $arch in
@@ -11,7 +11,14 @@ case $arch in
arch="32"
;;
"armv5tel" | "armv6l" | "armv7l")
arch="-$arch"
features=`cat /proc/cpuinfo | grep Features`
if [[ ! "$features" =~ "vfp" ]]; then
#arm without vfp must use GOARM=5 binary
#see https://github.com/golang/go/wiki/GoArm
arch="-armv5tel"
else
arch="-$arch"
fi
;;
*)
echo "$arch currently has no precompiled binary"

11
log.go
View File

@@ -6,10 +6,11 @@ package main
import (
"flag"
"fmt"
"github.com/cyfdecyf/color"
"io"
"log"
"os"
"github.com/cyfdecyf/color"
)
type infoLogging bool
@@ -28,10 +29,10 @@ var (
logFile io.Writer
// make sure logger can be called before initLog
errorLog = log.New(os.Stdout, "", log.LstdFlags)
debugLog = errorLog
requestLog = errorLog
responseLog = errorLog
errorLog = log.New(os.Stdout, "[ERROR] ", log.LstdFlags)
debugLog = log.New(os.Stdout, "[DEBUG] ", log.LstdFlags)
requestLog = log.New(os.Stdout, "[>>>>>] ", log.LstdFlags)
responseLog = log.New(os.Stdout, "[<<<<<] ", log.LstdFlags)
verbose bool
colorize bool

52
main.go
View File

@@ -3,7 +3,7 @@ package main
import (
// "flag"
"os"
"os/signal"
"os/exec"
"runtime"
// "runtime/pprof"
"sync"
@@ -11,28 +11,25 @@ import (
)
// var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var (
quit chan struct{}
relaunch bool
)
func sigHandler() {
// TODO On Windows, these signals will not be triggered on closing cmd
// window. How to detect this?
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
for sig := range sigChan {
// May handle other signals in the future.
info.Printf("%v caught, exit\n", sig)
storeSiteStat(siteStatExit)
break
// This code is from goagain
func lookPath() (argv0 string, err error) {
argv0, err = exec.LookPath(os.Args[0])
if nil != err {
return
}
/*
if *cpuprofile != "" {
pprof.StopCPUProfile()
}
*/
os.Exit(0)
if _, err = os.Stat(argv0); nil != err {
return
}
return
}
func main() {
quit = make(chan struct{})
// Parse flags after load config to allow override options in config
cmdLineConfig := parseCmdLineConfig()
if cmdLineConfig.PrintVer {
@@ -77,7 +74,24 @@ func main() {
var wg sync.WaitGroup
wg.Add(len(listenProxy))
for _, proxy := range listenProxy {
go proxy.Serve(&wg)
go proxy.Serve(&wg, quit)
}
wg.Wait()
if relaunch {
info.Println("Relunching cow...")
// Need to fork me.
argv0, err := lookPath()
if nil != err {
errl.Println(err)
return
}
err = syscall.Exec(argv0, os.Args, os.Environ())
if err != nil {
errl.Println(err)
}
}
debug.Println("the main process is , exiting...")
}

30
main_unix.go Normal file
View File

@@ -0,0 +1,30 @@
// +build darwin freebsd linux netbsd openbsd
package main
import (
"os"
"os/signal"
"syscall"
)
func sigHandler() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1)
for sig := range sigChan {
// May handle other signals in the future.
info.Printf("%v caught, exit\n", sig)
storeSiteStat(siteStatExit)
if sig == syscall.SIGUSR1 {
relaunch = true
}
close(quit)
break
}
/*
if *cpuprofile != "" {
pprof.StopCPUProfile()
}
*/
}

33
main_windows.go Normal file
View File

@@ -0,0 +1,33 @@
package main
import (
"os"
"os/signal"
"syscall"
)
func sigHandler() {
// TODO On Windows, these signals will not be triggered on closing cmd
// window. How to detect this?
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
for sig := range sigChan {
// May handle other signals in the future.
info.Printf("%v caught, exit\n", sig)
storeSiteStat(siteStatExit)
// Windows has no SIGUSR1 signal, so relaunching is not supported now.
/*
if sig == syscall.SIGUSR1 {
relaunch = true
}
*/
close(quit)
break
}
/*
if *cpuprofile != "" {
pprof.StopCPUProfile()
}
*/
}

2
parent_proxy.go
View File

@@ -289,7 +289,7 @@ func (pp *latencyParentPool) updateLatency() {
// Sort according to latency.
sort.Stable(&cp)
debug.Println("lantency lowest proxy", cp.parent[0].getServer())
debug.Println("latency lowest proxy", cp.parent[0].getServer())
// Update parent slice.
latencyMutex.Lock()

40
proxy.go
View File

@@ -4,14 +4,15 @@ import (
"bytes"
"errors"
"fmt"
"github.com/cyfdecyf/bufio"
"github.com/cyfdecyf/leakybuf"
ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
"io"
"net"
"strings"
"sync"
"time"
"github.com/cyfdecyf/bufio"
"github.com/cyfdecyf/leakybuf"
ss "github.com/shadowsocks/shadowsocks-go/shadowsocks"
)
// As I'm using ReadSlice to read line, it's possible to get
@@ -105,7 +106,7 @@ var (
)
type Proxy interface {
Serve(*sync.WaitGroup)
Serve(*sync.WaitGroup, <-chan struct{})
Addr() string
genConfig() string // for upgrading config
}
@@ -142,7 +143,7 @@ func (proxy *httpProxy) Addr() string {
return proxy.addr
}
func (hp *httpProxy) Serve(wg *sync.WaitGroup) {
func (hp *httpProxy) Serve(wg *sync.WaitGroup, quit <-chan struct{}) {
defer func() {
wg.Done()
}()
@@ -151,6 +152,12 @@ func (hp *httpProxy) Serve(wg *sync.WaitGroup) {
fmt.Println("listen http failed:", err)
return
}
var exit bool
go func() {
<-quit
exit = true
ln.Close()
}()
host, _, _ := net.SplitHostPort(hp.addr)
var pacURL string
if host == "" || host == "0.0.0.0" {
@@ -164,7 +171,7 @@ func (hp *httpProxy) Serve(wg *sync.WaitGroup) {
for {
conn, err := ln.Accept()
if err != nil {
if err != nil && !exit {
errl.Printf("http proxy(%s) accept %v\n", ln.Addr(), err)
if isErrTooManyOpenFd(err) {
connPool.CloseAll()
@@ -172,8 +179,13 @@ func (hp *httpProxy) Serve(wg *sync.WaitGroup) {
time.Sleep(time.Millisecond)
continue
}
if exit {
debug.Println("exiting the http listner")
break
}
c := newClientConn(conn, hp)
go c.serve()
}
}
@@ -204,20 +216,26 @@ func (cp *cowProxy) Addr() string {
return cp.addr
}
func (cp *cowProxy) Serve(wg *sync.WaitGroup) {
func (cp *cowProxy) Serve(wg *sync.WaitGroup, quit <-chan struct{}) {
defer func() {
wg.Done()
}()
ln, err := net.Listen("tcp", cp.addr)
if err != nil {
fmt.Println("listen cow failed:", err)
return
}
info.Printf("COW %s cow proxy address %s\n", version, cp.addr)
var exit bool
go func() {
<-quit
exit = true
ln.Close()
}()
for {
conn, err := ln.Accept()
if err != nil {
if err != nil && !exit {
errl.Printf("cow proxy(%s) accept %v\n", ln.Addr(), err)
if isErrTooManyOpenFd(err) {
connPool.CloseAll()
@@ -225,6 +243,10 @@ func (cp *cowProxy) Serve(wg *sync.WaitGroup) {
time.Sleep(time.Millisecond)
continue
}
if exit {
debug.Println("exiting cow listner")
break
}
ssConn := ss.NewConn(conn, cp.cipher.Copy())
c := newClientConn(ssConn, cp)
go c.serve()

2
script/set-version.sh
View File

@@ -13,4 +13,4 @@ version=$1
sed -i -e "s,\(\tversion \+= \)\".*\"$,\1\"$version\"," config.go
sed -i -e "s/version=.*$/version=$version/" install-cow.sh
sed -i -e "s/当前版本：[^ ]\+ \(.*\)\$/当前版本：$version \1/" README.md
sed -i -e "s/Current version: [^ ]\+ \(.*\)\$/Current version: $version \1/" README-en.md

2
script/test.sh
View File

@@ -88,4 +88,6 @@ if [[ -z $TRAVIS ]]; then
fi
stop_cow
sleep 0.5
rm -f ./script/stat*
exit 0

48
sitestat.go
View File

@@ -253,10 +253,6 @@ func (ss *SiteStat) GetVisitCnt(url *URL) (vcnt *VisitCnt) {
}
func (ss *SiteStat) store(statPath string) (err error) {
if err = mkConfigDir(); err != nil {
return
}
now := time.Now()
var savedSS *SiteStat
if ss.Update == Date(zeroTime) {
@@ -293,7 +289,7 @@ func (ss *SiteStat) store(statPath string) (err error) {
// Ensures atomic update to stat file to avoid file damage.
// Create tmp file inside config firectory to avoid cross FS rename.
f, err := ioutil.TempFile(configPath.dir, "stat")
f, err := ioutil.TempFile(config.dir, "stat")
if err != nil {
errl.Println("create tmp file to store stat", err)
return
@@ -327,10 +323,10 @@ func (ss *SiteStat) loadBuiltinList() {
}
func (ss *SiteStat) loadUserList() {
if directList, err := loadSiteList(configPath.alwaysDirect); err == nil {
if directList, err := loadSiteList(config.DirectFile); err == nil {
ss.loadList(directList, userCnt, 0)
}
if blockedList, err := loadSiteList(configPath.alwaysBlocked); err == nil {
if blockedList, err := loadSiteList(config.BlockedFile); err == nil {
ss.loadList(blockedList, 0, userCnt)
}
}
@@ -382,27 +378,28 @@ func (ss *SiteStat) load(file string) (err error) {
}
}
}()
var exist bool
if exist, err = isFileExists(file); err != nil {
fmt.Println("Error loading stat:", err)
if file == "" {
return
}
if !exist {
if err = isFileExists(file); err != nil {
if !os.IsNotExist(err) {
errl.Println("Error loading stat:", err)
}
return
}
var f *os.File
if f, err = os.Open(file); err != nil {
fmt.Printf("Error opening site stat %s: %v\n", file, err)
errl.Printf("Error opening site stat %s: %v\n", file, err)
return
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
fmt.Println("Error reading site stat:", err)
errl.Println("Error reading site stat:", err)
return
}
if err = json.Unmarshal(b, ss); err != nil {
fmt.Println("Error decoding site stat:", err)
errl.Println("Error decoding site stat:", err)
return
}
return
@@ -427,15 +424,16 @@ func (ss *SiteStat) GetDirectList() []string {
var siteStat = newSiteStat()
func initSiteStat() {
err := siteStat.load(configPath.stat)
err := siteStat.load(config.StatFile)
if err != nil {
errl.Printf("loading stat file failed, reason : %s", err.Error())
// Simply try to load the stat.back
err = siteStat.load(configPath.stat + ".bak")
// Simply try to load the stat.back, create a new object to avoid error
// in default site list.
siteStat = newSiteStat()
err = siteStat.load(config.StatFile + ".bak")
// After all its not critical , simply re-create a stat object if anything is not ok
if err != nil {
errl.Printf("loading stat backup failed, creating new one , reason: %s", err.Error())
siteStat = newSiteStat()
siteStat.load("") // load default site list
}
}
@@ -466,18 +464,20 @@ func storeSiteStat(cont byte) {
if siteStatFini {
return
}
siteStat.store(configPath.stat)
siteStat.store(config.StatFile)
if cont == siteStatExit {
siteStatFini = true
}
}
func loadSiteList(fpath string) (lst []string, err error) {
var exists bool
if exists, err = isFileExists(fpath); err != nil {
errl.Printf("Error loading domaint list: %v\n", err)
if fpath == "" {
return
}
if !exists {
if err = isFileExists(fpath); err != nil {
if !os.IsNotExist(err) {
info.Printf("Error loading domaint list: %v\n", err)
}
return
}
f, err := os.Open(fpath)

94
util.go
View File

@@ -5,7 +5,6 @@ import (
"crypto/md5"
"errors"
"fmt"
"github.com/cyfdecyf/bufio"
"io"
"net"
"os"
@@ -13,6 +12,8 @@ import (
"runtime"
"strconv"
"strings"
"github.com/cyfdecyf/bufio"
)
const isWindows = runtime.GOOS == "windows"
@@ -205,32 +206,26 @@ func ParseIntFromBytes(b []byte, base int) (n int64, err error) {
return
}
func isFileExists(path string) (bool, error) {
func isFileExists(path string) error {
stat, err := os.Stat(path)
if err == nil {
if stat.Mode()&os.ModeType == 0 {
return true, nil
}
return false, errors.New(path + " exists but is not regular file")
if err != nil {
return err
}
if os.IsNotExist(err) {
return false, nil
if !stat.Mode().IsRegular() {
return fmt.Errorf("%s is not regular file", path)
}
return false, err
return nil
}
func isDirExists(path string) (bool, error) {
func isDirExists(path string) error {
stat, err := os.Stat(path)
if err == nil {
if stat.IsDir() {
return true, nil
}
return false, errors.New(path + " exists but is not directory")
if err != nil {
return err
}
if os.IsNotExist(err) {
return false, nil
if !stat.IsDir() {
return fmt.Errorf("%s is not directory", path)
}
return false, err
return nil
}
func getUserHomeDir() string {
@@ -286,67 +281,6 @@ func copyN(dst io.Writer, src *bufio.Reader, n, rdSize int) (err error) {
return err
}
// copyNWithBuf copys N bytes from src to dst, using the specified buf as buffer. pre and
// end are written to w before and after the n bytes. copyN will try to
// minimize number of writes.
// No longer used now.
func copyNWithBuf(dst io.Writer, src io.Reader, n int, buf, pre, end []byte) (err error) {
// XXX well, this is complicated in order to save writes
var nn int
bufLen := len(buf)
var b []byte
for n != 0 {
if pre != nil {
if len(pre) >= bufLen {
// pre is larger than bufLen, can't save write operation here
if _, err = dst.Write(pre); err != nil {
return
}
pre = nil
continue
}
// append pre to buf to save one write
copy(buf, pre)
if len(pre)+n < bufLen {
// only need to read n bytes
b = buf[len(pre) : len(pre)+n]
} else {
b = buf[len(pre):]
}
} else {
if n < bufLen {
b = buf[:n]
} else {
b = buf
}
}
if nn, err = src.Read(b); err != nil {
return
}
n -= nn
if pre != nil {
// nn is how much we need to write next
nn += len(pre)
pre = nil
}
// see if we can append end in buffer to save one write
if n == 0 && end != nil && nn+len(end) <= bufLen {
copy(buf[nn:], end)
nn += len(end)
end = nil
}
if _, err = dst.Write(buf[:nn]); err != nil {
return
}
}
if end != nil {
if _, err = dst.Write(end); err != nil {
return
}
}
return
}
func md5sum(ss ...string) string {
h := md5.New()
for _, s := range ss {

78
util_test.go
View File

@@ -3,9 +3,10 @@ package main
import (
"bytes"
"errors"
"github.com/cyfdecyf/bufio"
"strings"
"testing"
"github.com/cyfdecyf/bufio"
)
func TestASCIIToUpper(t *testing.T) {
@@ -201,81 +202,18 @@ func TestCopyN(t *testing.T) {
}
}
func TestCopyNWithBuf(t *testing.T) {
testStr := "hello world"
src := bytes.NewBufferString(testStr)
dst := new(bytes.Buffer)
buf := make([]byte, 5)
copyNWithBuf(dst, src, len(testStr), buf, nil, nil)
if dst.String() != "hello world" {
t.Error("copy without pre and end failed, got:", dst.String())
}
src.Reset()
dst.Reset()
src.WriteString(testStr)
copyNWithBuf(dst, src, len(testStr), buf, []byte("by cyf "), nil)
if dst.String() != "by cyf hello world" {
t.Error("copy with pre no end failed, got:", dst.String())
}
src.Reset()
dst.Reset()
src.WriteString(testStr)
copyNWithBuf(dst, src, len(testStr), buf, []byte("by cyf "), []byte(" welcome"))
if dst.String() != "by cyf hello world welcome" {
t.Error("copy with both pre and end failed, got:", dst.String())
}
src.Reset()
dst.Reset()
src.WriteString(testStr)
copyNWithBuf(dst, src, len(testStr), buf, []byte("pre longer then buffer "), []byte(" welcome"))
if dst.String() != "pre longer then buffer hello world welcome" {
t.Error("copy with long pre failed, got:", dst.String())
}
src.Reset()
dst.Reset()
testStr = "34"
src.WriteString(testStr)
copyNWithBuf(dst, src, len(testStr), buf, []byte("12"), []byte(" welcome"))
if dst.String() != "1234 welcome" {
t.Error("copy len(pre)+size<bufLen failed, got:", dst.String())
}
src.Reset()
dst.Reset()
testStr = "2"
src.WriteString(testStr)
copyNWithBuf(dst, src, len(testStr), buf, []byte("1"), []byte("34"))
if dst.String() != "1234" {
t.Error("copy len(pre)+size+len(end)<bufLen failed, got:", dst.String())
}
}
func TestIsFileExists(t *testing.T) {
exists, err := isFileExists("testdata")
err := isFileExists("testdata")
if err == nil {
t.Error("should return error is path is directory")
}
if exists {
t.Error("directory should return false")
err = isFileExists("testdata/none")
if err == nil {
t.Error("Not existing file should return error")
}
exists, err = isFileExists("testdata/none")
if exists {
t.Error("BOOM! You've found a non-existing file!")
}
if err != nil {
t.Error("Not existing file should just return false, on error")
}
exists, err = isFileExists("testdata/file")
if !exists {
t.Error("testdata/file exists, but returns false")
}
err = isFileExists("testdata/file")
if err != nil {
t.Error("Why error for existing file?")
}