Golang程序内存泄漏？

我的golang程序（url monitor）有内存泄漏，它最终被内核（oom）杀死。的ENV：Golang程序内存泄漏？

$ go version 
go version go1.0.3 

$ go env 
GOARCH="amd64" 
GOBIN="" 
GOCHAR="6" 
GOEXE="" 
GOGCCFLAGS="-g -O2 -fPIC -m64 -pthread" 
GOHOSTARCH="amd64" 
GOHOSTOS="linux" 
GOOS="linux" 
GOPATH="/data/apps/go" 
GOROOT="/usr/local/go" 
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64" 
CGO_ENABLED="1"

代码：

package main 

import (
    "bytes" 
    "database/sql" 
    "flag" 
    "fmt" 
    _ "github.com/Go-SQL-Driver/MySQL" 
    "ijinshan.com/cfg" 
    "log" 
    "net" 
    "net/http" 
    "net/smtp" 
    "os" 
    "strconv" 
    "strings" 
    "sync" 
    "time" 
) 

var (
    Log   *log.Logger 
    Conf   cfg.KVConfig 
    Debug   bool 
    CpuCore  int 
    HttpTransport = &http.Transport{ 
     Dial: func(netw, addr string) (net.Conn, error) { 
      deadline := time.Now().Add(30 * time.Second) 
      c, err := net.DialTimeout(netw, addr, 20*time.Second) 
      if err != nil { 
       return nil, err 
      } 

      c.SetDeadline(deadline) 
      return c, nil 
     }, 
     DisableKeepAlives: true, 
    } 
    HttpClient = &http.Client{ 
     Transport: HttpTransport, 
    } 
    WG   sync.WaitGroup 
) 

const (
    LogFileFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND 
    LogFileMode = 0644 
    LogFlag  = log.LstdFlags | log.Lshortfile 
    GET_VIDEO_SQL = `SELECT B.Name, A.TSID, A.Chapter, A.ChapterNum, 
    IFNULL(A.Website, ''), IFNULL(A.Descr, ''), 
    IFNULL(A.VideoId, ''), IFNULL(AndroidWebURL, ''), IFNULL(IOSWebURL, ''), 
    IFNULL(AndroidURL, ''), IFNULL(AndroidURL2, ''), IFNULL(IOSURL, '') 
    FROM Video A INNER JOIN TVS B ON A.TSID = B.ID LIMIT 200` 

    HtmlHead = `<table border=1 width=100% height=100%><tr><td>节目名 
    </td><td>tsid</td><td>章节</td><td>章节号</td><td>描述 
    </td><td>videoid</td><td>网站</td><td>地址</td></tr>` 
    HtmlTail = "</table>" 
) 

type videoInfo struct { 
    name   string 
    tsid   uint 
    chapter  string 
    chapterNum uint 
    descr   string 
    videoId  string 
    website  string 
    androidWebUrl string 
    iosWebUrl  string 
    androidUrl string 
    androidUrl2 string 
    iosUrl  string 
} 

func init() { 
    var (
     confFile string 
     err  error 
    ) 

    // parse command argument:w 
    flag.StringVar(&confFile, "c", "./vsmonitor.conf", " set config file path") 
    flag.Parse() 
    // read config 
    if Conf, err = cfg.Read(confFile); err != nil { 
     panic(fmt.Sprintf("Read config file \"%s\" failed (%s)", 
      confFile, err.Error())) 
    } 
    // open log file 
    file, err := os.OpenFile(Conf["log.file"], LogFileFlag, LogFileMode) 
    if err != nil { 
     panic(fmt.Sprintf("OpenFile \"%s\" failed (%s)", Conf["log.file"], 
      err.Error())) 
    } 
    // init LOG 
    Log = log.New(file, "", LogFlag) 
    Debug = false 
    i, err := strconv.ParseInt(Conf["cpucore.num"], 10, 32) 
    if err != nil { 
     panic(fmt.Sprintf("ParseInt \"%s\" failed (%s)", Conf["cpucore.num"], 
      err.Error())) 
    } 

    CpuCore = int(i) 
} 

func getHttpStatusCode(url string) int { 
    if url == "" { 
     return 200 
    } 

    req, err := http.NewRequest("GET", url, nil) 
    if err != nil { 
     return 0 
    } 

    req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.57 Safari/537.17") 
    req.Header.Add("Connection", "close") 
    resp, err := HttpClient.Do(req) 
    if err != nil { 
     return 0 
    } 

    defer resp.Body.Close() 
    return resp.StatusCode 
} 

func sendMail(host, user, pwd, from, to, subject, body, mailType string) error { 
    auth := smtp.PlainAuth("", user, pwd, strings.Split(host, ":")[0]) 
    cntType := fmt.Sprintf("Content-Type: text/%s;charset=UTF-8", mailType) 
    msg := fmt.Sprintf("To: %s\r\nFrom: %s<%s>\r\nSubject: %s\r\n%s\r\n\r\n%s", 
     to, from, user, subject, cntType, body) 

    return smtp.SendMail(host, auth, user, strings.Split(to, ","), []byte(msg)) 
} 

func getVideos(videoChan chan *videoInfo, htmlBuf *bytes.Buffer) error { 
    defer HttpTransport.CloseIdleConnections() 
    db, err := sql.Open("mysql", Conf["weikan.mysql"]) 
    if err != nil { 
     return err 
    } 

    rows, err := db.Query(GET_VIDEO_SQL) 
    if err != nil { 
     db.Close() 
     return err 
    } 

    for rows.Next() { 
     video := &videoInfo{} 
     err = rows.Scan(&video.name, &video.tsid, &video.chapter, 
      &video.chapterNum, 
      &video.website, &video.descr, &video.videoId, &video.androidWebUrl, 
      &video.iosWebUrl, &video.androidUrl, &video.androidUrl2, 
      &video.iosUrl) 
     if err != nil { 
      db.Close() 
      return err 
     } 

     videoChan <- video 
     WG.Add(1) 
    } 

    db.Close() 
    // wait check url finish 
    WG.Wait() 
    // send mail 
    for { 
     if htmlBuf.Len() == 0 { 
      Log.Print("no error found!!!!!!!!") 
      break 
     } 

     Log.Print("found error !!!!!!!!") 
     /* 
     err := sendMail("smtp.gmail.com:587", "xxxx", 
      "xxx", "xxx <xxx>", 
      Conf["mail.to"], "xxxxx", 
      HtmlHead+htmlBuf.String()+HtmlTail, "html") 
     if err != nil { 
      Log.Printf("sendMail failed (%s)", err.Error()) 
      time.Sleep(10 * time.Second) 
      continue 
     } 
     */ 

     Log.Print("send mail") 
     break 
    } 

    Log.Print("reset buf") 
    htmlBuf.Reset() 
    return nil 
} 

func checkUrl(videoChan chan *videoInfo, errChan chan string) { 
    defer func() { 
     if err := recover(); err != nil { 
      Log.Print("rouintes failed : ", err) 
     } 
    }() 

    for { 
     video := <-videoChan 
     ok := true 
     errUrl := "" 

     if code := getHttpStatusCode(video.androidWebUrl); code >= 400 { 
      errUrl += fmt.Sprintf("%s (%d)<br />", 
       video.androidWebUrl, code) 
      ok = false 
     } 

     if code := getHttpStatusCode(video.iosWebUrl); code >= 400 { 
      errUrl += fmt.Sprintf("%s (%d)<br />", 
       video.iosWebUrl, code) 
      ok = false 
     } 

     if code := getHttpStatusCode(video.androidUrl); code >= 400 { 
      errUrl += fmt.Sprintf("%s (%d)<br />", 
       video.androidUrl, code) 
      ok = false 
     } 

     if code := getHttpStatusCode(video.androidUrl2); code >= 400 { 
      errUrl += fmt.Sprintf("%s (%d)<br />", 
       video.androidUrl2, code) 
      ok = false 
     } 

     if code := getHttpStatusCode(video.iosUrl); code >= 400 { 
      errUrl += fmt.Sprintf("%s (%d)<br />", 
       video.iosUrl, code) 
      ok = false 
     } 

     if !ok { 
      errChan <- fmt.Sprintf(`<tr><td>%s</td><td>%d</td><td>%s</td> 
      <td>%d</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>`, 
       video.name, video.tsid, video.chapter, video.chapterNum, 
       video.descr, video.videoId, 
       video.website, errUrl) 
      Log.Printf("\"%s\" (%s) —— \"%s\" checkurl err", video.name, 
       video.chapter, video.descr) 
     } else { 
      Log.Printf("\"%s\" (%s) —— \"%s\" checkurl ok", video.name, 
       video.chapter, video.descr) 
      WG.Done() 
     } 
    } 
} 

func mergeErr(errChan chan string, htmlBuf *bytes.Buffer) { 
    defer func() { 
     if err := recover(); err != nil { 
      Log.Print("rouintes failed : ", err) 
     } 
    }() 

    for { 
     html := <-errChan 
     _, err := htmlBuf.WriteString(html) 
     if err != nil { 
      Log.Printf("htmlBuf WriteString \"%s\" failed (%s)", html, 
       err.Error()) 
      panic(err) 
     } 

     WG.Done() 
    } 
} 

func main() { 
    videoChan := make(chan *videoInfo, 100000) 
    errChan := make(chan string, 100000) 
    htmlBuf := &bytes.Buffer{} 
    defer func() { 
     if err := recover(); err != nil { 
      Log.Print("rouintes failed : ", err) 
     } 
    }() 

    // check url 
    for i := 0; i < CpuCore; i++ { 
     go checkUrl(videoChan, errChan) 
    } 
    // merge error string then send mail 
    go mergeErr(errChan, htmlBuf) 

    for { 
     // get Video and LiveSrc video source 
     if err := getVideos(videoChan, htmlBuf); err != nil { 
      Log.Printf("getVideos failed (%s)", err.Error()) 
      time.Sleep(10 * time.Second) 
      continue 
     } 

     // time.Sleep(1 * time.Hour) 
    } 

    Log.Print("exit...") 
}

代码有四个funcs：

getHttpStatusCode

免费资源利用resp.Body.Close（）

Sendmail的

我不需要使用htmlBuf（* bytes.Buffer）以释放资源手动

mergeErr

CONCAT的ERR串

getVideos

首先它获取Video urls，然后将它们发送到videoChan，然后等待所有例程完成其检查作业。然后发送邮件并重置htmlBuf。

我没有找到任何需要免费的资源，但是。

$顶部

显示：

PID USER  PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND                              
6451 root  20 0 3946m 115m 2808 S 0.7 0.2 6:11.20 vsmonitor

的VIRT和RES将增长......

内存分析：

(pprof) top 
Total: 10.8 MB 
2.3 21.2% 21.2%  2.3 21.2% main.main 
2.0 18.5% 39.8%  2.0 18.5% bufio.NewWriterSize 
1.5 13.9% 53.7%  1.5 13.9% bufio.NewReaderSize 
1.5 13.9% 67.6%  1.5 13.9% compress/flate.NewReader 
0.5 4.6% 72.2%  0.5 4.6% net.newFD 
0.5 4.6% 76.8%  0.5 4.6% net.sockaddrToTCP 
0.5 4.6% 81.5%  4.5 41.7% net/http.(*Transport).getConn 
0.5 4.6% 86.1%  2.5 23.2% net/http.(*persistConn).readLoop 
0.5 4.6% 90.7%  0.5 4.6% net/textproto.(*Reader).ReadMIMEHeader 
0.5 4.6% 95.4%  0.5 4.6% net/url.(*URL).ResolveReference

来源

2013-03-28 Terry

的递归可能会占用大量的堆空间，最终，为Go使用分离电池堆，其能成长而不是固定的存储区域。寻找无尽（足够大）的递归。 – nemo 2013-03-28 05:40:47

但我的代码只包含**循环** – Terry 2013-03-28 05:53:48

注意让分析运行一段时间，直到可以看到实际的增长？ – nemo 2013-03-28 23:09:01

这是很容易加您的程序的一个选项，以便它将记录正在使用的内存的位置。在你的节目中，没有什么东西突出我的错误。你下载的文件非常大吗？你可以做一个HEAD请求吗？我不知道这是否有帮助;如果你有大量的请求，也许会。

有随身博客关于内存分析的（旧-ISH）的文章在http://blog.golang.org/2011/06/profiling-go-programs.html和文档在http://golang.org/pkg/runtime/pprof/和http://golang.org/pkg/net/http/pprof/

来源

2013-03-28 06:58:41

Golang程序内存泄漏？

回答

相关问题