在线观看不卡亚洲电影_亚洲妓女99综合网_91青青青亚洲娱乐在线观看_日韩无码高清综合久久

鍍金池/ 問答/GO/ go協(xié)程請(qǐng)教?

go協(xié)程請(qǐng)教?

需求是這樣的:需要查詢數(shù)據(jù)庫里面的數(shù)據(jù),然后生成對(duì)應(yīng)格式的xml,打算用協(xié)程實(shí)現(xiàn),我的大致思路是這樣的:查詢出總數(shù)據(jù)量,然后根據(jù)配置的每個(gè)xml的數(shù)據(jù)個(gè)數(shù),計(jì)算出需要生成多少個(gè)xml,每個(gè)xml就啟動(dòng)一個(gè)協(xié)程去執(zhí)行生成xml,代碼已經(jīng)寫完,但是經(jīng)常跑一會(huì)就卡主不執(zhí)行了。。不知道為啥,有大神指導(dǎo)一下,或者提供一個(gè)更加好的思路么,數(shù)據(jù)量不多,26萬條左右。。如下是協(xié)程代碼:

func getSiteMaps(db *sql.DB, offset int64, size int64, c chan Sitemap) {
    fmt.Println("offset in channel:", offset)
    sql := "select a.id as outerID,a.name,a.sale_price as price, a.category_id,a.market_price as value,b.name as brand from item as a left join item_brand as b on a.brand_id = b.id where a.status = 1 order by a.id asc limit " + strconv.FormatInt(offset, 10) + "," + strconv.FormatInt(size, 10)
    var r Urlset   //初始化xml
    var urls []Url //初始化urls
    //查詢具體的數(shù)據(jù)
    if rows, rowsErr := db.Query(sql); rowsErr != nil {
        panic(rowsErr)
    } else {
        for rows.Next() {
            var (
                outerID, name, brand, category_id string
                price, value                      float64
            )
            rows.Scan(&outerID, &name, &price, &category_id, &value, &brand)
            if outerID == "" {
                continue
            }
            //loc獲取
            _loc := getLocUrl(outerID)
            //通過categoryId獲取分類信息
            _categoryInfo := getCategoryInfoByParentId(category_id, db)
            // fmt.Println(_categoryInfo)
            // os.Exit(1)
            if _categoryInfo["categoryId"] == "" {
                continue
            }
            //獲取商品圖片
            _imgs := getImages(outerID, db)
            if len(_imgs) <= 0 {
                continue
            }
            _firstImg := ""
            var _moreImages []Images
            for i, v := range _imgs {
                if i == 0 {
                    _firstImg = v
                } else {
                    _moreImages = append(_moreImages, Images{InnerText: v, Index: i})
                }
            }
            moreImages := Mimages{Img: _moreImages}
            // moreImages := _moreImages
            subAttribute := []SubAttribute{SubAttribute{Key: "ext_down_load", Value: "https://m.mia.com/detail-a-" + outerID + ".html"}, SubAttribute{Key: "ext_put_url", Value: "https://m.mia.com/detail-a-" + outerID + ".html"}}
            //Data數(shù)據(jù)
            _choice := Attribute{Attribute: subAttribute}
            itemData := Data{
                OuterID:          outerID,
                Name:             name,
                Price:            price,
                Value:            value,
                PriceUnit:        _PRICE_UNIT,
                Availability:     _AVAILABILITY,
                Image:            _firstImg,
                MoreImages:       moreImages,
                Brand:            brand,
                Loc:              _loc,
                PcLoc:            getPcLoc(outerID),
                SellerSiteUrl:    _SELLER_SITE_URL,
                ShopName:         _SHOP_NAME,
                SearchWiseUrl:    getSearchWiseUrl(outerID),
                Category:         _categoryInfo["categoryName"],
                CategoryUrl:      getCategoryUrl(category_id),
                CategoryPcUrl:    getCategoryPcUrl(category_id),
                SubCategory:      _categoryInfo["subCategoryName"],
                SubCategoryUrl:   getCategoryUrl(_categoryInfo["subCategoryId"]),
                SubcategoryPcUrl: getCategoryPcUrl(_categoryInfo["subCategoryId"]),
                Choice:           _choice,
                SellerName:       _SELLER_NAME,
                Logo:             _LOGO}
            // fmt.Println(itemData)
            // os.Exit(1)
            //url數(shù)據(jù)
            itemUrl := Url{Loc: _loc, Data: itemData}
            //urls數(shù)據(jù)
            urls = append(urls, itemUrl)

        }
        //生成xml數(shù)據(jù)
        r.Url = urls
    }
    //生成xml文件
    if len(urls) > 1 {
        //取數(shù)據(jù),
        fileName := getXmlName(offset, size)
        generateXml(fileName, r, func() {
            xmlUrl := _URL_PREFIX + fileName
            itemSiteMap := Sitemap{Loc: xmlUrl, Lastmod: getCurrentTime()}
            // sitemap = append(sitemap, itemSiteMap)
            //需要停頓幾秒,待寫文件完畢。主要是防止最后一次寫文件沒寫完,程序就執(zhí)行完退出
            time.Sleep(5)
            fmt.Println("done...", offset)
            c <- itemSiteMap
        })
    }
}

如下是main里面的內(nèi)容:

func main() {

    db, dbErr := sql.Open("mysql", _DB)
    if dbErr != nil {
        fmt.Println("db connection err")
        os.Exit(1)
    } else {
        fmt.Println("db ok")
    }
    sql2 := "select count(*) as total from item as a left join item_brand as b on a.brand_id = b.id where a.status = 1"
    var total float64
    if rowCountErr := db.QueryRow(sql2).Scan(&total); rowCountErr != nil {
        panic(rowCountErr)
    } else {
        fmt.Println("total:", total)
        //根據(jù)條數(shù),獲取協(xié)程數(shù)量
        pages := int(math.Ceil(total / _NUM_PER_XML))
        ch := make(chan Sitemap, pages)
        for i := 0; i <= pages; i++ {
            var offset int64
            if i > 0 {
                offset = int64(_NUM_PER_XML*i + 1)
            }
            go getSiteMaps(db, offset, _NUM_PER_XML, ch)
        }
        for v1 := range ch {

            fmt.Println("val:", v1)
        }
    }
}

新手,求指導(dǎo)?

回答
編輯回答
浪婳

cpu密集型的任務(wù),攜程多了作用不大
把攜程數(shù)設(shè)置成cpu數(shù)試試,然后通過channel把xml內(nèi)容傳過去

2017年1月23日 17:32
編輯回答
假灑脫

這樣寫的話,相當(dāng)于在同一刻進(jìn)行g(shù)etSiteMaps動(dòng)作,按你說有26w筆數(shù)據(jù)吧,那就是同一刻進(jìn)行26w次的getSiteMaps操作,你確定你不是在做"壓力測(cè)試"?這都不死機(jī)的話,除非你是天河二號(hào)。

解決你的問題從兩個(gè)方面入手,一個(gè)是開啟多核心支持,一個(gè)是調(diào)整代碼

首先你要在main體里面加上這句runtime.GOMAXPROCS(runtime.NumCPU()),這樣才算用上了多核心,否則程序運(yùn)行時(shí)只會(huì)用1刻核心,哪怕你有128核,也只用1核。
比較復(fù)雜的是代碼方面,建議你實(shí)現(xiàn)個(gè)工人池的代碼,讓工人來做getSiteMaps,并根據(jù)cpu數(shù)量來決定工人數(shù)量,還要做到一個(gè)工人在一個(gè)時(shí)間只做1件事情。

func main() {

    runtime.GOMAXPROCS(runtime.NumCPU())
    
    jobs := make(chan int, 100)
    results := make(chan int, 100)
    
    // 招聘3個(gè)工人,讓他們工作(待命),每個(gè)工人會(huì)從jobs管道里獲得任務(wù),工人干活,把結(jié)果放到results管道
    for w := 1; w <= 3; w++ {
        go worker(w, jobs, results)
    }
    
    // 老板端,實(shí)時(shí)查看任務(wù)結(jié)果
    go func(){
        for v1 := range results {
            fmt.Println("val:", v1)
        }
    }
    
    // 分配任務(wù),共有26w筆任務(wù)
    for j := 1; j <= 26000; j++ {
        jobs <- j
    }
    close(jobs)
}

func worker(id int, jobs <-chan int, results chan<- int) {
    for j := range jobs {
        fmt.Println("worker", id, "started  job", j)
        time.Sleep(time.Second)
        fmt.Println("worker", id, "finished job", j)
        results <- j * 2
    }
}
2017年9月27日 22:48